feat: build vehicle data platform and production pipeline

This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions

View File

@@ -22,20 +22,21 @@ import (
)
type config struct {
MySQLDSN string
TDengineDriver string
TDengineDSN string
TDengineDatabase string
DateFrom string
DateTo string
Protocols []envelope.Protocol
Method string
Limit int
DryRun bool
Reset bool
Debug bool
ProgressEvery int64
Location *time.Location
MySQLDSN string
TDengineDriver string
TDengineDSN string
TDengineDatabase string
DateFrom string
DateTo string
Protocols []envelope.Protocol
Method string
Limit int
DryRun bool
Reset bool
Debug bool
EventTimeFullScan bool
ProgressEvery int64
Location *time.Location
}
type rawFrameRow struct {
@@ -62,6 +63,9 @@ type metricAgg struct {
Phone string
DeviceID string
SourceEndpoint string
SourceCode string
PlatformName string
SourceKind string
FirstEventTime time.Time
LatestEventTime time.Time
QualityStatus string
@@ -74,6 +78,9 @@ type dailySourceLast struct {
Phone string
DeviceID string
SourceEndpoint string
SourceCode string
PlatformName string
SourceKind string
FirstTS time.Time
TS time.Time
FirstTotalKM float64
@@ -81,8 +88,19 @@ type dailySourceLast struct {
RawSampleCount int64
}
type sourceHistoryID struct {
VIN string
SourceKey string
}
var defaultBackfillEnvFiles = []string{
"/opt/lingniu-go-native/env/base.env",
"/opt/lingniu-go-native/env/stat-writer.env",
}
func main() {
if err := loadEnvFiles(env("BACKFILL_ENV_FILES", os.Getenv("BACKFILL_ENV_FILE"))); err != nil {
envFiles := backfillEnvFiles(os.Getenv("BACKFILL_ENV_FILES"), os.Getenv("BACKFILL_ENV_FILE"), defaultBackfillEnvFiles)
if err := loadEnvFiles(envFiles); err != nil {
fail("load env file", err)
}
cfg, err := loadConfig()
@@ -121,18 +139,27 @@ func main() {
slog.Info("reset stats rows", "deleted", deleted)
}
if cfg.Method == "last_diff" {
aggregates, err := buildLastDiffAggregates(ctx, td, cfg)
aggregates, err := buildLastDiffAggregates(ctx, mysqlDB, td, cfg)
if err != nil {
fail("build last-diff aggregates", err)
}
fallbacks, err := addRealtimeLocationFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
if err != nil {
fail("build realtime-location fallback aggregates", err)
}
var written int64
var normalized int
if !cfg.DryRun {
written, err = writeAggregates(ctx, mysqlDB, aggregates, 500)
if err != nil {
fail("write aggregates", err)
}
normalized, err = normalizeHistoricalPlatformSources(ctx, mysqlDB, cfg)
if err != nil {
fail("normalize historical platform sources", err)
}
}
slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "written", written)
slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "written", written, "platformSourcesNormalized", normalized)
return
}
@@ -234,6 +261,7 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
Phone: sample.Phone,
DeviceID: sample.DeviceID,
SourceEndpoint: sample.SourceEndpoint,
PlatformName: sample.PlatformName,
FirstEventTime: sample.EventTime,
LatestEventTime: sample.EventTime,
QualityStatus: stats.QualityOK,
@@ -255,6 +283,9 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
if strings.TrimSpace(sample.DeviceID) != "" {
agg.DeviceID = sample.DeviceID
}
if strings.TrimSpace(sample.PlatformName) != "" {
agg.PlatformName = sample.PlatformName
}
}
agg.Count++
}
@@ -271,6 +302,9 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
Protocol: agg.Protocol,
SourceIP: stats.NormalizeSourceIP(agg.SourceEndpoint),
SourceEndpoint: agg.SourceEndpoint,
SourceCode: agg.SourceCode,
PlatformName: agg.PlatformName,
SourceKind: agg.SourceKind,
}
if strings.TrimSpace(identity.SourceIP) == "" {
continue
@@ -285,7 +319,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
if err := stats.UpsertDataSource(ctx, db, identity, agg.LatestEventTime); err != nil {
return written, err
}
dailyKM := agg.LatestKM - agg.FirstKM
dailyKM := stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM)
candidate := stats.SourceMileageSample{
VIN: agg.VIN,
StatDate: agg.Date,
@@ -295,6 +329,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
SourceEndpoint: agg.SourceEndpoint,
Phone: agg.Phone,
DeviceID: agg.DeviceID,
PlatformName: agg.PlatformName,
FirstTotalKM: agg.FirstKM,
LatestTotalKM: agg.LatestKM,
DailyKM: dailyKM,
@@ -310,10 +345,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
if candidate.QualityReason == "" {
candidate.QualityReason = "same_source_previous_day"
}
if candidate.QualityStatus == stats.QualityOK && (candidate.DailyKM < 0 || candidate.DailyKM > maxTrustedDailyMileageKM) {
candidate.QualityStatus = stats.QualityInvalidDelta
candidate.QualityReason = "outside_daily_range"
}
stats.ApplyMileageQualityRules(&candidate)
if err := stats.UpsertSourceMileage(ctx, db, candidate); err != nil {
return written, err
}
@@ -325,6 +357,27 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
return written, nil
}
func normalizeHistoricalPlatformSources(ctx context.Context, db *sql.DB, cfg config) (int, error) {
dates, err := dateRange(cfg.DateFrom, cfg.DateTo)
if err != nil {
return 0, err
}
normalized := 0
for _, protocol := range cfg.Protocols {
if protocol != envelope.ProtocolJT808 {
continue
}
for _, date := range dates {
count, err := stats.NormalizePlatformSourceMileageForDate(ctx, db, date, protocol)
if err != nil {
return normalized, err
}
normalized += count
}
}
return normalized, nil
}
func clearBackfillTargetMileage(ctx context.Context, db *sql.DB, vin string, statDate string, protocol envelope.Protocol) error {
if db == nil || strings.TrimSpace(vin) == "" || strings.TrimSpace(statDate) == "" || strings.TrimSpace(string(protocol)) == "" {
return nil
@@ -337,47 +390,173 @@ func clearBackfillTargetMileage(ctx context.Context, db *sql.DB, vin string, sta
return err
}
func buildLastDiffAggregates(ctx context.Context, db *sql.DB, cfg config) (map[string]*metricAgg, error) {
func buildLastDiffAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config) (map[string]*metricAgg, error) {
targetDates, err := dateRange(cfg.DateFrom, cfg.DateTo)
if err != nil {
return nil, err
}
aggregates := map[string]*metricAgg{}
for _, protocol := range cfg.Protocols {
latestHistory := map[sourceHistoryID]dailySourceLast{}
preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
if err != nil {
return nil, err
}
rememberLatestSourceRows(latestHistory, preWindow)
slog.Info("pre-window baseline loaded", "protocol", protocol, "before_date", targetDates[0], "vehicles", len(preWindow))
for _, date := range targetDates {
current, err := queryDailyLastSourceRows(ctx, db, cfg, protocol, date)
current, err := queryDailyLastSourceRows(ctx, tdDB, cfg, protocol, date)
if err != nil {
return nil, err
}
previous, err := queryPreviousLastSourceRows(ctx, db, cfg, protocol, date)
if err != nil {
return nil, err
}
slog.Info("daily last loaded", "protocol", protocol, "date", date, "vehicles", len(current), "previousVehicles", len(previous))
slog.Info("daily last loaded", "protocol", protocol, "date", date, "vehicles", len(current), "historicalSources", len(latestHistory))
for vin, currentSources := range current {
previousBySource := map[string]dailySourceLast{}
for _, previousRow := range previous[vin] {
previousBySource[previousRow.SourceKey] = previousRow
}
for _, currentRow := range currentSources {
key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey
previousRow, hasPrevious := previousBySource[currentRow.SourceKey]
previousRow, hasPrevious, err := resolvePreviousSourceRow(ctx, mysqlDB, date, protocol, latestHistory, currentRow)
if err != nil {
return nil, err
}
aggregates[key] = aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious)
}
}
// Current-day last values become eligible only for later target dates.
rememberLatestSourceRows(latestHistory, current)
}
}
return aggregates, nil
}
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
if aggregates == nil {
return 0, fmt.Errorf("aggregates map is nil")
}
targetDates, err := dateRange(cfg.DateFrom, cfg.DateTo)
if err != nil {
return 0, err
}
var added int
for _, protocol := range cfg.Protocols {
if protocol != envelope.ProtocolYutongMQTT {
continue
}
latestHistory := map[sourceHistoryID]dailySourceLast{}
preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
if err != nil {
return added, err
}
rememberLatestSourceRows(latestHistory, preWindow)
aggregateRowsByDate := indexAggregateSourceRowsByDate(aggregates, protocol)
for _, date := range targetDates {
current, err := queryRealtimeLocationLastRows(ctx, mysqlDB, cfg, protocol, date)
if err != nil {
return added, err
}
for vin, currentSources := range current {
for _, currentRow := range currentSources {
key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey
if _, exists := aggregates[key]; exists {
continue
}
previousRow, hasPrevious, err := resolvePreviousSourceRow(ctx, mysqlDB, date, protocol, latestHistory, currentRow)
if err != nil {
return added, err
}
agg := aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious)
if hasPrevious {
agg.QualityReason = "realtime_location_fallback_historical_baseline"
} else {
agg.QualityReason = "realtime_location_fallback_current_day_first_baseline"
}
aggregates[key] = agg
added++
}
}
rememberLatestSourceRows(latestHistory, aggregateRowsByDate[date])
rememberLatestSourceRows(latestHistory, current)
if len(current) > 0 {
slog.Info("realtime location fallback loaded", "protocol", protocol, "date", date, "vehicles", len(current), "added", added)
}
}
}
return added, nil
}
func indexAggregateSourceRowsByDate(aggregates map[string]*metricAgg, protocol envelope.Protocol) map[string]map[string][]dailySourceLast {
indexed := map[string]map[string][]dailySourceLast{}
for _, agg := range aggregates {
if agg == nil || agg.Protocol != protocol || strings.TrimSpace(agg.Date) == "" {
continue
}
rows := indexed[agg.Date]
if rows == nil {
rows = map[string][]dailySourceLast{}
indexed[agg.Date] = rows
}
rows[agg.VIN] = append(rows[agg.VIN], dailySourceLast{
VIN: agg.VIN,
SourceKey: agg.SourceKey,
Phone: agg.Phone,
DeviceID: agg.DeviceID,
SourceEndpoint: agg.SourceEndpoint,
SourceCode: agg.SourceCode,
PlatformName: agg.PlatformName,
SourceKind: agg.SourceKind,
FirstTS: agg.LatestEventTime,
TS: agg.LatestEventTime,
FirstTotalKM: agg.LatestKM,
TotalKM: agg.LatestKM,
RawSampleCount: agg.Count,
})
}
return indexed
}
func rememberLatestSourceRows(history map[sourceHistoryID]dailySourceLast, rows map[string][]dailySourceLast) {
for vin, sourceRows := range rows {
for _, row := range sourceRows {
rowVIN := strings.TrimSpace(row.VIN)
if rowVIN == "" {
rowVIN = strings.TrimSpace(vin)
row.VIN = rowVIN
}
id := sourceHistoryID{VIN: rowVIN, SourceKey: strings.TrimSpace(row.SourceKey)}
if id.VIN == "" || id.SourceKey == "" || row.TS.IsZero() {
continue
}
if existing, ok := history[id]; !ok || row.TS.After(existing.TS) {
history[id] = row
}
}
}
}
func latestHistoricalSourceRow(history map[sourceHistoryID]dailySourceLast, current dailySourceLast) (dailySourceLast, bool) {
id := sourceHistoryID{
VIN: strings.TrimSpace(current.VIN),
SourceKey: strings.TrimSpace(current.SourceKey),
}
previous, ok := history[id]
return previous, ok && previous.TS.Before(current.TS)
}
func resolvePreviousSourceRow(ctx context.Context, db *sql.DB, date string, protocol envelope.Protocol, history map[sourceHistoryID]dailySourceLast, current dailySourceLast) (dailySourceLast, bool, error) {
if previous, ok := latestHistoricalSourceRow(history, current); ok {
return previous, true, nil
}
return queryDurablePreviousSourceRow(ctx, db, date, protocol, current)
}
func aggregateFromDailySource(date string, protocol envelope.Protocol, current dailySourceLast, previous dailySourceLast, hasPrevious bool) *metricAgg {
firstKM := current.FirstTotalKM
firstEventTime := current.FirstTS
qualityReason := "current_day_first_sample"
qualityStatus := stats.QualityOK
qualityReason := stats.QualityReasonCurrentDayFirst
if hasPrevious {
firstKM = previous.TotalKM
firstEventTime = previous.TS
qualityReason = "historical_source_baseline"
qualityStatus = stats.QualityOK
qualityReason = stats.QualityReasonHistorical
}
if firstEventTime.IsZero() {
firstEventTime = current.TS
@@ -386,7 +565,7 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d
if count <= 0 {
count = 1
}
return &metricAgg{
agg := &metricAgg{
VIN: current.VIN,
Date: date,
Protocol: protocol,
@@ -397,11 +576,43 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d
Phone: current.Phone,
DeviceID: current.DeviceID,
SourceEndpoint: current.SourceEndpoint,
SourceCode: current.SourceCode,
PlatformName: current.PlatformName,
SourceKind: current.SourceKind,
FirstEventTime: firstEventTime,
LatestEventTime: current.TS,
QualityStatus: stats.QualityOK,
QualityStatus: qualityStatus,
QualityReason: qualityReason,
}
candidate := stats.SourceMileageSample{
FirstTotalKM: agg.FirstKM,
LatestTotalKM: agg.LatestKM,
DailyKM: stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM),
FirstEventTime: agg.FirstEventTime,
LatestEventTime: agg.LatestEventTime,
QualityStatus: agg.QualityStatus,
QualityReason: agg.QualityReason,
}
stats.ApplyMileageQualityRules(&candidate)
agg.QualityStatus = candidate.QualityStatus
agg.QualityReason = candidate.QualityReason
return agg
}
func queryDurablePreviousSourceRow(ctx context.Context, db *sql.DB, date string, protocol envelope.Protocol, current dailySourceLast) (dailySourceLast, bool, error) {
if db == nil {
return dailySourceLast{}, false, nil
}
totalKM, eventTime, found, err := stats.LookupLatestSourceBaselineBefore(ctx, db, current.VIN, date, protocol, current.SourceKey)
if err != nil || !found {
return dailySourceLast{}, found, err
}
previous := current
previous.FirstTS = eventTime
previous.TS = eventTime
previous.FirstTotalKM = totalKM
previous.TotalKM = totalKM
return previous, true, nil
}
type trustedChoice struct {
@@ -409,8 +620,6 @@ type trustedChoice struct {
previous dailySourceLast
}
const maxTrustedDailyMileageKM = 1000
func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast) (trustedChoice, bool) {
previousBySource := map[string]dailySourceLast{}
for _, row := range previous {
@@ -423,8 +632,8 @@ func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast)
if !ok {
continue
}
delta := currentRow.TotalKM - previousRow.TotalKM
if delta < 0 || delta > maxTrustedDailyMileageKM {
delta, ok, _ := stats.NormalizeDailyMileageDeltaForWindow(stats.DailyMileageFromDayBoundary(previousRow.TotalKM, currentRow.TotalKM), previousRow.TS, currentRow.TS)
if !ok {
continue
}
if chosen.current.SourceKey == "" || delta < chosenDelta {
@@ -436,19 +645,18 @@ func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast)
}
func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
where := []string{
fmt.Sprintf("ts >= '%s 00:00:00'", quote(date)),
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(date))),
where := backfillTimePredicates(cfg, date, nextDate(date))
where = append(where,
"parse_status = 'OK'",
"vin IS NOT NULL",
"vin <> ''",
fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
realtimeMileageFramePredicate(),
}
)
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(ts), FIRST(parsed_json), LAST(ts), LAST(parsed_json), COUNT(*)
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(event_time), FIRST(parsed_json), LAST(event_time), LAST(parsed_json), COUNT(*)
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
@@ -456,24 +664,103 @@ GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), s
}
func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
where := []string{
fmt.Sprintf("ts < '%s 00:00:00'", quote(date)),
// The baseline is the nearest earlier sample, not necessarily yesterday's.
// LAST aggregates the complete pre-window history once per source so empty
// calendar days do not make a backfill fall back to the current day's first
// sample.
where := backfillBeforePredicates(date)
where = append(where,
"parse_status = 'OK'",
"vin IS NOT NULL",
"vin <> ''",
fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
realtimeMileageFramePredicate(),
}
)
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(ts), LAST(parsed_json), COUNT(*)
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(event_time), LAST(parsed_json), COUNT(*)
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
}
func backfillBeforePredicates(eventDateExclusive string) []string {
return []string{
fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateExclusive)),
}
}
func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
if db == nil {
return nil, nil
}
loc := cfg.Location
if loc == nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
sqlText := `SELECT l.vin, COALESCE(NULLIF(s.peer, ''), ''), l.total_mileage_event_time, l.total_mileage_km
FROM vehicle_realtime_location l
LEFT JOIN vehicle_realtime_snapshot s ON s.protocol = l.protocol AND s.vin = l.vin
WHERE l.protocol = ?
AND l.vin IS NOT NULL AND l.vin <> ''
AND l.total_mileage_km IS NOT NULL AND l.total_mileage_km > 0
AND l.total_mileage_event_time >= ?
AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY)`
rows, err := db.QueryContext(ctx, sqlText, string(protocol), date, date)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string][]dailySourceLast{}
for rows.Next() {
var vin string
var peer sql.NullString
var eventTime time.Time
var totalKM float64
if err := rows.Scan(&vin, &peer, &eventTime, &totalKM); err != nil {
return nil, err
}
vin = strings.TrimSpace(vin)
if vin == "" || totalKM <= 0 {
continue
}
sourceEndpoint := strings.TrimSpace(peer.String)
if sourceEndpoint == "" && protocol == envelope.ProtocolYutongMQTT {
sourceEndpoint = "mqtt://yutong/realtime-location"
}
deviceID := ""
if protocol == envelope.ProtocolYutongMQTT {
deviceID = vin
}
sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
sourceKey := normalizedSourceKeyForSource(string(protocol), "", deviceID, sourceEndpoint, sourceKind, sourceCode)
if sourceKey == "" {
continue
}
row := dailySourceLast{
VIN: vin,
SourceKey: sourceKey,
DeviceID: deviceID,
SourceEndpoint: sourceEndpoint,
SourceCode: sourceCode,
PlatformName: platformName,
SourceKind: sourceKind,
FirstTS: eventTime.In(loc),
TS: eventTime.In(loc),
FirstTotalKM: totalKM,
TotalKM: totalKM,
RawSampleCount: 1,
}
out[vin] = append(out[vin], row)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool) (map[string][]dailySourceLast, error) {
rows, err := db.QueryContext(ctx, sqlText)
if err != nil {
@@ -512,7 +799,8 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
firstTotalKM = parsedFirst
}
}
sourceKey := normalizedSourceKey(string(protocol), phone, deviceID, sourceEndpoint)
sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
sourceKey := normalizedSourceKeyForSource(string(protocol), phone, deviceID, sourceEndpoint, sourceKind, sourceCode)
if sourceKey == "" {
continue
}
@@ -522,8 +810,11 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
Phone: strings.TrimSpace(phone),
DeviceID: strings.TrimSpace(deviceID),
SourceEndpoint: strings.TrimSpace(sourceEndpoint),
SourceCode: sourceCode,
PlatformName: platformName,
SourceKind: sourceKind,
FirstTS: firstTS.In(cfg.Location),
TS: ts,
TS: ts.In(cfg.Location),
FirstTotalKM: firstTotalKM,
TotalKM: latestTotalKM,
RawSampleCount: rawSampleCount,
@@ -562,8 +853,19 @@ func mileageFromParsed(protocol envelope.Protocol, vin string, parsedJSON string
}
func normalizedSourceKey(protocol string, phone string, deviceID string, endpoint string) string {
return normalizedSourceKeyForSource(protocol, phone, deviceID, endpoint, "", "")
}
func normalizedSourceKeyForSource(protocol string, phone string, deviceID string, endpoint string, sourceKind string, sourceCode string) string {
sourceIP := stats.NormalizeSourceIP(endpoint)
return stats.SourceKey(envelope.Protocol(protocol), phone, deviceID, sourceIP)
return stats.SourceKeyForSource(envelope.Protocol(protocol), phone, deviceID, sourceIP, sourceKind, sourceCode)
}
func knownPlatformSourceMetadata(protocol envelope.Protocol, endpoint string) (sourceCode string, platformName string, sourceKind string) {
if protocol == envelope.ProtocolYutongMQTT && stats.NormalizeSourceIP(endpoint) == "mqtt" {
return "yutong", "宇通", "PLATFORM"
}
return "", "", ""
}
func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[string]any {
@@ -574,7 +876,7 @@ func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[str
}
parsed := map[string]any{}
if err := json.Unmarshal([]byte(text), &parsed); err == nil && len(parsed) > 0 {
if flattened, _, ok := realtime.ParsedFieldsForEnvelope(envelope.FrameEnvelope{
if flattened, _, ok := realtime.ComputeParsedFieldsForEnvelope(envelope.FrameEnvelope{
Protocol: protocol,
VIN: vin,
Parsed: parsed,
@@ -604,7 +906,12 @@ func mileageKeys(protocol envelope.Protocol) []string {
case envelope.ProtocolJT808:
return []string{"jt808.location.total_mileage_km"}
case envelope.ProtocolYutongMQTT:
return []string{"yutong_mqtt.data.total_mileage", "yutong_mqtt.root.data.total_mileage"}
return []string{
"yutong_mqtt.data.total_mileage_km",
"yutong_mqtt.root.data.total_mileage_km",
"yutong_mqtt.data.total_mileage",
"yutong_mqtt.root.data.total_mileage",
}
default:
return nil
}
@@ -624,38 +931,67 @@ func loadConfig() (config, error) {
if err != nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
dateTo := env("BACKFILL_DATE_TO", time.Now().In(loc).Format("2006-01-02"))
dateFrom := env("BACKFILL_DATE_FROM", dateTo)
dateFrom, dateTo := resolveBackfillDateRange(time.Now(), loc)
protocols, err := parseProtocols(env("BACKFILL_PROTOCOLS", "GB32960,JT808,YUTONG_MQTT"))
if err != nil {
return config{}, err
}
return config{
MySQLDSN: env("MYSQL_DSN", ""),
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
TDengineDSN: env("TDENGINE_DSN", ""),
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
DateFrom: dateFrom,
DateTo: dateTo,
Protocols: protocols,
Method: env("BACKFILL_METHOD", "last_diff"),
Limit: envInt("BACKFILL_LIMIT", 0),
DryRun: envBool("BACKFILL_DRY_RUN", true),
Reset: envBool("BACKFILL_RESET", false),
Debug: envBool("BACKFILL_DEBUG", false),
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
Location: loc,
MySQLDSN: env("MYSQL_DSN", ""),
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
TDengineDSN: env("TDENGINE_DSN", ""),
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
DateFrom: dateFrom,
DateTo: dateTo,
Protocols: protocols,
Method: env("BACKFILL_METHOD", "last_diff"),
Limit: envInt("BACKFILL_LIMIT", 0),
DryRun: envBool("BACKFILL_DRY_RUN", true),
Reset: envBool("BACKFILL_RESET", false),
Debug: envBool("BACKFILL_DEBUG", false),
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
Location: loc,
}, nil
}
func resolveBackfillDateRange(now time.Time, loc *time.Location) (string, string) {
if loc == nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
explicitFrom := strings.TrimSpace(os.Getenv("BACKFILL_DATE_FROM"))
explicitTo := strings.TrimSpace(os.Getenv("BACKFILL_DATE_TO"))
if explicitTo != "" || explicitFrom != "" {
dateTo := explicitTo
if dateTo == "" {
dateTo = now.In(loc).Format("2006-01-02")
}
dateFrom := explicitFrom
if dateFrom == "" {
dateFrom = dateTo
}
return dateFrom, dateTo
}
daysBack := envInt("BACKFILL_DAYS_BACK", 0)
if daysBack < 0 {
daysBack = 0
}
windowDays := envInt("BACKFILL_WINDOW_DAYS", 1)
if windowDays < 1 {
windowDays = 1
}
dateToTime := now.In(loc).AddDate(0, 0, -daysBack)
dateFromTime := dateToTime.AddDate(0, 0, -(windowDays - 1))
return dateFromTime.Format("2006-01-02"), dateToTime.Format("2006-01-02")
}
func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, error) {
where := []string{
fmt.Sprintf("ts >= '%s 00:00:00'", quote(cfg.DateFrom)),
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(cfg.DateTo))),
where := backfillTimePredicates(cfg, cfg.DateFrom, nextDate(cfg.DateTo))
where = append(where,
"parse_status = 'OK'",
"vin IS NOT NULL",
"vin <> ''",
}
)
if len(cfg.Protocols) > 0 {
quoted := make([]string, 0, len(cfg.Protocols))
for _, protocol := range cfg.Protocols {
@@ -667,7 +1003,7 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json
FROM %s.raw_frames
WHERE %s
ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
ORDER BY event_time ASC, ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
if cfg.Limit > 0 {
sqlText += fmt.Sprintf(" LIMIT %d", cfg.Limit)
}
@@ -675,6 +1011,23 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
return db.QueryContext(ctx, sqlText)
}
func backfillTimePredicates(cfg config, eventDateFrom string, eventDateToExclusive string) []string {
where := make([]string, 0, 4)
if !cfg.EventTimeFullScan {
// ts is the TDengine primary timestamp and may be adjusted slightly to keep
// rows unique. Use it only as a broad index-friendly window; event_time is
// the protocol business boundary used for the final natural-day result.
where = append(where,
fmt.Sprintf("ts >= '%s 00:00:00'", quote(previousDate(eventDateFrom))),
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(eventDateToExclusive))),
)
}
return append(where,
fmt.Sprintf("event_time >= '%s 00:00:00'", quote(eventDateFrom)),
fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateToExclusive)),
)
}
func realtimeMileageFramePredicate() string {
return `(
(protocol = 'GB32960' AND message_id IN (2,3))
@@ -684,17 +1037,39 @@ func realtimeMileageFramePredicate() string {
}
func mileageBearingFramePredicate(protocol envelope.Protocol) string {
keys := mileageKeys(protocol)
if len(keys) == 0 {
tokens := mileageSearchTokens(protocol)
if len(tokens) == 0 {
return ""
}
conditions := make([]string, 0, len(keys))
for _, key := range keys {
conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(key)))
conditions := make([]string, 0, len(tokens))
for _, token := range tokens {
conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(token)))
}
return "(" + strings.Join(conditions, " OR ") + ")"
}
func mileageSearchTokens(protocol envelope.Protocol) []string {
tokens := append([]string{}, mileageKeys(protocol)...)
switch protocol {
case envelope.ProtocolYutongMQTT:
tokens = append(tokens, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km")
}
seen := make(map[string]struct{}, len(tokens))
out := make([]string, 0, len(tokens))
for _, token := range tokens {
token = strings.TrimSpace(token)
if token == "" {
continue
}
if _, ok := seen[token]; ok {
continue
}
seen[token] = struct{}{}
out = append(out, token)
}
return out
}
func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed string
var messageID int64
@@ -804,6 +1179,26 @@ func loadEnvFiles(paths string) error {
return nil
}
func backfillEnvFiles(explicitFiles string, explicitFile string, defaults []string) string {
if value := strings.TrimSpace(explicitFiles); value != "" {
return value
}
if value := strings.TrimSpace(explicitFile); value != "" {
return value
}
var existing []string
for _, path := range defaults {
path = strings.TrimSpace(path)
if path == "" {
continue
}
if _, err := os.Stat(path); err == nil {
existing = append(existing, path)
}
}
return strings.Join(existing, ",")
}
func loadEnvFile(path string) error {
path = strings.TrimSpace(path)
if path == "" {

View File

@@ -2,6 +2,9 @@ package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -47,6 +50,58 @@ func TestChooseTrustedSourceKeepsContinuingSourceAndRejectsNewJump(t *testing.T)
}
}
func TestChooseTrustedSourceAcceptsSmallNegativeMileageJitter(t *testing.T) {
previous := []dailySourceLast{{
VIN: "LB9A32A28R0LS1574",
SourceKey: normalizedSourceKey("JT808", "64115156034", "", "115.159.85.149:53330"),
Phone: "64115156034",
SourceEndpoint: "115.159.85.149:53330",
TotalKM: 15355.4,
}}
current := []dailySourceLast{{
VIN: "LB9A32A28R0LS1574",
SourceKey: normalizedSourceKey("JT808", "64115156034", "", "115.159.85.149:53338"),
Phone: "64115156034",
SourceEndpoint: "115.159.85.149:53338",
TotalKM: 15355.3,
}}
chosen, ok := chooseTrustedSource(current, previous)
if !ok {
t.Fatal("chooseTrustedSource() should accept tiny negative mileage jitter")
}
if chosen.current.SourceKey != current[0].SourceKey {
t.Fatalf("chosen source = %q", chosen.current.SourceKey)
}
}
func TestChooseTrustedSourceAcceptsPlausibleMultiDayFallbackDelta(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
sourceKey := normalizedSourceKey("GB32960", "", "", "8.134.95.166:37720")
previous := []dailySourceLast{{
VIN: "LNXNEGRR1SR319498",
SourceKey: sourceKey,
SourceEndpoint: "8.134.95.166:37720",
TS: time.Date(2026, 7, 4, 11, 7, 58, 0, loc),
TotalKM: 8832.1,
}}
current := []dailySourceLast{{
VIN: "LNXNEGRR1SR319498",
SourceKey: sourceKey,
SourceEndpoint: "8.134.95.166:37720",
TS: time.Date(2026, 7, 12, 2, 52, 47, 0, loc),
TotalKM: 16665.6,
}}
chosen, ok := chooseTrustedSource(current, previous)
if !ok {
t.Fatal("chooseTrustedSource() should accept delta within the historical baseline window")
}
if delta := chosen.current.TotalKM - chosen.previous.TotalKM; delta < 7833.4 || delta > 7833.6 {
t.Fatalf("delta = %v, want historical gap delta", delta)
}
}
func TestDailySourceLastBuildsCandidateKeysBySourceIP(t *testing.T) {
sourceA := dailySourceLast{
VIN: "LA9GG64L7PBAF4001",
@@ -96,7 +151,7 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
if agg.FirstEventTime != previous.TS || agg.LatestEventTime != current.TS {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != "historical_source_baseline" {
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonHistorical {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
if agg.Count != 15 {
@@ -104,7 +159,40 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
}
}
func TestAggregateFromDailySourceUsesCurrentFirstSampleWithoutHistory(t *testing.T) {
func TestAggregateFromDailySourceRejectsHistoricalBaselineJump(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
current := dailySourceLast{
VIN: "LNXNEGRR6SR319464",
SourceKey: normalizedSourceKey("GB32960", "", "", "8.134.95.166:49206"),
SourceEndpoint: "8.134.95.166:49206",
FirstTS: time.Date(2026, 7, 12, 8, 5, 42, 0, loc),
TS: time.Date(2026, 7, 12, 10, 44, 56, 0, loc),
FirstTotalKM: 28004.2,
TotalKM: 40009.7,
RawSampleCount: 1938,
}
previous := dailySourceLast{
VIN: current.VIN,
SourceKey: current.SourceKey,
SourceEndpoint: current.SourceEndpoint,
TS: time.Date(2026, 7, 3, 19, 0, 39, 0, loc),
TotalKM: 10009.7,
}
agg := aggregateFromDailySource("2026-07-12", envelope.ProtocolGB32960, current, previous, true)
if agg.FirstKM != previous.TotalKM || agg.LatestKM != current.TotalKM {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if !agg.FirstEventTime.Equal(previous.TS) || !agg.LatestEventTime.Equal(current.TS) {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityInvalidDelta || agg.QualityReason != "outside_daily_range" {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
}
func TestAggregateFromDailySourceUsesCurrentDayFirstWhenHistoryIsEmpty(t *testing.T) {
current := dailySourceLast{
VIN: "LMRKH9AC2R1004087",
SourceKey: normalizedSourceKey("YUTONG_MQTT", "", "LMRKH9AC2R1004087", "mqtt://yutong/ytforward/shln/3"),
@@ -125,11 +213,264 @@ func TestAggregateFromDailySourceUsesCurrentFirstSampleWithoutHistory(t *testing
if agg.FirstEventTime != current.FirstTS || agg.LatestEventTime != current.TS {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != "current_day_first_sample" {
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonCurrentDayFirst {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
}
func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.T) {
tdDB, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer tdDB.Close()
mock.MatchExpectationsInOrder(true)
loc := time.FixedZone("Asia/Shanghai", 8*3600)
vin := "LMRKH9AC2R1004087"
endpoint := "mqtt://yutong/ytforward/shln/3"
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
currentRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
})
}
previousRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
})
}
mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(previousRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, int64(4)))
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, int64(6)))
aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
DateFrom: "2026-07-09",
DateTo: "2026-07-11",
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
Location: loc,
})
if err != nil {
t.Fatalf("buildLastDiffAggregates() error = %v", err)
}
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong")
agg := aggregates[vin+"|2026-07-11|YUTONG_MQTT|"+sourceKey]
if agg == nil {
t.Fatalf("missing day-three aggregate; keys=%v", aggregateKeys(aggregates))
}
if agg.FirstKM != 100 || agg.LatestKM != 120 {
t.Fatalf("km range = %v -> %v, want 100 -> 120", agg.FirstKM, agg.LatestKM)
}
if !agg.FirstEventTime.Equal(dayOneTS) || agg.QualityReason != stats.QualityReasonHistorical {
t.Fatalf("baseline = %v reason=%q, want day-one historical baseline", agg.FirstEventTime, agg.QualityReason)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func aggregateKeys(aggregates map[string]*metricAgg) []string {
keys := make([]string, 0, len(aggregates))
for key := range aggregates {
keys = append(keys, key)
}
return keys
}
func TestQueryRealtimeLocationLastRowsBuildsYutongSourceFromPeer(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
eventTime := time.Date(2026, 7, 12, 5, 54, 50, 0, loc)
mock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*LEFT JOIN vehicle_realtime_snapshot s").
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", eventTime, 11578.0))
rows, err := queryRealtimeLocationLastRows(context.Background(), db, config{Location: loc}, envelope.ProtocolYutongMQTT, "2026-07-12")
if err != nil {
t.Fatalf("queryRealtimeLocationLastRows() error = %v", err)
}
sourceRows := rows["LMRKH9AC0R1004086"]
if len(sourceRows) != 1 {
t.Fatalf("rows = %d, want 1", len(sourceRows))
}
row := sourceRows[0]
wantSourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong")
if row.SourceKey != wantSourceKey {
t.Fatalf("source key = %q", row.SourceKey)
}
if row.SourceCode != "yutong" || row.PlatformName != "宇通" || row.SourceKind != "PLATFORM" {
t.Fatalf("source metadata = code:%q platform:%q kind:%q", row.SourceCode, row.PlatformName, row.SourceKind)
}
if row.TotalKM != 11578 || row.DeviceID != "LMRKH9AC0R1004086" || row.RawSampleCount != 1 {
t.Fatalf("unexpected realtime fallback row: %#v", row)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.T) {
mysqlDB, mysqlMock, err := sqlmock.New()
if err != nil {
t.Fatalf("mysql sqlmock.New() error = %v", err)
}
defer mysqlDB.Close()
tdDB, tdMock, err := sqlmock.New()
if err != nil {
t.Fatalf("td sqlmock.New() error = %v", err)
}
defer tdDB.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
currentTS := time.Date(2026, 7, 12, 5, 54, 50, 0, loc)
previousTS := time.Date(2026, 7, 11, 23, 58, 0, 0, loc)
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", currentTS, 11578.0))
tdMock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-12 00:00:00'.*protocol = 'YUTONG_MQTT'.*TOTAL_MILEAGE").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC0R1004086",
"",
"LMRKH9AC0R1004086",
"mqtt://yutong/ytforward/shln/4",
previousTS,
`{"data":{"TOTAL_MILEAGE":11500000}}`,
int64(12),
))
aggregates := map[string]*metricAgg{}
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
DateFrom: "2026-07-12",
DateTo: "2026-07-12",
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
Location: loc,
}, aggregates)
if err != nil {
t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err)
}
if added != 1 || len(aggregates) != 1 {
t.Fatalf("added=%d aggregates=%d", added, len(aggregates))
}
for _, agg := range aggregates {
if agg.FirstKM != 11500 || agg.LatestKM != 11578 {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if agg.SourceKey != stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong") {
t.Fatalf("source key = %q", agg.SourceKey)
}
if agg.SourceCode != "yutong" || agg.PlatformName != "宇通" || agg.SourceKind != "PLATFORM" {
t.Fatalf("source metadata = code:%q platform:%q kind:%q", agg.SourceCode, agg.PlatformName, agg.SourceKind)
}
if agg.QualityReason != "realtime_location_fallback_historical_baseline" {
t.Fatalf("quality reason = %q", agg.QualityReason)
}
}
if err := mysqlMock.ExpectationsWereMet(); err != nil {
t.Fatalf("mysql sql expectations: %v", err)
}
if err := tdMock.ExpectationsWereMet(); err != nil {
t.Fatalf("td sql expectations: %v", err)
}
}
func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *testing.T) {
mysqlDB, mysqlMock, err := sqlmock.New()
if err != nil {
t.Fatalf("mysql sqlmock.New() error = %v", err)
}
defer mysqlDB.Close()
tdDB, tdMock, err := sqlmock.New()
if err != nil {
t.Fatalf("td sqlmock.New() error = %v", err)
}
defer tdDB.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
vin := "LMRKH9AC2R1004087"
endpoint := "mqtt://yutong/ytforward/shln/3"
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong")
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
tdMock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}))
for _, date := range []string{"2026-07-09", "2026-07-10"} {
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
WithArgs("YUTONG_MQTT", date, date).
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}))
}
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
WithArgs("YUTONG_MQTT", "2026-07-11", "2026-07-11").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
AddRow(vin, endpoint, dayThreeTS, 120.0))
aggregates := map[string]*metricAgg{
vin + "|2026-07-09|YUTONG_MQTT|" + sourceKey: {
VIN: vin,
Date: "2026-07-09",
Protocol: envelope.ProtocolYutongMQTT,
LatestKM: 100,
Count: 4,
SourceKey: sourceKey,
DeviceID: vin,
SourceEndpoint: endpoint,
SourceCode: "yutong",
PlatformName: "宇通",
SourceKind: "PLATFORM",
LatestEventTime: dayOneTS,
},
}
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
DateFrom: "2026-07-09",
DateTo: "2026-07-11",
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
Location: loc,
}, aggregates)
if err != nil {
t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err)
}
if added != 1 || len(aggregates) != 2 {
t.Fatalf("added=%d aggregates=%d, want 1 and 2", added, len(aggregates))
}
agg := aggregates[vin+"|2026-07-11|YUTONG_MQTT|"+sourceKey]
if agg == nil {
t.Fatal("missing realtime-location fallback aggregate")
}
if agg.FirstKM != 100 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayOneTS) {
t.Fatalf("fallback range = %v@%v -> %v, want 100@day-one -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM)
}
if agg.QualityReason != "realtime_location_fallback_historical_baseline" {
t.Fatalf("quality reason = %q", agg.QualityReason)
}
if err := mysqlMock.ExpectationsWereMet(); err != nil {
t.Fatalf("mysql sql expectations: %v", err)
}
if err := tdMock.ExpectationsWereMet(); err != nil {
t.Fatalf("td sql expectations: %v", err)
}
}
func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
@@ -140,9 +481,9 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
firstTS := time.Date(2026, 7, 8, 8, 0, 0, 0, loc)
lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'").
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(ts)", "FIRST(parsed_json)", "LAST(ts)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
@@ -183,9 +524,9 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'").
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(ts)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
@@ -215,6 +556,89 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
}
}
func TestBackfillBeforePredicatesSearchesAllEarlierHistory(t *testing.T) {
where := strings.Join(backfillBeforePredicates("2026-07-08"), " AND ")
if !strings.Contains(where, "event_time < '2026-07-08 00:00:00'") {
t.Fatalf("pre-window predicate missing exclusive upper bound: %s", where)
}
if strings.Contains(where, "event_time >=") || strings.Contains(where, "ts >=") {
t.Fatalf("pre-window predicate must not stop at the previous day: %s", where)
}
}
func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC7R1004098",
"",
"LMRKH9AC7R1004098",
"mqtt://yutong/ytforward/shln/4",
utcInstant,
`{"yutong_mqtt.data.total_mileage":"41249000"}`,
int64(1),
))
rows, err := queryPreviousLastSourceRows(context.Background(), db, config{
TDengineDatabase: "lingniu_vehicle_ts",
Location: loc,
}, envelope.ProtocolYutongMQTT, "2026-07-13")
if err != nil {
t.Fatalf("queryPreviousLastSourceRows() error = %v", err)
}
sourceRows := rows["LMRKH9AC7R1004098"]
if len(sourceRows) != 1 {
t.Fatalf("rows = %d, want 1", len(sourceRows))
}
want := time.Date(2026, 7, 12, 23, 59, 59, 0, loc)
if !sourceRows[0].TS.Equal(want) || sourceRows[0].TS.Location().String() != loc.String() {
t.Fatalf("previous event time = %s (%s), want %s (%s)", sourceRows[0].TS, sourceRows[0].TS.Location(), want, want.Location())
}
}
func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
fields := fieldsForStats(envelope.ProtocolYutongMQTT, "LMRKH9AC6R1004108", `{
"data": {
"TOTAL_MILEAGE": 65423000,
"METER_SPEED": 12.3
},
"root": {
"device": "LMRKH9AC6R1004108"
}
}`)
if got := fields["yutong_mqtt.data.total_mileage"]; got == nil {
t.Fatalf("fields missing raw yutong total mileage: %#v", fields)
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
VIN: "LMRKH9AC6R1004108",
EventTimeMS: time.Date(2026, 7, 8, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
ReceivedAtMS: time.Date(2026, 7, 8, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
Fields: fields,
}
samples, err := stats.SamplesFromEnvelope(env, time.FixedZone("Asia/Shanghai", 8*3600))
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 1 {
t.Fatalf("samples = %d, want 1", len(samples))
}
if samples[0].TotalMileageKM != 65423 {
t.Fatalf("total mileage km = %v, want 65423", samples[0].TotalMileageKM)
}
}
func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
@@ -269,7 +693,7 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO vehicle_data_source`).
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()).
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`).
WithArgs(
@@ -292,18 +716,16 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
"outside_daily_range",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectBegin()
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(2500)).
WillReturnResult(sqlmock.NewResult(1, 0))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
int64(1000),
int64(2500),
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
@@ -319,6 +741,7 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
"JT808",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
written, err := writeAggregates(context.Background(), db, aggregates, 500)
if err != nil {
@@ -419,6 +842,8 @@ func TestAddSamplesUsesCurrentDayFirstSampleBaseline(t *testing.T) {
func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
t.Setenv("BACKFILL_METHOD", "")
t.Setenv("BACKFILL_DAYS_BACK", "")
t.Setenv("BACKFILL_WINDOW_DAYS", "")
t.Setenv("BACKFILL_DATE_FROM", "2026-07-08")
t.Setenv("BACKFILL_DATE_TO", "2026-07-08")
t.Setenv("BACKFILL_PROTOCOLS", "JT808")
@@ -431,4 +856,101 @@ func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
if cfg.Method != "last_diff" {
t.Fatalf("method = %q, want last_diff", cfg.Method)
}
if cfg.EventTimeFullScan {
t.Fatal("event-time full scan should be opt-in")
}
}
func TestBackfillTimePredicatesUsePrimaryTimeForCoarseScanAndEventTimeForBusinessDay(t *testing.T) {
where := strings.Join(backfillTimePredicates(config{}, "2026-07-13", "2026-07-14"), " AND ")
for _, want := range []string{
"ts >= '2026-07-12 00:00:00'",
"ts < '2026-07-15 00:00:00'",
"event_time >= '2026-07-13 00:00:00'",
"event_time < '2026-07-14 00:00:00'",
} {
if !strings.Contains(where, want) {
t.Fatalf("time predicates missing %q: %s", want, where)
}
}
}
func TestBackfillTimePredicatesAllowExplicitDeepEventTimeScan(t *testing.T) {
where := strings.Join(backfillTimePredicates(config{EventTimeFullScan: true}, "2026-07-13", "2026-07-14"), " AND ")
if strings.Contains(where, "ts >=") || strings.Contains(where, "ts <") {
t.Fatalf("deep event-time scan must not apply storage-time bounds: %s", where)
}
for _, want := range []string{
"event_time >= '2026-07-13 00:00:00'",
"event_time < '2026-07-14 00:00:00'",
} {
if !strings.Contains(where, want) {
t.Fatalf("deep event-time scan missing %q: %s", want, where)
}
}
}
func TestResolveBackfillDateRangeDefaultsToToday(t *testing.T) {
t.Setenv("BACKFILL_DATE_FROM", "")
t.Setenv("BACKFILL_DATE_TO", "")
t.Setenv("BACKFILL_DAYS_BACK", "")
t.Setenv("BACKFILL_WINDOW_DAYS", "")
loc := time.FixedZone("Asia/Shanghai", 8*3600)
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
if dateFrom != "2026-07-12" || dateTo != "2026-07-12" {
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
}
}
func TestResolveBackfillDateRangeUsesRelativeWindow(t *testing.T) {
t.Setenv("BACKFILL_DATE_FROM", "")
t.Setenv("BACKFILL_DATE_TO", "")
t.Setenv("BACKFILL_DAYS_BACK", "1")
t.Setenv("BACKFILL_WINDOW_DAYS", "3")
loc := time.FixedZone("Asia/Shanghai", 8*3600)
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
if dateFrom != "2026-07-09" || dateTo != "2026-07-11" {
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
}
}
func TestResolveBackfillDateRangePrefersExplicitDates(t *testing.T) {
t.Setenv("BACKFILL_DATE_FROM", "2026-07-01")
t.Setenv("BACKFILL_DATE_TO", "2026-07-03")
t.Setenv("BACKFILL_DAYS_BACK", "1")
t.Setenv("BACKFILL_WINDOW_DAYS", "3")
loc := time.FixedZone("Asia/Shanghai", 8*3600)
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
if dateFrom != "2026-07-01" || dateTo != "2026-07-03" {
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
}
}
func TestBackfillEnvFilesPrefersExplicitList(t *testing.T) {
got := backfillEnvFiles("/tmp/a.env,/tmp/b.env", "/tmp/legacy.env", []string{"/tmp/default.env"})
if got != "/tmp/a.env,/tmp/b.env" {
t.Fatalf("env files = %q", got)
}
}
func TestBackfillEnvFilesUsesExistingDefaults(t *testing.T) {
dir := t.TempDir()
missing := filepath.Join(dir, "missing.env")
base := filepath.Join(dir, "base.env")
stat := filepath.Join(dir, "stat-writer.env")
if err := os.WriteFile(base, []byte("A=1\n"), 0600); err != nil {
t.Fatalf("write base env: %v", err)
}
if err := os.WriteFile(stat, []byte("B=2\n"), 0600); err != nil {
t.Fatalf("write stat env: %v", err)
}
got := backfillEnvFiles("", "", []string{missing, base, stat})
want := base + "," + stat
if got != want {
t.Fatalf("env files = %q, want %q", got, want)
}
}