feat: build vehicle data platform and production pipeline
This commit is contained in:
@@ -22,20 +22,21 @@ import (
|
||||
)
|
||||
|
||||
type config struct {
|
||||
MySQLDSN string
|
||||
TDengineDriver string
|
||||
TDengineDSN string
|
||||
TDengineDatabase string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Protocols []envelope.Protocol
|
||||
Method string
|
||||
Limit int
|
||||
DryRun bool
|
||||
Reset bool
|
||||
Debug bool
|
||||
ProgressEvery int64
|
||||
Location *time.Location
|
||||
MySQLDSN string
|
||||
TDengineDriver string
|
||||
TDengineDSN string
|
||||
TDengineDatabase string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Protocols []envelope.Protocol
|
||||
Method string
|
||||
Limit int
|
||||
DryRun bool
|
||||
Reset bool
|
||||
Debug bool
|
||||
EventTimeFullScan bool
|
||||
ProgressEvery int64
|
||||
Location *time.Location
|
||||
}
|
||||
|
||||
type rawFrameRow struct {
|
||||
@@ -62,6 +63,9 @@ type metricAgg struct {
|
||||
Phone string
|
||||
DeviceID string
|
||||
SourceEndpoint string
|
||||
SourceCode string
|
||||
PlatformName string
|
||||
SourceKind string
|
||||
FirstEventTime time.Time
|
||||
LatestEventTime time.Time
|
||||
QualityStatus string
|
||||
@@ -74,6 +78,9 @@ type dailySourceLast struct {
|
||||
Phone string
|
||||
DeviceID string
|
||||
SourceEndpoint string
|
||||
SourceCode string
|
||||
PlatformName string
|
||||
SourceKind string
|
||||
FirstTS time.Time
|
||||
TS time.Time
|
||||
FirstTotalKM float64
|
||||
@@ -81,8 +88,19 @@ type dailySourceLast struct {
|
||||
RawSampleCount int64
|
||||
}
|
||||
|
||||
type sourceHistoryID struct {
|
||||
VIN string
|
||||
SourceKey string
|
||||
}
|
||||
|
||||
var defaultBackfillEnvFiles = []string{
|
||||
"/opt/lingniu-go-native/env/base.env",
|
||||
"/opt/lingniu-go-native/env/stat-writer.env",
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := loadEnvFiles(env("BACKFILL_ENV_FILES", os.Getenv("BACKFILL_ENV_FILE"))); err != nil {
|
||||
envFiles := backfillEnvFiles(os.Getenv("BACKFILL_ENV_FILES"), os.Getenv("BACKFILL_ENV_FILE"), defaultBackfillEnvFiles)
|
||||
if err := loadEnvFiles(envFiles); err != nil {
|
||||
fail("load env file", err)
|
||||
}
|
||||
cfg, err := loadConfig()
|
||||
@@ -121,18 +139,27 @@ func main() {
|
||||
slog.Info("reset stats rows", "deleted", deleted)
|
||||
}
|
||||
if cfg.Method == "last_diff" {
|
||||
aggregates, err := buildLastDiffAggregates(ctx, td, cfg)
|
||||
aggregates, err := buildLastDiffAggregates(ctx, mysqlDB, td, cfg)
|
||||
if err != nil {
|
||||
fail("build last-diff aggregates", err)
|
||||
}
|
||||
fallbacks, err := addRealtimeLocationFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
|
||||
if err != nil {
|
||||
fail("build realtime-location fallback aggregates", err)
|
||||
}
|
||||
var written int64
|
||||
var normalized int
|
||||
if !cfg.DryRun {
|
||||
written, err = writeAggregates(ctx, mysqlDB, aggregates, 500)
|
||||
if err != nil {
|
||||
fail("write aggregates", err)
|
||||
}
|
||||
normalized, err = normalizeHistoricalPlatformSources(ctx, mysqlDB, cfg)
|
||||
if err != nil {
|
||||
fail("normalize historical platform sources", err)
|
||||
}
|
||||
}
|
||||
slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "written", written)
|
||||
slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "written", written, "platformSourcesNormalized", normalized)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -234,6 +261,7 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
|
||||
Phone: sample.Phone,
|
||||
DeviceID: sample.DeviceID,
|
||||
SourceEndpoint: sample.SourceEndpoint,
|
||||
PlatformName: sample.PlatformName,
|
||||
FirstEventTime: sample.EventTime,
|
||||
LatestEventTime: sample.EventTime,
|
||||
QualityStatus: stats.QualityOK,
|
||||
@@ -255,6 +283,9 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
|
||||
if strings.TrimSpace(sample.DeviceID) != "" {
|
||||
agg.DeviceID = sample.DeviceID
|
||||
}
|
||||
if strings.TrimSpace(sample.PlatformName) != "" {
|
||||
agg.PlatformName = sample.PlatformName
|
||||
}
|
||||
}
|
||||
agg.Count++
|
||||
}
|
||||
@@ -271,6 +302,9 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
|
||||
Protocol: agg.Protocol,
|
||||
SourceIP: stats.NormalizeSourceIP(agg.SourceEndpoint),
|
||||
SourceEndpoint: agg.SourceEndpoint,
|
||||
SourceCode: agg.SourceCode,
|
||||
PlatformName: agg.PlatformName,
|
||||
SourceKind: agg.SourceKind,
|
||||
}
|
||||
if strings.TrimSpace(identity.SourceIP) == "" {
|
||||
continue
|
||||
@@ -285,7 +319,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
|
||||
if err := stats.UpsertDataSource(ctx, db, identity, agg.LatestEventTime); err != nil {
|
||||
return written, err
|
||||
}
|
||||
dailyKM := agg.LatestKM - agg.FirstKM
|
||||
dailyKM := stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM)
|
||||
candidate := stats.SourceMileageSample{
|
||||
VIN: agg.VIN,
|
||||
StatDate: agg.Date,
|
||||
@@ -295,6 +329,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
|
||||
SourceEndpoint: agg.SourceEndpoint,
|
||||
Phone: agg.Phone,
|
||||
DeviceID: agg.DeviceID,
|
||||
PlatformName: agg.PlatformName,
|
||||
FirstTotalKM: agg.FirstKM,
|
||||
LatestTotalKM: agg.LatestKM,
|
||||
DailyKM: dailyKM,
|
||||
@@ -310,10 +345,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
|
||||
if candidate.QualityReason == "" {
|
||||
candidate.QualityReason = "same_source_previous_day"
|
||||
}
|
||||
if candidate.QualityStatus == stats.QualityOK && (candidate.DailyKM < 0 || candidate.DailyKM > maxTrustedDailyMileageKM) {
|
||||
candidate.QualityStatus = stats.QualityInvalidDelta
|
||||
candidate.QualityReason = "outside_daily_range"
|
||||
}
|
||||
stats.ApplyMileageQualityRules(&candidate)
|
||||
if err := stats.UpsertSourceMileage(ctx, db, candidate); err != nil {
|
||||
return written, err
|
||||
}
|
||||
@@ -325,6 +357,27 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func normalizeHistoricalPlatformSources(ctx context.Context, db *sql.DB, cfg config) (int, error) {
|
||||
dates, err := dateRange(cfg.DateFrom, cfg.DateTo)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
normalized := 0
|
||||
for _, protocol := range cfg.Protocols {
|
||||
if protocol != envelope.ProtocolJT808 {
|
||||
continue
|
||||
}
|
||||
for _, date := range dates {
|
||||
count, err := stats.NormalizePlatformSourceMileageForDate(ctx, db, date, protocol)
|
||||
if err != nil {
|
||||
return normalized, err
|
||||
}
|
||||
normalized += count
|
||||
}
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func clearBackfillTargetMileage(ctx context.Context, db *sql.DB, vin string, statDate string, protocol envelope.Protocol) error {
|
||||
if db == nil || strings.TrimSpace(vin) == "" || strings.TrimSpace(statDate) == "" || strings.TrimSpace(string(protocol)) == "" {
|
||||
return nil
|
||||
@@ -337,47 +390,173 @@ func clearBackfillTargetMileage(ctx context.Context, db *sql.DB, vin string, sta
|
||||
return err
|
||||
}
|
||||
|
||||
func buildLastDiffAggregates(ctx context.Context, db *sql.DB, cfg config) (map[string]*metricAgg, error) {
|
||||
func buildLastDiffAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config) (map[string]*metricAgg, error) {
|
||||
targetDates, err := dateRange(cfg.DateFrom, cfg.DateTo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aggregates := map[string]*metricAgg{}
|
||||
for _, protocol := range cfg.Protocols {
|
||||
latestHistory := map[sourceHistoryID]dailySourceLast{}
|
||||
preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rememberLatestSourceRows(latestHistory, preWindow)
|
||||
slog.Info("pre-window baseline loaded", "protocol", protocol, "before_date", targetDates[0], "vehicles", len(preWindow))
|
||||
for _, date := range targetDates {
|
||||
current, err := queryDailyLastSourceRows(ctx, db, cfg, protocol, date)
|
||||
current, err := queryDailyLastSourceRows(ctx, tdDB, cfg, protocol, date)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
previous, err := queryPreviousLastSourceRows(ctx, db, cfg, protocol, date)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slog.Info("daily last loaded", "protocol", protocol, "date", date, "vehicles", len(current), "previousVehicles", len(previous))
|
||||
slog.Info("daily last loaded", "protocol", protocol, "date", date, "vehicles", len(current), "historicalSources", len(latestHistory))
|
||||
for vin, currentSources := range current {
|
||||
previousBySource := map[string]dailySourceLast{}
|
||||
for _, previousRow := range previous[vin] {
|
||||
previousBySource[previousRow.SourceKey] = previousRow
|
||||
}
|
||||
for _, currentRow := range currentSources {
|
||||
key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey
|
||||
previousRow, hasPrevious := previousBySource[currentRow.SourceKey]
|
||||
previousRow, hasPrevious, err := resolvePreviousSourceRow(ctx, mysqlDB, date, protocol, latestHistory, currentRow)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aggregates[key] = aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious)
|
||||
}
|
||||
}
|
||||
// Current-day last values become eligible only for later target dates.
|
||||
rememberLatestSourceRows(latestHistory, current)
|
||||
}
|
||||
}
|
||||
return aggregates, nil
|
||||
}
|
||||
|
||||
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
|
||||
if aggregates == nil {
|
||||
return 0, fmt.Errorf("aggregates map is nil")
|
||||
}
|
||||
targetDates, err := dateRange(cfg.DateFrom, cfg.DateTo)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var added int
|
||||
for _, protocol := range cfg.Protocols {
|
||||
if protocol != envelope.ProtocolYutongMQTT {
|
||||
continue
|
||||
}
|
||||
latestHistory := map[sourceHistoryID]dailySourceLast{}
|
||||
preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
|
||||
if err != nil {
|
||||
return added, err
|
||||
}
|
||||
rememberLatestSourceRows(latestHistory, preWindow)
|
||||
aggregateRowsByDate := indexAggregateSourceRowsByDate(aggregates, protocol)
|
||||
for _, date := range targetDates {
|
||||
current, err := queryRealtimeLocationLastRows(ctx, mysqlDB, cfg, protocol, date)
|
||||
if err != nil {
|
||||
return added, err
|
||||
}
|
||||
for vin, currentSources := range current {
|
||||
for _, currentRow := range currentSources {
|
||||
key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey
|
||||
if _, exists := aggregates[key]; exists {
|
||||
continue
|
||||
}
|
||||
previousRow, hasPrevious, err := resolvePreviousSourceRow(ctx, mysqlDB, date, protocol, latestHistory, currentRow)
|
||||
if err != nil {
|
||||
return added, err
|
||||
}
|
||||
agg := aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious)
|
||||
if hasPrevious {
|
||||
agg.QualityReason = "realtime_location_fallback_historical_baseline"
|
||||
} else {
|
||||
agg.QualityReason = "realtime_location_fallback_current_day_first_baseline"
|
||||
}
|
||||
aggregates[key] = agg
|
||||
added++
|
||||
}
|
||||
}
|
||||
rememberLatestSourceRows(latestHistory, aggregateRowsByDate[date])
|
||||
rememberLatestSourceRows(latestHistory, current)
|
||||
if len(current) > 0 {
|
||||
slog.Info("realtime location fallback loaded", "protocol", protocol, "date", date, "vehicles", len(current), "added", added)
|
||||
}
|
||||
}
|
||||
}
|
||||
return added, nil
|
||||
}
|
||||
|
||||
func indexAggregateSourceRowsByDate(aggregates map[string]*metricAgg, protocol envelope.Protocol) map[string]map[string][]dailySourceLast {
|
||||
indexed := map[string]map[string][]dailySourceLast{}
|
||||
for _, agg := range aggregates {
|
||||
if agg == nil || agg.Protocol != protocol || strings.TrimSpace(agg.Date) == "" {
|
||||
continue
|
||||
}
|
||||
rows := indexed[agg.Date]
|
||||
if rows == nil {
|
||||
rows = map[string][]dailySourceLast{}
|
||||
indexed[agg.Date] = rows
|
||||
}
|
||||
rows[agg.VIN] = append(rows[agg.VIN], dailySourceLast{
|
||||
VIN: agg.VIN,
|
||||
SourceKey: agg.SourceKey,
|
||||
Phone: agg.Phone,
|
||||
DeviceID: agg.DeviceID,
|
||||
SourceEndpoint: agg.SourceEndpoint,
|
||||
SourceCode: agg.SourceCode,
|
||||
PlatformName: agg.PlatformName,
|
||||
SourceKind: agg.SourceKind,
|
||||
FirstTS: agg.LatestEventTime,
|
||||
TS: agg.LatestEventTime,
|
||||
FirstTotalKM: agg.LatestKM,
|
||||
TotalKM: agg.LatestKM,
|
||||
RawSampleCount: agg.Count,
|
||||
})
|
||||
}
|
||||
return indexed
|
||||
}
|
||||
|
||||
func rememberLatestSourceRows(history map[sourceHistoryID]dailySourceLast, rows map[string][]dailySourceLast) {
|
||||
for vin, sourceRows := range rows {
|
||||
for _, row := range sourceRows {
|
||||
rowVIN := strings.TrimSpace(row.VIN)
|
||||
if rowVIN == "" {
|
||||
rowVIN = strings.TrimSpace(vin)
|
||||
row.VIN = rowVIN
|
||||
}
|
||||
id := sourceHistoryID{VIN: rowVIN, SourceKey: strings.TrimSpace(row.SourceKey)}
|
||||
if id.VIN == "" || id.SourceKey == "" || row.TS.IsZero() {
|
||||
continue
|
||||
}
|
||||
if existing, ok := history[id]; !ok || row.TS.After(existing.TS) {
|
||||
history[id] = row
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func latestHistoricalSourceRow(history map[sourceHistoryID]dailySourceLast, current dailySourceLast) (dailySourceLast, bool) {
|
||||
id := sourceHistoryID{
|
||||
VIN: strings.TrimSpace(current.VIN),
|
||||
SourceKey: strings.TrimSpace(current.SourceKey),
|
||||
}
|
||||
previous, ok := history[id]
|
||||
return previous, ok && previous.TS.Before(current.TS)
|
||||
}
|
||||
|
||||
func resolvePreviousSourceRow(ctx context.Context, db *sql.DB, date string, protocol envelope.Protocol, history map[sourceHistoryID]dailySourceLast, current dailySourceLast) (dailySourceLast, bool, error) {
|
||||
if previous, ok := latestHistoricalSourceRow(history, current); ok {
|
||||
return previous, true, nil
|
||||
}
|
||||
return queryDurablePreviousSourceRow(ctx, db, date, protocol, current)
|
||||
}
|
||||
|
||||
func aggregateFromDailySource(date string, protocol envelope.Protocol, current dailySourceLast, previous dailySourceLast, hasPrevious bool) *metricAgg {
|
||||
firstKM := current.FirstTotalKM
|
||||
firstEventTime := current.FirstTS
|
||||
qualityReason := "current_day_first_sample"
|
||||
qualityStatus := stats.QualityOK
|
||||
qualityReason := stats.QualityReasonCurrentDayFirst
|
||||
if hasPrevious {
|
||||
firstKM = previous.TotalKM
|
||||
firstEventTime = previous.TS
|
||||
qualityReason = "historical_source_baseline"
|
||||
qualityStatus = stats.QualityOK
|
||||
qualityReason = stats.QualityReasonHistorical
|
||||
}
|
||||
if firstEventTime.IsZero() {
|
||||
firstEventTime = current.TS
|
||||
@@ -386,7 +565,7 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d
|
||||
if count <= 0 {
|
||||
count = 1
|
||||
}
|
||||
return &metricAgg{
|
||||
agg := &metricAgg{
|
||||
VIN: current.VIN,
|
||||
Date: date,
|
||||
Protocol: protocol,
|
||||
@@ -397,11 +576,43 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d
|
||||
Phone: current.Phone,
|
||||
DeviceID: current.DeviceID,
|
||||
SourceEndpoint: current.SourceEndpoint,
|
||||
SourceCode: current.SourceCode,
|
||||
PlatformName: current.PlatformName,
|
||||
SourceKind: current.SourceKind,
|
||||
FirstEventTime: firstEventTime,
|
||||
LatestEventTime: current.TS,
|
||||
QualityStatus: stats.QualityOK,
|
||||
QualityStatus: qualityStatus,
|
||||
QualityReason: qualityReason,
|
||||
}
|
||||
candidate := stats.SourceMileageSample{
|
||||
FirstTotalKM: agg.FirstKM,
|
||||
LatestTotalKM: agg.LatestKM,
|
||||
DailyKM: stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM),
|
||||
FirstEventTime: agg.FirstEventTime,
|
||||
LatestEventTime: agg.LatestEventTime,
|
||||
QualityStatus: agg.QualityStatus,
|
||||
QualityReason: agg.QualityReason,
|
||||
}
|
||||
stats.ApplyMileageQualityRules(&candidate)
|
||||
agg.QualityStatus = candidate.QualityStatus
|
||||
agg.QualityReason = candidate.QualityReason
|
||||
return agg
|
||||
}
|
||||
|
||||
func queryDurablePreviousSourceRow(ctx context.Context, db *sql.DB, date string, protocol envelope.Protocol, current dailySourceLast) (dailySourceLast, bool, error) {
|
||||
if db == nil {
|
||||
return dailySourceLast{}, false, nil
|
||||
}
|
||||
totalKM, eventTime, found, err := stats.LookupLatestSourceBaselineBefore(ctx, db, current.VIN, date, protocol, current.SourceKey)
|
||||
if err != nil || !found {
|
||||
return dailySourceLast{}, found, err
|
||||
}
|
||||
previous := current
|
||||
previous.FirstTS = eventTime
|
||||
previous.TS = eventTime
|
||||
previous.FirstTotalKM = totalKM
|
||||
previous.TotalKM = totalKM
|
||||
return previous, true, nil
|
||||
}
|
||||
|
||||
type trustedChoice struct {
|
||||
@@ -409,8 +620,6 @@ type trustedChoice struct {
|
||||
previous dailySourceLast
|
||||
}
|
||||
|
||||
const maxTrustedDailyMileageKM = 1000
|
||||
|
||||
func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast) (trustedChoice, bool) {
|
||||
previousBySource := map[string]dailySourceLast{}
|
||||
for _, row := range previous {
|
||||
@@ -423,8 +632,8 @@ func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
delta := currentRow.TotalKM - previousRow.TotalKM
|
||||
if delta < 0 || delta > maxTrustedDailyMileageKM {
|
||||
delta, ok, _ := stats.NormalizeDailyMileageDeltaForWindow(stats.DailyMileageFromDayBoundary(previousRow.TotalKM, currentRow.TotalKM), previousRow.TS, currentRow.TS)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if chosen.current.SourceKey == "" || delta < chosenDelta {
|
||||
@@ -436,19 +645,18 @@ func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast)
|
||||
}
|
||||
|
||||
func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
|
||||
where := []string{
|
||||
fmt.Sprintf("ts >= '%s 00:00:00'", quote(date)),
|
||||
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(date))),
|
||||
where := backfillTimePredicates(cfg, date, nextDate(date))
|
||||
where = append(where,
|
||||
"parse_status = 'OK'",
|
||||
"vin IS NOT NULL",
|
||||
"vin <> ''",
|
||||
fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
|
||||
realtimeMileageFramePredicate(),
|
||||
}
|
||||
)
|
||||
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(ts), FIRST(parsed_json), LAST(ts), LAST(parsed_json), COUNT(*)
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(event_time), FIRST(parsed_json), LAST(event_time), LAST(parsed_json), COUNT(*)
|
||||
FROM %s.raw_frames
|
||||
WHERE %s
|
||||
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
@@ -456,24 +664,103 @@ GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), s
|
||||
}
|
||||
|
||||
func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
|
||||
where := []string{
|
||||
fmt.Sprintf("ts < '%s 00:00:00'", quote(date)),
|
||||
// The baseline is the nearest earlier sample, not necessarily yesterday's.
|
||||
// LAST aggregates the complete pre-window history once per source so empty
|
||||
// calendar days do not make a backfill fall back to the current day's first
|
||||
// sample.
|
||||
where := backfillBeforePredicates(date)
|
||||
where = append(where,
|
||||
"parse_status = 'OK'",
|
||||
"vin IS NOT NULL",
|
||||
"vin <> ''",
|
||||
fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
|
||||
realtimeMileageFramePredicate(),
|
||||
}
|
||||
)
|
||||
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(ts), LAST(parsed_json), COUNT(*)
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(event_time), LAST(parsed_json), COUNT(*)
|
||||
FROM %s.raw_frames
|
||||
WHERE %s
|
||||
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
|
||||
}
|
||||
|
||||
func backfillBeforePredicates(eventDateExclusive string) []string {
|
||||
return []string{
|
||||
fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateExclusive)),
|
||||
}
|
||||
}
|
||||
|
||||
func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
|
||||
if db == nil {
|
||||
return nil, nil
|
||||
}
|
||||
loc := cfg.Location
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
sqlText := `SELECT l.vin, COALESCE(NULLIF(s.peer, ''), ''), l.total_mileage_event_time, l.total_mileage_km
|
||||
FROM vehicle_realtime_location l
|
||||
LEFT JOIN vehicle_realtime_snapshot s ON s.protocol = l.protocol AND s.vin = l.vin
|
||||
WHERE l.protocol = ?
|
||||
AND l.vin IS NOT NULL AND l.vin <> ''
|
||||
AND l.total_mileage_km IS NOT NULL AND l.total_mileage_km > 0
|
||||
AND l.total_mileage_event_time >= ?
|
||||
AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY)`
|
||||
rows, err := db.QueryContext(ctx, sqlText, string(protocol), date, date)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string][]dailySourceLast{}
|
||||
for rows.Next() {
|
||||
var vin string
|
||||
var peer sql.NullString
|
||||
var eventTime time.Time
|
||||
var totalKM float64
|
||||
if err := rows.Scan(&vin, &peer, &eventTime, &totalKM); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vin = strings.TrimSpace(vin)
|
||||
if vin == "" || totalKM <= 0 {
|
||||
continue
|
||||
}
|
||||
sourceEndpoint := strings.TrimSpace(peer.String)
|
||||
if sourceEndpoint == "" && protocol == envelope.ProtocolYutongMQTT {
|
||||
sourceEndpoint = "mqtt://yutong/realtime-location"
|
||||
}
|
||||
deviceID := ""
|
||||
if protocol == envelope.ProtocolYutongMQTT {
|
||||
deviceID = vin
|
||||
}
|
||||
sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
|
||||
sourceKey := normalizedSourceKeyForSource(string(protocol), "", deviceID, sourceEndpoint, sourceKind, sourceCode)
|
||||
if sourceKey == "" {
|
||||
continue
|
||||
}
|
||||
row := dailySourceLast{
|
||||
VIN: vin,
|
||||
SourceKey: sourceKey,
|
||||
DeviceID: deviceID,
|
||||
SourceEndpoint: sourceEndpoint,
|
||||
SourceCode: sourceCode,
|
||||
PlatformName: platformName,
|
||||
SourceKind: sourceKind,
|
||||
FirstTS: eventTime.In(loc),
|
||||
TS: eventTime.In(loc),
|
||||
FirstTotalKM: totalKM,
|
||||
TotalKM: totalKM,
|
||||
RawSampleCount: 1,
|
||||
}
|
||||
out[vin] = append(out[vin], row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool) (map[string][]dailySourceLast, error) {
|
||||
rows, err := db.QueryContext(ctx, sqlText)
|
||||
if err != nil {
|
||||
@@ -512,7 +799,8 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
|
||||
firstTotalKM = parsedFirst
|
||||
}
|
||||
}
|
||||
sourceKey := normalizedSourceKey(string(protocol), phone, deviceID, sourceEndpoint)
|
||||
sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
|
||||
sourceKey := normalizedSourceKeyForSource(string(protocol), phone, deviceID, sourceEndpoint, sourceKind, sourceCode)
|
||||
if sourceKey == "" {
|
||||
continue
|
||||
}
|
||||
@@ -522,8 +810,11 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
|
||||
Phone: strings.TrimSpace(phone),
|
||||
DeviceID: strings.TrimSpace(deviceID),
|
||||
SourceEndpoint: strings.TrimSpace(sourceEndpoint),
|
||||
SourceCode: sourceCode,
|
||||
PlatformName: platformName,
|
||||
SourceKind: sourceKind,
|
||||
FirstTS: firstTS.In(cfg.Location),
|
||||
TS: ts,
|
||||
TS: ts.In(cfg.Location),
|
||||
FirstTotalKM: firstTotalKM,
|
||||
TotalKM: latestTotalKM,
|
||||
RawSampleCount: rawSampleCount,
|
||||
@@ -562,8 +853,19 @@ func mileageFromParsed(protocol envelope.Protocol, vin string, parsedJSON string
|
||||
}
|
||||
|
||||
func normalizedSourceKey(protocol string, phone string, deviceID string, endpoint string) string {
|
||||
return normalizedSourceKeyForSource(protocol, phone, deviceID, endpoint, "", "")
|
||||
}
|
||||
|
||||
func normalizedSourceKeyForSource(protocol string, phone string, deviceID string, endpoint string, sourceKind string, sourceCode string) string {
|
||||
sourceIP := stats.NormalizeSourceIP(endpoint)
|
||||
return stats.SourceKey(envelope.Protocol(protocol), phone, deviceID, sourceIP)
|
||||
return stats.SourceKeyForSource(envelope.Protocol(protocol), phone, deviceID, sourceIP, sourceKind, sourceCode)
|
||||
}
|
||||
|
||||
func knownPlatformSourceMetadata(protocol envelope.Protocol, endpoint string) (sourceCode string, platformName string, sourceKind string) {
|
||||
if protocol == envelope.ProtocolYutongMQTT && stats.NormalizeSourceIP(endpoint) == "mqtt" {
|
||||
return "yutong", "宇通", "PLATFORM"
|
||||
}
|
||||
return "", "", ""
|
||||
}
|
||||
|
||||
func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[string]any {
|
||||
@@ -574,7 +876,7 @@ func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[str
|
||||
}
|
||||
parsed := map[string]any{}
|
||||
if err := json.Unmarshal([]byte(text), &parsed); err == nil && len(parsed) > 0 {
|
||||
if flattened, _, ok := realtime.ParsedFieldsForEnvelope(envelope.FrameEnvelope{
|
||||
if flattened, _, ok := realtime.ComputeParsedFieldsForEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: protocol,
|
||||
VIN: vin,
|
||||
Parsed: parsed,
|
||||
@@ -604,7 +906,12 @@ func mileageKeys(protocol envelope.Protocol) []string {
|
||||
case envelope.ProtocolJT808:
|
||||
return []string{"jt808.location.total_mileage_km"}
|
||||
case envelope.ProtocolYutongMQTT:
|
||||
return []string{"yutong_mqtt.data.total_mileage", "yutong_mqtt.root.data.total_mileage"}
|
||||
return []string{
|
||||
"yutong_mqtt.data.total_mileage_km",
|
||||
"yutong_mqtt.root.data.total_mileage_km",
|
||||
"yutong_mqtt.data.total_mileage",
|
||||
"yutong_mqtt.root.data.total_mileage",
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
@@ -624,38 +931,67 @@ func loadConfig() (config, error) {
|
||||
if err != nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
dateTo := env("BACKFILL_DATE_TO", time.Now().In(loc).Format("2006-01-02"))
|
||||
dateFrom := env("BACKFILL_DATE_FROM", dateTo)
|
||||
dateFrom, dateTo := resolveBackfillDateRange(time.Now(), loc)
|
||||
protocols, err := parseProtocols(env("BACKFILL_PROTOCOLS", "GB32960,JT808,YUTONG_MQTT"))
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
return config{
|
||||
MySQLDSN: env("MYSQL_DSN", ""),
|
||||
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
|
||||
TDengineDSN: env("TDENGINE_DSN", ""),
|
||||
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
|
||||
DateFrom: dateFrom,
|
||||
DateTo: dateTo,
|
||||
Protocols: protocols,
|
||||
Method: env("BACKFILL_METHOD", "last_diff"),
|
||||
Limit: envInt("BACKFILL_LIMIT", 0),
|
||||
DryRun: envBool("BACKFILL_DRY_RUN", true),
|
||||
Reset: envBool("BACKFILL_RESET", false),
|
||||
Debug: envBool("BACKFILL_DEBUG", false),
|
||||
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
|
||||
Location: loc,
|
||||
MySQLDSN: env("MYSQL_DSN", ""),
|
||||
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
|
||||
TDengineDSN: env("TDENGINE_DSN", ""),
|
||||
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
|
||||
DateFrom: dateFrom,
|
||||
DateTo: dateTo,
|
||||
Protocols: protocols,
|
||||
Method: env("BACKFILL_METHOD", "last_diff"),
|
||||
Limit: envInt("BACKFILL_LIMIT", 0),
|
||||
DryRun: envBool("BACKFILL_DRY_RUN", true),
|
||||
Reset: envBool("BACKFILL_RESET", false),
|
||||
Debug: envBool("BACKFILL_DEBUG", false),
|
||||
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
|
||||
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
|
||||
Location: loc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveBackfillDateRange(now time.Time, loc *time.Location) (string, string) {
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
explicitFrom := strings.TrimSpace(os.Getenv("BACKFILL_DATE_FROM"))
|
||||
explicitTo := strings.TrimSpace(os.Getenv("BACKFILL_DATE_TO"))
|
||||
if explicitTo != "" || explicitFrom != "" {
|
||||
dateTo := explicitTo
|
||||
if dateTo == "" {
|
||||
dateTo = now.In(loc).Format("2006-01-02")
|
||||
}
|
||||
dateFrom := explicitFrom
|
||||
if dateFrom == "" {
|
||||
dateFrom = dateTo
|
||||
}
|
||||
return dateFrom, dateTo
|
||||
}
|
||||
daysBack := envInt("BACKFILL_DAYS_BACK", 0)
|
||||
if daysBack < 0 {
|
||||
daysBack = 0
|
||||
}
|
||||
windowDays := envInt("BACKFILL_WINDOW_DAYS", 1)
|
||||
if windowDays < 1 {
|
||||
windowDays = 1
|
||||
}
|
||||
dateToTime := now.In(loc).AddDate(0, 0, -daysBack)
|
||||
dateFromTime := dateToTime.AddDate(0, 0, -(windowDays - 1))
|
||||
return dateFromTime.Format("2006-01-02"), dateToTime.Format("2006-01-02")
|
||||
}
|
||||
|
||||
func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, error) {
|
||||
where := []string{
|
||||
fmt.Sprintf("ts >= '%s 00:00:00'", quote(cfg.DateFrom)),
|
||||
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(cfg.DateTo))),
|
||||
where := backfillTimePredicates(cfg, cfg.DateFrom, nextDate(cfg.DateTo))
|
||||
where = append(where,
|
||||
"parse_status = 'OK'",
|
||||
"vin IS NOT NULL",
|
||||
"vin <> ''",
|
||||
}
|
||||
)
|
||||
if len(cfg.Protocols) > 0 {
|
||||
quoted := make([]string, 0, len(cfg.Protocols))
|
||||
for _, protocol := range cfg.Protocols {
|
||||
@@ -667,7 +1003,7 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err
|
||||
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json
|
||||
FROM %s.raw_frames
|
||||
WHERE %s
|
||||
ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
ORDER BY event_time ASC, ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
if cfg.Limit > 0 {
|
||||
sqlText += fmt.Sprintf(" LIMIT %d", cfg.Limit)
|
||||
}
|
||||
@@ -675,6 +1011,23 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
return db.QueryContext(ctx, sqlText)
|
||||
}
|
||||
|
||||
func backfillTimePredicates(cfg config, eventDateFrom string, eventDateToExclusive string) []string {
|
||||
where := make([]string, 0, 4)
|
||||
if !cfg.EventTimeFullScan {
|
||||
// ts is the TDengine primary timestamp and may be adjusted slightly to keep
|
||||
// rows unique. Use it only as a broad index-friendly window; event_time is
|
||||
// the protocol business boundary used for the final natural-day result.
|
||||
where = append(where,
|
||||
fmt.Sprintf("ts >= '%s 00:00:00'", quote(previousDate(eventDateFrom))),
|
||||
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(eventDateToExclusive))),
|
||||
)
|
||||
}
|
||||
return append(where,
|
||||
fmt.Sprintf("event_time >= '%s 00:00:00'", quote(eventDateFrom)),
|
||||
fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateToExclusive)),
|
||||
)
|
||||
}
|
||||
|
||||
func realtimeMileageFramePredicate() string {
|
||||
return `(
|
||||
(protocol = 'GB32960' AND message_id IN (2,3))
|
||||
@@ -684,17 +1037,39 @@ func realtimeMileageFramePredicate() string {
|
||||
}
|
||||
|
||||
func mileageBearingFramePredicate(protocol envelope.Protocol) string {
|
||||
keys := mileageKeys(protocol)
|
||||
if len(keys) == 0 {
|
||||
tokens := mileageSearchTokens(protocol)
|
||||
if len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
conditions := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(key)))
|
||||
conditions := make([]string, 0, len(tokens))
|
||||
for _, token := range tokens {
|
||||
conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(token)))
|
||||
}
|
||||
return "(" + strings.Join(conditions, " OR ") + ")"
|
||||
}
|
||||
|
||||
func mileageSearchTokens(protocol envelope.Protocol) []string {
|
||||
tokens := append([]string{}, mileageKeys(protocol)...)
|
||||
switch protocol {
|
||||
case envelope.ProtocolYutongMQTT:
|
||||
tokens = append(tokens, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(tokens))
|
||||
out := make([]string, 0, len(tokens))
|
||||
for _, token := range tokens {
|
||||
token = strings.TrimSpace(token)
|
||||
if token == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[token]; ok {
|
||||
continue
|
||||
}
|
||||
seen[token] = struct{}{}
|
||||
out = append(out, token)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
|
||||
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed string
|
||||
var messageID int64
|
||||
@@ -804,6 +1179,26 @@ func loadEnvFiles(paths string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func backfillEnvFiles(explicitFiles string, explicitFile string, defaults []string) string {
|
||||
if value := strings.TrimSpace(explicitFiles); value != "" {
|
||||
return value
|
||||
}
|
||||
if value := strings.TrimSpace(explicitFile); value != "" {
|
||||
return value
|
||||
}
|
||||
var existing []string
|
||||
for _, path := range defaults {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
existing = append(existing, path)
|
||||
}
|
||||
}
|
||||
return strings.Join(existing, ",")
|
||||
}
|
||||
|
||||
func loadEnvFile(path string) error {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
|
||||
@@ -2,6 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -47,6 +50,58 @@ func TestChooseTrustedSourceKeepsContinuingSourceAndRejectsNewJump(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChooseTrustedSourceAcceptsSmallNegativeMileageJitter(t *testing.T) {
|
||||
previous := []dailySourceLast{{
|
||||
VIN: "LB9A32A28R0LS1574",
|
||||
SourceKey: normalizedSourceKey("JT808", "64115156034", "", "115.159.85.149:53330"),
|
||||
Phone: "64115156034",
|
||||
SourceEndpoint: "115.159.85.149:53330",
|
||||
TotalKM: 15355.4,
|
||||
}}
|
||||
current := []dailySourceLast{{
|
||||
VIN: "LB9A32A28R0LS1574",
|
||||
SourceKey: normalizedSourceKey("JT808", "64115156034", "", "115.159.85.149:53338"),
|
||||
Phone: "64115156034",
|
||||
SourceEndpoint: "115.159.85.149:53338",
|
||||
TotalKM: 15355.3,
|
||||
}}
|
||||
|
||||
chosen, ok := chooseTrustedSource(current, previous)
|
||||
if !ok {
|
||||
t.Fatal("chooseTrustedSource() should accept tiny negative mileage jitter")
|
||||
}
|
||||
if chosen.current.SourceKey != current[0].SourceKey {
|
||||
t.Fatalf("chosen source = %q", chosen.current.SourceKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChooseTrustedSourceAcceptsPlausibleMultiDayFallbackDelta(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
sourceKey := normalizedSourceKey("GB32960", "", "", "8.134.95.166:37720")
|
||||
previous := []dailySourceLast{{
|
||||
VIN: "LNXNEGRR1SR319498",
|
||||
SourceKey: sourceKey,
|
||||
SourceEndpoint: "8.134.95.166:37720",
|
||||
TS: time.Date(2026, 7, 4, 11, 7, 58, 0, loc),
|
||||
TotalKM: 8832.1,
|
||||
}}
|
||||
current := []dailySourceLast{{
|
||||
VIN: "LNXNEGRR1SR319498",
|
||||
SourceKey: sourceKey,
|
||||
SourceEndpoint: "8.134.95.166:37720",
|
||||
TS: time.Date(2026, 7, 12, 2, 52, 47, 0, loc),
|
||||
TotalKM: 16665.6,
|
||||
}}
|
||||
|
||||
chosen, ok := chooseTrustedSource(current, previous)
|
||||
if !ok {
|
||||
t.Fatal("chooseTrustedSource() should accept delta within the historical baseline window")
|
||||
}
|
||||
if delta := chosen.current.TotalKM - chosen.previous.TotalKM; delta < 7833.4 || delta > 7833.6 {
|
||||
t.Fatalf("delta = %v, want historical gap delta", delta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDailySourceLastBuildsCandidateKeysBySourceIP(t *testing.T) {
|
||||
sourceA := dailySourceLast{
|
||||
VIN: "LA9GG64L7PBAF4001",
|
||||
@@ -96,7 +151,7 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
|
||||
if agg.FirstEventTime != previous.TS || agg.LatestEventTime != current.TS {
|
||||
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
|
||||
}
|
||||
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != "historical_source_baseline" {
|
||||
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonHistorical {
|
||||
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
|
||||
}
|
||||
if agg.Count != 15 {
|
||||
@@ -104,7 +159,40 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateFromDailySourceUsesCurrentFirstSampleWithoutHistory(t *testing.T) {
|
||||
func TestAggregateFromDailySourceRejectsHistoricalBaselineJump(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
current := dailySourceLast{
|
||||
VIN: "LNXNEGRR6SR319464",
|
||||
SourceKey: normalizedSourceKey("GB32960", "", "", "8.134.95.166:49206"),
|
||||
SourceEndpoint: "8.134.95.166:49206",
|
||||
FirstTS: time.Date(2026, 7, 12, 8, 5, 42, 0, loc),
|
||||
TS: time.Date(2026, 7, 12, 10, 44, 56, 0, loc),
|
||||
FirstTotalKM: 28004.2,
|
||||
TotalKM: 40009.7,
|
||||
RawSampleCount: 1938,
|
||||
}
|
||||
previous := dailySourceLast{
|
||||
VIN: current.VIN,
|
||||
SourceKey: current.SourceKey,
|
||||
SourceEndpoint: current.SourceEndpoint,
|
||||
TS: time.Date(2026, 7, 3, 19, 0, 39, 0, loc),
|
||||
TotalKM: 10009.7,
|
||||
}
|
||||
|
||||
agg := aggregateFromDailySource("2026-07-12", envelope.ProtocolGB32960, current, previous, true)
|
||||
|
||||
if agg.FirstKM != previous.TotalKM || agg.LatestKM != current.TotalKM {
|
||||
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
|
||||
}
|
||||
if !agg.FirstEventTime.Equal(previous.TS) || !agg.LatestEventTime.Equal(current.TS) {
|
||||
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
|
||||
}
|
||||
if agg.QualityStatus != stats.QualityInvalidDelta || agg.QualityReason != "outside_daily_range" {
|
||||
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateFromDailySourceUsesCurrentDayFirstWhenHistoryIsEmpty(t *testing.T) {
|
||||
current := dailySourceLast{
|
||||
VIN: "LMRKH9AC2R1004087",
|
||||
SourceKey: normalizedSourceKey("YUTONG_MQTT", "", "LMRKH9AC2R1004087", "mqtt://yutong/ytforward/shln/3"),
|
||||
@@ -125,11 +213,264 @@ func TestAggregateFromDailySourceUsesCurrentFirstSampleWithoutHistory(t *testing
|
||||
if agg.FirstEventTime != current.FirstTS || agg.LatestEventTime != current.TS {
|
||||
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
|
||||
}
|
||||
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != "current_day_first_sample" {
|
||||
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonCurrentDayFirst {
|
||||
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.T) {
|
||||
tdDB, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer tdDB.Close()
|
||||
mock.MatchExpectationsInOrder(true)
|
||||
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
vin := "LMRKH9AC2R1004087"
|
||||
endpoint := "mqtt://yutong/ytforward/shln/3"
|
||||
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
|
||||
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
|
||||
currentRows := func() *sqlmock.Rows {
|
||||
return sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
})
|
||||
}
|
||||
previousRows := func() *sqlmock.Rows {
|
||||
return sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
})
|
||||
}
|
||||
|
||||
mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(previousRows())
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, int64(4)))
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows())
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, int64(6)))
|
||||
|
||||
aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
DateFrom: "2026-07-09",
|
||||
DateTo: "2026-07-11",
|
||||
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
|
||||
Location: loc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("buildLastDiffAggregates() error = %v", err)
|
||||
}
|
||||
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong")
|
||||
agg := aggregates[vin+"|2026-07-11|YUTONG_MQTT|"+sourceKey]
|
||||
if agg == nil {
|
||||
t.Fatalf("missing day-three aggregate; keys=%v", aggregateKeys(aggregates))
|
||||
}
|
||||
if agg.FirstKM != 100 || agg.LatestKM != 120 {
|
||||
t.Fatalf("km range = %v -> %v, want 100 -> 120", agg.FirstKM, agg.LatestKM)
|
||||
}
|
||||
if !agg.FirstEventTime.Equal(dayOneTS) || agg.QualityReason != stats.QualityReasonHistorical {
|
||||
t.Fatalf("baseline = %v reason=%q, want day-one historical baseline", agg.FirstEventTime, agg.QualityReason)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func aggregateKeys(aggregates map[string]*metricAgg) []string {
|
||||
keys := make([]string, 0, len(aggregates))
|
||||
for key := range aggregates {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func TestQueryRealtimeLocationLastRowsBuildsYutongSourceFromPeer(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
eventTime := time.Date(2026, 7, 12, 5, 54, 50, 0, loc)
|
||||
mock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*LEFT JOIN vehicle_realtime_snapshot s").
|
||||
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
|
||||
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", eventTime, 11578.0))
|
||||
|
||||
rows, err := queryRealtimeLocationLastRows(context.Background(), db, config{Location: loc}, envelope.ProtocolYutongMQTT, "2026-07-12")
|
||||
if err != nil {
|
||||
t.Fatalf("queryRealtimeLocationLastRows() error = %v", err)
|
||||
}
|
||||
sourceRows := rows["LMRKH9AC0R1004086"]
|
||||
if len(sourceRows) != 1 {
|
||||
t.Fatalf("rows = %d, want 1", len(sourceRows))
|
||||
}
|
||||
row := sourceRows[0]
|
||||
wantSourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong")
|
||||
if row.SourceKey != wantSourceKey {
|
||||
t.Fatalf("source key = %q", row.SourceKey)
|
||||
}
|
||||
if row.SourceCode != "yutong" || row.PlatformName != "宇通" || row.SourceKind != "PLATFORM" {
|
||||
t.Fatalf("source metadata = code:%q platform:%q kind:%q", row.SourceCode, row.PlatformName, row.SourceKind)
|
||||
}
|
||||
if row.TotalKM != 11578 || row.DeviceID != "LMRKH9AC0R1004086" || row.RawSampleCount != 1 {
|
||||
t.Fatalf("unexpected realtime fallback row: %#v", row)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.T) {
|
||||
mysqlDB, mysqlMock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("mysql sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer mysqlDB.Close()
|
||||
tdDB, tdMock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("td sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer tdDB.Close()
|
||||
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
currentTS := time.Date(2026, 7, 12, 5, 54, 50, 0, loc)
|
||||
previousTS := time.Date(2026, 7, 11, 23, 58, 0, 0, loc)
|
||||
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
|
||||
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
|
||||
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", currentTS, 11578.0))
|
||||
tdMock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-12 00:00:00'.*protocol = 'YUTONG_MQTT'.*TOTAL_MILEAGE").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC0R1004086",
|
||||
"",
|
||||
"LMRKH9AC0R1004086",
|
||||
"mqtt://yutong/ytforward/shln/4",
|
||||
previousTS,
|
||||
`{"data":{"TOTAL_MILEAGE":11500000}}`,
|
||||
int64(12),
|
||||
))
|
||||
|
||||
aggregates := map[string]*metricAgg{}
|
||||
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
DateFrom: "2026-07-12",
|
||||
DateTo: "2026-07-12",
|
||||
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
|
||||
Location: loc,
|
||||
}, aggregates)
|
||||
if err != nil {
|
||||
t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err)
|
||||
}
|
||||
if added != 1 || len(aggregates) != 1 {
|
||||
t.Fatalf("added=%d aggregates=%d", added, len(aggregates))
|
||||
}
|
||||
for _, agg := range aggregates {
|
||||
if agg.FirstKM != 11500 || agg.LatestKM != 11578 {
|
||||
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
|
||||
}
|
||||
if agg.SourceKey != stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong") {
|
||||
t.Fatalf("source key = %q", agg.SourceKey)
|
||||
}
|
||||
if agg.SourceCode != "yutong" || agg.PlatformName != "宇通" || agg.SourceKind != "PLATFORM" {
|
||||
t.Fatalf("source metadata = code:%q platform:%q kind:%q", agg.SourceCode, agg.PlatformName, agg.SourceKind)
|
||||
}
|
||||
if agg.QualityReason != "realtime_location_fallback_historical_baseline" {
|
||||
t.Fatalf("quality reason = %q", agg.QualityReason)
|
||||
}
|
||||
}
|
||||
if err := mysqlMock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("mysql sql expectations: %v", err)
|
||||
}
|
||||
if err := tdMock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("td sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *testing.T) {
|
||||
mysqlDB, mysqlMock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("mysql sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer mysqlDB.Close()
|
||||
tdDB, tdMock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("td sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer tdDB.Close()
|
||||
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
vin := "LMRKH9AC2R1004087"
|
||||
endpoint := "mqtt://yutong/ytforward/shln/3"
|
||||
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong")
|
||||
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
|
||||
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
|
||||
|
||||
tdMock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
}))
|
||||
for _, date := range []string{"2026-07-09", "2026-07-10"} {
|
||||
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
|
||||
WithArgs("YUTONG_MQTT", date, date).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}))
|
||||
}
|
||||
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
|
||||
WithArgs("YUTONG_MQTT", "2026-07-11", "2026-07-11").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
|
||||
AddRow(vin, endpoint, dayThreeTS, 120.0))
|
||||
|
||||
aggregates := map[string]*metricAgg{
|
||||
vin + "|2026-07-09|YUTONG_MQTT|" + sourceKey: {
|
||||
VIN: vin,
|
||||
Date: "2026-07-09",
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
LatestKM: 100,
|
||||
Count: 4,
|
||||
SourceKey: sourceKey,
|
||||
DeviceID: vin,
|
||||
SourceEndpoint: endpoint,
|
||||
SourceCode: "yutong",
|
||||
PlatformName: "宇通",
|
||||
SourceKind: "PLATFORM",
|
||||
LatestEventTime: dayOneTS,
|
||||
},
|
||||
}
|
||||
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
DateFrom: "2026-07-09",
|
||||
DateTo: "2026-07-11",
|
||||
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
|
||||
Location: loc,
|
||||
}, aggregates)
|
||||
if err != nil {
|
||||
t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err)
|
||||
}
|
||||
if added != 1 || len(aggregates) != 2 {
|
||||
t.Fatalf("added=%d aggregates=%d, want 1 and 2", added, len(aggregates))
|
||||
}
|
||||
agg := aggregates[vin+"|2026-07-11|YUTONG_MQTT|"+sourceKey]
|
||||
if agg == nil {
|
||||
t.Fatal("missing realtime-location fallback aggregate")
|
||||
}
|
||||
if agg.FirstKM != 100 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayOneTS) {
|
||||
t.Fatalf("fallback range = %v@%v -> %v, want 100@day-one -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM)
|
||||
}
|
||||
if agg.QualityReason != "realtime_location_fallback_historical_baseline" {
|
||||
t.Fatalf("quality reason = %q", agg.QualityReason)
|
||||
}
|
||||
if err := mysqlMock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("mysql sql expectations: %v", err)
|
||||
}
|
||||
if err := tdMock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("td sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
@@ -140,9 +481,9 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
firstTS := time.Date(2026, 7, 8, 8, 0, 0, 0, loc)
|
||||
lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'").
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(ts)", "FIRST(parsed_json)", "LAST(ts)", "LAST(parsed_json)", "COUNT(*)",
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC6R1004108",
|
||||
"",
|
||||
@@ -183,9 +524,9 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'").
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(ts)", "LAST(parsed_json)", "COUNT(*)",
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC6R1004108",
|
||||
"",
|
||||
@@ -215,6 +556,89 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillBeforePredicatesSearchesAllEarlierHistory(t *testing.T) {
|
||||
where := strings.Join(backfillBeforePredicates("2026-07-08"), " AND ")
|
||||
if !strings.Contains(where, "event_time < '2026-07-08 00:00:00'") {
|
||||
t.Fatalf("pre-window predicate missing exclusive upper bound: %s", where)
|
||||
}
|
||||
if strings.Contains(where, "event_time >=") || strings.Contains(where, "ts >=") {
|
||||
t.Fatalf("pre-window predicate must not stop at the previous day: %s", where)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC7R1004098",
|
||||
"",
|
||||
"LMRKH9AC7R1004098",
|
||||
"mqtt://yutong/ytforward/shln/4",
|
||||
utcInstant,
|
||||
`{"yutong_mqtt.data.total_mileage":"41249000"}`,
|
||||
int64(1),
|
||||
))
|
||||
|
||||
rows, err := queryPreviousLastSourceRows(context.Background(), db, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
Location: loc,
|
||||
}, envelope.ProtocolYutongMQTT, "2026-07-13")
|
||||
if err != nil {
|
||||
t.Fatalf("queryPreviousLastSourceRows() error = %v", err)
|
||||
}
|
||||
sourceRows := rows["LMRKH9AC7R1004098"]
|
||||
if len(sourceRows) != 1 {
|
||||
t.Fatalf("rows = %d, want 1", len(sourceRows))
|
||||
}
|
||||
want := time.Date(2026, 7, 12, 23, 59, 59, 0, loc)
|
||||
if !sourceRows[0].TS.Equal(want) || sourceRows[0].TS.Location().String() != loc.String() {
|
||||
t.Fatalf("previous event time = %s (%s), want %s (%s)", sourceRows[0].TS, sourceRows[0].TS.Location(), want, want.Location())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
|
||||
fields := fieldsForStats(envelope.ProtocolYutongMQTT, "LMRKH9AC6R1004108", `{
|
||||
"data": {
|
||||
"TOTAL_MILEAGE": 65423000,
|
||||
"METER_SPEED": 12.3
|
||||
},
|
||||
"root": {
|
||||
"device": "LMRKH9AC6R1004108"
|
||||
}
|
||||
}`)
|
||||
|
||||
if got := fields["yutong_mqtt.data.total_mileage"]; got == nil {
|
||||
t.Fatalf("fields missing raw yutong total mileage: %#v", fields)
|
||||
}
|
||||
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
VIN: "LMRKH9AC6R1004108",
|
||||
EventTimeMS: time.Date(2026, 7, 8, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||
ReceivedAtMS: time.Date(2026, 7, 8, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||
Fields: fields,
|
||||
}
|
||||
samples, err := stats.SamplesFromEnvelope(env, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 1 {
|
||||
t.Fatalf("samples = %d, want 1", len(samples))
|
||||
}
|
||||
if samples[0].TotalMileageKM != 65423 {
|
||||
t.Fatalf("total mileage km = %v, want 65423", samples[0].TotalMileageKM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
@@ -269,7 +693,7 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
|
||||
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`INSERT INTO vehicle_data_source`).
|
||||
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()).
|
||||
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`).
|
||||
WithArgs(
|
||||
@@ -292,18 +716,16 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
|
||||
"outside_daily_range",
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`).
|
||||
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`).
|
||||
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)).
|
||||
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(2500)).
|
||||
WillReturnResult(sqlmock.NewResult(1, 0))
|
||||
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`).
|
||||
WithArgs(
|
||||
"LA9GG64L7PBAF4001",
|
||||
"2026-07-08",
|
||||
"JT808",
|
||||
int64(1000),
|
||||
int64(2500),
|
||||
"LA9GG64L7PBAF4001",
|
||||
"2026-07-08",
|
||||
"JT808",
|
||||
@@ -319,6 +741,7 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
|
||||
"JT808",
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
written, err := writeAggregates(context.Background(), db, aggregates, 500)
|
||||
if err != nil {
|
||||
@@ -419,6 +842,8 @@ func TestAddSamplesUsesCurrentDayFirstSampleBaseline(t *testing.T) {
|
||||
|
||||
func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
|
||||
t.Setenv("BACKFILL_METHOD", "")
|
||||
t.Setenv("BACKFILL_DAYS_BACK", "")
|
||||
t.Setenv("BACKFILL_WINDOW_DAYS", "")
|
||||
t.Setenv("BACKFILL_DATE_FROM", "2026-07-08")
|
||||
t.Setenv("BACKFILL_DATE_TO", "2026-07-08")
|
||||
t.Setenv("BACKFILL_PROTOCOLS", "JT808")
|
||||
@@ -431,4 +856,101 @@ func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
|
||||
if cfg.Method != "last_diff" {
|
||||
t.Fatalf("method = %q, want last_diff", cfg.Method)
|
||||
}
|
||||
if cfg.EventTimeFullScan {
|
||||
t.Fatal("event-time full scan should be opt-in")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillTimePredicatesUsePrimaryTimeForCoarseScanAndEventTimeForBusinessDay(t *testing.T) {
|
||||
where := strings.Join(backfillTimePredicates(config{}, "2026-07-13", "2026-07-14"), " AND ")
|
||||
for _, want := range []string{
|
||||
"ts >= '2026-07-12 00:00:00'",
|
||||
"ts < '2026-07-15 00:00:00'",
|
||||
"event_time >= '2026-07-13 00:00:00'",
|
||||
"event_time < '2026-07-14 00:00:00'",
|
||||
} {
|
||||
if !strings.Contains(where, want) {
|
||||
t.Fatalf("time predicates missing %q: %s", want, where)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillTimePredicatesAllowExplicitDeepEventTimeScan(t *testing.T) {
|
||||
where := strings.Join(backfillTimePredicates(config{EventTimeFullScan: true}, "2026-07-13", "2026-07-14"), " AND ")
|
||||
if strings.Contains(where, "ts >=") || strings.Contains(where, "ts <") {
|
||||
t.Fatalf("deep event-time scan must not apply storage-time bounds: %s", where)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"event_time >= '2026-07-13 00:00:00'",
|
||||
"event_time < '2026-07-14 00:00:00'",
|
||||
} {
|
||||
if !strings.Contains(where, want) {
|
||||
t.Fatalf("deep event-time scan missing %q: %s", want, where)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBackfillDateRangeDefaultsToToday(t *testing.T) {
|
||||
t.Setenv("BACKFILL_DATE_FROM", "")
|
||||
t.Setenv("BACKFILL_DATE_TO", "")
|
||||
t.Setenv("BACKFILL_DAYS_BACK", "")
|
||||
t.Setenv("BACKFILL_WINDOW_DAYS", "")
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
|
||||
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
|
||||
if dateFrom != "2026-07-12" || dateTo != "2026-07-12" {
|
||||
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBackfillDateRangeUsesRelativeWindow(t *testing.T) {
|
||||
t.Setenv("BACKFILL_DATE_FROM", "")
|
||||
t.Setenv("BACKFILL_DATE_TO", "")
|
||||
t.Setenv("BACKFILL_DAYS_BACK", "1")
|
||||
t.Setenv("BACKFILL_WINDOW_DAYS", "3")
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
|
||||
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
|
||||
if dateFrom != "2026-07-09" || dateTo != "2026-07-11" {
|
||||
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBackfillDateRangePrefersExplicitDates(t *testing.T) {
|
||||
t.Setenv("BACKFILL_DATE_FROM", "2026-07-01")
|
||||
t.Setenv("BACKFILL_DATE_TO", "2026-07-03")
|
||||
t.Setenv("BACKFILL_DAYS_BACK", "1")
|
||||
t.Setenv("BACKFILL_WINDOW_DAYS", "3")
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
|
||||
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
|
||||
if dateFrom != "2026-07-01" || dateTo != "2026-07-03" {
|
||||
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillEnvFilesPrefersExplicitList(t *testing.T) {
|
||||
got := backfillEnvFiles("/tmp/a.env,/tmp/b.env", "/tmp/legacy.env", []string{"/tmp/default.env"})
|
||||
if got != "/tmp/a.env,/tmp/b.env" {
|
||||
t.Fatalf("env files = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillEnvFilesUsesExistingDefaults(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "missing.env")
|
||||
base := filepath.Join(dir, "base.env")
|
||||
stat := filepath.Join(dir, "stat-writer.env")
|
||||
if err := os.WriteFile(base, []byte("A=1\n"), 0600); err != nil {
|
||||
t.Fatalf("write base env: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(stat, []byte("B=2\n"), 0600); err != nil {
|
||||
t.Fatalf("write stat env: %v", err)
|
||||
}
|
||||
|
||||
got := backfillEnvFiles("", "", []string{missing, base, stat})
|
||||
want := base + "," + stat
|
||||
if got != want {
|
||||
t.Fatalf("env files = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user