shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

548
go/vehicle-gateway/cmd/stats-backfill/main_test.go
View File

@@ -2,6 +2,9 @@ package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -47,6 +50,58 @@ func TestChooseTrustedSourceKeepsContinuingSourceAndRejectsNewJump(t *testing.T)
}
}
func TestChooseTrustedSourceAcceptsSmallNegativeMileageJitter(t *testing.T) {
previous := []dailySourceLast{{
VIN: "LB9A32A28R0LS1574",
SourceKey: normalizedSourceKey("JT808", "64115156034", "", "115.159.85.149:53330"),
Phone: "64115156034",
SourceEndpoint: "115.159.85.149:53330",
TotalKM: 15355.4,
}}
current := []dailySourceLast{{
VIN: "LB9A32A28R0LS1574",
SourceKey: normalizedSourceKey("JT808", "64115156034", "", "115.159.85.149:53338"),
Phone: "64115156034",
SourceEndpoint: "115.159.85.149:53338",
TotalKM: 15355.3,
}}
chosen, ok := chooseTrustedSource(current, previous)
if !ok {
t.Fatal("chooseTrustedSource() should accept tiny negative mileage jitter")
}
if chosen.current.SourceKey != current[0].SourceKey {
t.Fatalf("chosen source = %q", chosen.current.SourceKey)
}
}
func TestChooseTrustedSourceAcceptsPlausibleMultiDayFallbackDelta(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
sourceKey := normalizedSourceKey("GB32960", "", "", "8.134.95.166:37720")
previous := []dailySourceLast{{
VIN: "LNXNEGRR1SR319498",
SourceKey: sourceKey,
SourceEndpoint: "8.134.95.166:37720",
TS: time.Date(2026, 7, 4, 11, 7, 58, 0, loc),
TotalKM: 8832.1,
}}
current := []dailySourceLast{{
VIN: "LNXNEGRR1SR319498",
SourceKey: sourceKey,
SourceEndpoint: "8.134.95.166:37720",
TS: time.Date(2026, 7, 12, 2, 52, 47, 0, loc),
TotalKM: 16665.6,
}}
chosen, ok := chooseTrustedSource(current, previous)
if !ok {
t.Fatal("chooseTrustedSource() should accept delta within the historical baseline window")
}
if delta := chosen.current.TotalKM - chosen.previous.TotalKM; delta < 7833.4 || delta > 7833.6 {
t.Fatalf("delta = %v, want historical gap delta", delta)
}
}
func TestDailySourceLastBuildsCandidateKeysBySourceIP(t *testing.T) {
sourceA := dailySourceLast{
VIN: "LA9GG64L7PBAF4001",
@@ -96,7 +151,7 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
if agg.FirstEventTime != previous.TS || agg.LatestEventTime != current.TS {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != "historical_source_baseline" {
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonHistorical {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
if agg.Count != 15 {
@@ -104,7 +159,40 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
}
}
func TestAggregateFromDailySourceUsesCurrentFirstSampleWithoutHistory(t *testing.T) {
func TestAggregateFromDailySourceRejectsHistoricalBaselineJump(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
current := dailySourceLast{
VIN: "LNXNEGRR6SR319464",
SourceKey: normalizedSourceKey("GB32960", "", "", "8.134.95.166:49206"),
SourceEndpoint: "8.134.95.166:49206",
FirstTS: time.Date(2026, 7, 12, 8, 5, 42, 0, loc),
TS: time.Date(2026, 7, 12, 10, 44, 56, 0, loc),
FirstTotalKM: 28004.2,
TotalKM: 40009.7,
RawSampleCount: 1938,
}
previous := dailySourceLast{
VIN: current.VIN,
SourceKey: current.SourceKey,
SourceEndpoint: current.SourceEndpoint,
TS: time.Date(2026, 7, 3, 19, 0, 39, 0, loc),
TotalKM: 10009.7,
}
agg := aggregateFromDailySource("2026-07-12", envelope.ProtocolGB32960, current, previous, true)
if agg.FirstKM != previous.TotalKM || agg.LatestKM != current.TotalKM {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if !agg.FirstEventTime.Equal(previous.TS) || !agg.LatestEventTime.Equal(current.TS) {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityInvalidDelta || agg.QualityReason != "outside_daily_range" {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
}
func TestAggregateFromDailySourceUsesCurrentDayFirstWhenHistoryIsEmpty(t *testing.T) {
current := dailySourceLast{
VIN: "LMRKH9AC2R1004087",
SourceKey: normalizedSourceKey("YUTONG_MQTT", "", "LMRKH9AC2R1004087", "mqtt://yutong/ytforward/shln/3"),
@@ -125,11 +213,264 @@ func TestAggregateFromDailySourceUsesCurrentFirstSampleWithoutHistory(t *testing
if agg.FirstEventTime != current.FirstTS || agg.LatestEventTime != current.TS {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != "current_day_first_sample" {
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonCurrentDayFirst {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
}
func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.T) {
tdDB, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer tdDB.Close()
mock.MatchExpectationsInOrder(true)
loc := time.FixedZone("Asia/Shanghai", 8*3600)
vin := "LMRKH9AC2R1004087"
endpoint := "mqtt://yutong/ytforward/shln/3"
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
currentRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
})
}
previousRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
})
}
mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(previousRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, int64(4)))
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, int64(6)))
aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
DateFrom: "2026-07-09",
DateTo: "2026-07-11",
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
Location: loc,
})
if err != nil {
t.Fatalf("buildLastDiffAggregates() error = %v", err)
}
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong")
agg := aggregates[vin+"|2026-07-11|YUTONG_MQTT|"+sourceKey]
if agg == nil {
t.Fatalf("missing day-three aggregate; keys=%v", aggregateKeys(aggregates))
}
if agg.FirstKM != 100 || agg.LatestKM != 120 {
t.Fatalf("km range = %v -> %v, want 100 -> 120", agg.FirstKM, agg.LatestKM)
}
if !agg.FirstEventTime.Equal(dayOneTS) || agg.QualityReason != stats.QualityReasonHistorical {
t.Fatalf("baseline = %v reason=%q, want day-one historical baseline", agg.FirstEventTime, agg.QualityReason)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func aggregateKeys(aggregates map[string]*metricAgg) []string {
keys := make([]string, 0, len(aggregates))
for key := range aggregates {
keys = append(keys, key)
}
return keys
}
func TestQueryRealtimeLocationLastRowsBuildsYutongSourceFromPeer(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
eventTime := time.Date(2026, 7, 12, 5, 54, 50, 0, loc)
mock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*LEFT JOIN vehicle_realtime_snapshot s").
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", eventTime, 11578.0))
rows, err := queryRealtimeLocationLastRows(context.Background(), db, config{Location: loc}, envelope.ProtocolYutongMQTT, "2026-07-12")
if err != nil {
t.Fatalf("queryRealtimeLocationLastRows() error = %v", err)
}
sourceRows := rows["LMRKH9AC0R1004086"]
if len(sourceRows) != 1 {
t.Fatalf("rows = %d, want 1", len(sourceRows))
}
row := sourceRows[0]
wantSourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong")
if row.SourceKey != wantSourceKey {
t.Fatalf("source key = %q", row.SourceKey)
}
if row.SourceCode != "yutong" || row.PlatformName != "宇通" || row.SourceKind != "PLATFORM" {
t.Fatalf("source metadata = code:%q platform:%q kind:%q", row.SourceCode, row.PlatformName, row.SourceKind)
}
if row.TotalKM != 11578 || row.DeviceID != "LMRKH9AC0R1004086" || row.RawSampleCount != 1 {
t.Fatalf("unexpected realtime fallback row: %#v", row)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.T) {
mysqlDB, mysqlMock, err := sqlmock.New()
if err != nil {
t.Fatalf("mysql sqlmock.New() error = %v", err)
}
defer mysqlDB.Close()
tdDB, tdMock, err := sqlmock.New()
if err != nil {
t.Fatalf("td sqlmock.New() error = %v", err)
}
defer tdDB.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
currentTS := time.Date(2026, 7, 12, 5, 54, 50, 0, loc)
previousTS := time.Date(2026, 7, 11, 23, 58, 0, 0, loc)
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", currentTS, 11578.0))
tdMock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-12 00:00:00'.*protocol = 'YUTONG_MQTT'.*TOTAL_MILEAGE").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC0R1004086",
"",
"LMRKH9AC0R1004086",
"mqtt://yutong/ytforward/shln/4",
previousTS,
`{"data":{"TOTAL_MILEAGE":11500000}}`,
int64(12),
))
aggregates := map[string]*metricAgg{}
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
DateFrom: "2026-07-12",
DateTo: "2026-07-12",
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
Location: loc,
}, aggregates)
if err != nil {
t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err)
}
if added != 1 || len(aggregates) != 1 {
t.Fatalf("added=%d aggregates=%d", added, len(aggregates))
}
for _, agg := range aggregates {
if agg.FirstKM != 11500 || agg.LatestKM != 11578 {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if agg.SourceKey != stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong") {
t.Fatalf("source key = %q", agg.SourceKey)
}
if agg.SourceCode != "yutong" || agg.PlatformName != "宇通" || agg.SourceKind != "PLATFORM" {
t.Fatalf("source metadata = code:%q platform:%q kind:%q", agg.SourceCode, agg.PlatformName, agg.SourceKind)
}
if agg.QualityReason != "realtime_location_fallback_historical_baseline" {
t.Fatalf("quality reason = %q", agg.QualityReason)
}
}
if err := mysqlMock.ExpectationsWereMet(); err != nil {
t.Fatalf("mysql sql expectations: %v", err)
}
if err := tdMock.ExpectationsWereMet(); err != nil {
t.Fatalf("td sql expectations: %v", err)
}
}
func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *testing.T) {
mysqlDB, mysqlMock, err := sqlmock.New()
if err != nil {
t.Fatalf("mysql sqlmock.New() error = %v", err)
}
defer mysqlDB.Close()
tdDB, tdMock, err := sqlmock.New()
if err != nil {
t.Fatalf("td sqlmock.New() error = %v", err)
}
defer tdDB.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
vin := "LMRKH9AC2R1004087"
endpoint := "mqtt://yutong/ytforward/shln/3"
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong")
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
tdMock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}))
for _, date := range []string{"2026-07-09", "2026-07-10"} {
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
WithArgs("YUTONG_MQTT", date, date).
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}))
}
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
WithArgs("YUTONG_MQTT", "2026-07-11", "2026-07-11").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
AddRow(vin, endpoint, dayThreeTS, 120.0))
aggregates := map[string]*metricAgg{
vin + "|2026-07-09|YUTONG_MQTT|" + sourceKey: {
VIN: vin,
Date: "2026-07-09",
Protocol: envelope.ProtocolYutongMQTT,
LatestKM: 100,
Count: 4,
SourceKey: sourceKey,
DeviceID: vin,
SourceEndpoint: endpoint,
SourceCode: "yutong",
PlatformName: "宇通",
SourceKind: "PLATFORM",
LatestEventTime: dayOneTS,
},
}
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
DateFrom: "2026-07-09",
DateTo: "2026-07-11",
Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT},
Location: loc,
}, aggregates)
if err != nil {
t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err)
}
if added != 1 || len(aggregates) != 2 {
t.Fatalf("added=%d aggregates=%d, want 1 and 2", added, len(aggregates))
}
agg := aggregates[vin+"|2026-07-11|YUTONG_MQTT|"+sourceKey]
if agg == nil {
t.Fatal("missing realtime-location fallback aggregate")
}
if agg.FirstKM != 100 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayOneTS) {
t.Fatalf("fallback range = %v@%v -> %v, want 100@day-one -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM)
}
if agg.QualityReason != "realtime_location_fallback_historical_baseline" {
t.Fatalf("quality reason = %q", agg.QualityReason)
}
if err := mysqlMock.ExpectationsWereMet(); err != nil {
t.Fatalf("mysql sql expectations: %v", err)
}
if err := tdMock.ExpectationsWereMet(); err != nil {
t.Fatalf("td sql expectations: %v", err)
}
}
func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
@@ -140,9 +481,9 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
firstTS := time.Date(2026, 7, 8, 8, 0, 0, 0, loc)
lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'").
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(ts)", "FIRST(parsed_json)", "LAST(ts)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
@@ -183,9 +524,9 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'").
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(ts)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
@@ -215,6 +556,89 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
}
}
func TestBackfillBeforePredicatesSearchesAllEarlierHistory(t *testing.T) {
where := strings.Join(backfillBeforePredicates("2026-07-08"), " AND ")
if !strings.Contains(where, "event_time < '2026-07-08 00:00:00'") {
t.Fatalf("pre-window predicate missing exclusive upper bound: %s", where)
}
if strings.Contains(where, "event_time >=") || strings.Contains(where, "ts >=") {
t.Fatalf("pre-window predicate must not stop at the previous day: %s", where)
}
}
func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC7R1004098",
"",
"LMRKH9AC7R1004098",
"mqtt://yutong/ytforward/shln/4",
utcInstant,
`{"yutong_mqtt.data.total_mileage":"41249000"}`,
int64(1),
))
rows, err := queryPreviousLastSourceRows(context.Background(), db, config{
TDengineDatabase: "lingniu_vehicle_ts",
Location: loc,
}, envelope.ProtocolYutongMQTT, "2026-07-13")
if err != nil {
t.Fatalf("queryPreviousLastSourceRows() error = %v", err)
}
sourceRows := rows["LMRKH9AC7R1004098"]
if len(sourceRows) != 1 {
t.Fatalf("rows = %d, want 1", len(sourceRows))
}
want := time.Date(2026, 7, 12, 23, 59, 59, 0, loc)
if !sourceRows[0].TS.Equal(want) || sourceRows[0].TS.Location().String() != loc.String() {
t.Fatalf("previous event time = %s (%s), want %s (%s)", sourceRows[0].TS, sourceRows[0].TS.Location(), want, want.Location())
}
}
func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
fields := fieldsForStats(envelope.ProtocolYutongMQTT, "LMRKH9AC6R1004108", `{
"data": {
"TOTAL_MILEAGE": 65423000,
"METER_SPEED": 12.3
},
"root": {
"device": "LMRKH9AC6R1004108"
}
}`)
if got := fields["yutong_mqtt.data.total_mileage"]; got == nil {
t.Fatalf("fields missing raw yutong total mileage: %#v", fields)
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
VIN: "LMRKH9AC6R1004108",
EventTimeMS: time.Date(2026, 7, 8, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
ReceivedAtMS: time.Date(2026, 7, 8, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
Fields: fields,
}
samples, err := stats.SamplesFromEnvelope(env, time.FixedZone("Asia/Shanghai", 8*3600))
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 1 {
t.Fatalf("samples = %d, want 1", len(samples))
}
if samples[0].TotalMileageKM != 65423 {
t.Fatalf("total mileage km = %v, want 65423", samples[0].TotalMileageKM)
}
}
func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
@@ -269,7 +693,7 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO vehicle_data_source`).
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()).
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`).
WithArgs(
@@ -292,18 +716,16 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
"outside_daily_range",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectBegin()
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(2500)).
WillReturnResult(sqlmock.NewResult(1, 0))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
int64(1000),
int64(2500),
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
@@ -319,6 +741,7 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
"JT808",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
written, err := writeAggregates(context.Background(), db, aggregates, 500)
if err != nil {
@@ -419,6 +842,8 @@ func TestAddSamplesUsesCurrentDayFirstSampleBaseline(t *testing.T) {
func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
t.Setenv("BACKFILL_METHOD", "")
t.Setenv("BACKFILL_DAYS_BACK", "")
t.Setenv("BACKFILL_WINDOW_DAYS", "")
t.Setenv("BACKFILL_DATE_FROM", "2026-07-08")
t.Setenv("BACKFILL_DATE_TO", "2026-07-08")
t.Setenv("BACKFILL_PROTOCOLS", "JT808")
@@ -431,4 +856,101 @@ func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
if cfg.Method != "last_diff" {
t.Fatalf("method = %q, want last_diff", cfg.Method)
}
if cfg.EventTimeFullScan {
t.Fatal("event-time full scan should be opt-in")
}
}
func TestBackfillTimePredicatesUsePrimaryTimeForCoarseScanAndEventTimeForBusinessDay(t *testing.T) {
where := strings.Join(backfillTimePredicates(config{}, "2026-07-13", "2026-07-14"), " AND ")
for _, want := range []string{
"ts >= '2026-07-12 00:00:00'",
"ts < '2026-07-15 00:00:00'",
"event_time >= '2026-07-13 00:00:00'",
"event_time < '2026-07-14 00:00:00'",
} {
if !strings.Contains(where, want) {
t.Fatalf("time predicates missing %q: %s", want, where)
}
}
}
func TestBackfillTimePredicatesAllowExplicitDeepEventTimeScan(t *testing.T) {
where := strings.Join(backfillTimePredicates(config{EventTimeFullScan: true}, "2026-07-13", "2026-07-14"), " AND ")
if strings.Contains(where, "ts >=") || strings.Contains(where, "ts <") {
t.Fatalf("deep event-time scan must not apply storage-time bounds: %s", where)
}
for _, want := range []string{
"event_time >= '2026-07-13 00:00:00'",
"event_time < '2026-07-14 00:00:00'",
} {
if !strings.Contains(where, want) {
t.Fatalf("deep event-time scan missing %q: %s", want, where)
}
}
}
func TestResolveBackfillDateRangeDefaultsToToday(t *testing.T) {
t.Setenv("BACKFILL_DATE_FROM", "")
t.Setenv("BACKFILL_DATE_TO", "")
t.Setenv("BACKFILL_DAYS_BACK", "")
t.Setenv("BACKFILL_WINDOW_DAYS", "")
loc := time.FixedZone("Asia/Shanghai", 8*3600)
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
if dateFrom != "2026-07-12" || dateTo != "2026-07-12" {
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
}
}
func TestResolveBackfillDateRangeUsesRelativeWindow(t *testing.T) {
t.Setenv("BACKFILL_DATE_FROM", "")
t.Setenv("BACKFILL_DATE_TO", "")
t.Setenv("BACKFILL_DAYS_BACK", "1")
t.Setenv("BACKFILL_WINDOW_DAYS", "3")
loc := time.FixedZone("Asia/Shanghai", 8*3600)
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
if dateFrom != "2026-07-09" || dateTo != "2026-07-11" {
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
}
}
func TestResolveBackfillDateRangePrefersExplicitDates(t *testing.T) {
t.Setenv("BACKFILL_DATE_FROM", "2026-07-01")
t.Setenv("BACKFILL_DATE_TO", "2026-07-03")
t.Setenv("BACKFILL_DAYS_BACK", "1")
t.Setenv("BACKFILL_WINDOW_DAYS", "3")
loc := time.FixedZone("Asia/Shanghai", 8*3600)
dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc)
if dateFrom != "2026-07-01" || dateTo != "2026-07-03" {
t.Fatalf("date range = %s -> %s", dateFrom, dateTo)
}
}
func TestBackfillEnvFilesPrefersExplicitList(t *testing.T) {
got := backfillEnvFiles("/tmp/a.env,/tmp/b.env", "/tmp/legacy.env", []string{"/tmp/default.env"})
if got != "/tmp/a.env,/tmp/b.env" {
t.Fatalf("env files = %q", got)
}
}
func TestBackfillEnvFilesUsesExistingDefaults(t *testing.T) {
dir := t.TempDir()
missing := filepath.Join(dir, "missing.env")
base := filepath.Join(dir, "base.env")
stat := filepath.Join(dir, "stat-writer.env")
if err := os.WriteFile(base, []byte("A=1\n"), 0600); err != nil {
t.Fatalf("write base env: %v", err)
}
if err := os.WriteFile(stat, []byte("B=2\n"), 0600); err != nil {
t.Fatalf("write stat env: %v", err)
}
got := backfillEnvFiles("", "", []string{missing, base, stat})
want := base + "," + stat
if got != want {
t.Fatalf("env files = %q, want %q", got, want)
}
}