shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

功能：完善里程边界与氢耗流式统计

Browse Source
This commit is contained in:
lingniu
2026-09-02 14:05:04 +08:00
parent 4201878c35
commit 8759668d39
23 changed files with 1638 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/vehicle-gateway/cmd/stats-backfill/main.go
+127 -11
View File
@@ -29,6 +29,7 @@ type config struct {
DateFrom string
DateTo string
Protocols []envelope.Protocol
VINs []string
Method string
Limit int
DryRun bool
@@ -39,6 +40,8 @@ type config struct {
GPSFallback bool
ProgressEvery int64
BaselineLookback int
ListHydrogenVINs bool
ListActiveVINs bool
Location *time.Location
}
@@ -117,6 +120,21 @@ func main() {
}
ctx := context.Background()
mysqlDB, err := sql.Open("mysql", cfg.MySQLDSN)
if err != nil {
fail("open mysql", err)
}
defer mysqlDB.Close()
if err := mysqlDB.PingContext(ctx); err != nil {
fail("ping mysql", err)
}
if cfg.ListHydrogenVINs {
if err := printActiveHydrogenVINs(ctx, mysqlDB); err != nil {
fail("list active hydrogen vins", err)
}
return
}
td, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
if err != nil {
fail("open tdengine", err)
@@ -125,14 +143,11 @@ func main() {
if err := td.PingContext(ctx); err != nil {
fail("ping tdengine", err)
}
mysqlDB, err := sql.Open("mysql", cfg.MySQLDSN)
if err != nil {
fail("open mysql", err)
}
defer mysqlDB.Close()
if err := mysqlDB.PingContext(ctx); err != nil {
fail("ping mysql", err)
if cfg.ListActiveVINs {
if err := printActiveFrameVINs(ctx, td, cfg); err != nil {
fail("list active frame vins", err)
}
return
}
schemaWriter := stats.NewWriter(mysqlDB, cfg.Location)
@@ -516,10 +531,11 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
currentRow.TotalKM = previousRow.TotalKM
currentSources[i] = currentRow
}
usesPreviousDayBoundary := hasPrevious && stats.IsUsableDailyMileageBoundary(date, previousRow.TS, currentRow.FirstTS.Location())
agg := aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious)
if cfg.StationaryCarry {
agg.QualityReason = stats.QualityReasonStationaryCarry
} else if hasPrevious {
} else if usesPreviousDayBoundary {
agg.QualityReason = "realtime_location_fallback_historical_baseline"
} else {
agg.QualityReason = "realtime_location_fallback_current_day_first_baseline"
@@ -909,7 +925,7 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d
firstEventTime := current.FirstTS
qualityStatus := stats.QualityOK
qualityReason := stats.QualityReasonCurrentDayFirst
if hasPrevious {
if hasPrevious && stats.IsUsableDailyMileageBoundary(date, previous.TS, current.FirstTS.Location()) {
firstKM = previous.TotalKM
firstEventTime = previous.TS
qualityStatus = stats.QualityOK
@@ -1295,10 +1311,32 @@ func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[str
}
fields[key] = value
}
promoteVehicleRunningModeForStats(protocol, fields, text)
promoteFuelCellWorkModeForStats(protocol, fields, text)
return fields
}
func promoteVehicleRunningModeForStats(protocol envelope.Protocol, fields map[string]any, text string) {
if protocol != envelope.ProtocolGB32960 || fields == nil {
return
}
if _, exists := fields[envelope.FieldVehicleRunningMode]; exists {
return
}
for _, key := range []string{"gb32960.vehicle.running_mode", "running_mode"} {
if value, exists := fields[key]; exists {
fields[envelope.FieldVehicleRunningMode] = value
return
}
}
for _, key := range []string{"gb32960.vehicle.running_mode", "running_mode"} {
if value, ok := extractJSONStringField(text, key); ok {
fields[envelope.FieldVehicleRunningMode] = value
return
}
}
}
func promoteFuelCellWorkModeForStats(protocol envelope.Protocol, fields map[string]any, text string) {
if fields == nil {
return
@@ -1383,6 +1421,7 @@ func loadConfig() (config, error) {
DateFrom: dateFrom,
DateTo: dateTo,
Protocols: protocols,
VINs: parseVINs(env("BACKFILL_VINS", "")),
Method: env("BACKFILL_METHOD", "last_diff"),
Limit: envInt("BACKFILL_LIMIT", 0),
DryRun: envBool("BACKFILL_DRY_RUN", true),
@@ -1393,10 +1432,29 @@ func loadConfig() (config, error) {
GPSFallback: envBool("BACKFILL_GPS_FALLBACK", envBool("BACKFILL_JT808_GPS_FALLBACK", true)),
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
BaselineLookback: envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
ListHydrogenVINs: envBool("BACKFILL_LIST_HYDROGEN_VINS", false),
ListActiveVINs: envBool("BACKFILL_LIST_ACTIVE_VINS", false),
Location: loc,
}, nil
}
func parseVINs(value string) []string {
seen := make(map[string]struct{})
vins := make([]string, 0)
for _, raw := range strings.Split(value, ",") {
vin := strings.ToUpper(strings.TrimSpace(raw))
if vin == "" {
continue
}
if _, ok := seen[vin]; ok {
continue
}
seen[vin] = struct{}{}
vins = append(vins, vin)
}
return vins
}
func resolveBackfillDateRange(now time.Time, loc *time.Location) (string, string) {
if loc == nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
@@ -1441,6 +1499,13 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err
}
where = append(where, "protocol IN ("+strings.Join(quoted, ",")+")")
}
if len(cfg.VINs) > 0 {
quoted := make([]string, 0, len(cfg.VINs))
for _, vin := range cfg.VINs {
quoted = append(quoted, "'"+quote(vin)+"'")
}
where = append(where, "vin IN ("+strings.Join(quoted, ",")+")")
}
where = append(where, realtimeMileageFramePredicate())
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json, raw_text
FROM %s.raw_frames
@@ -1449,10 +1514,61 @@ WHERE %s
if cfg.Limit > 0 {
sqlText += fmt.Sprintf(" LIMIT %d", cfg.Limit)
}
slog.Info("query raw frames", "date_from", cfg.DateFrom, "date_to", cfg.DateTo, "protocols", cfg.Protocols, "limit", cfg.Limit)
slog.Info("query raw frames", "date_from", cfg.DateFrom, "date_to", cfg.DateTo, "protocols", cfg.Protocols, "vins", cfg.VINs, "limit", cfg.Limit)
return db.QueryContext(ctx, sqlText)
}
func printActiveHydrogenVINs(ctx context.Context, db *sql.DB) error {
rows, err := db.QueryContext(ctx, `SELECT DISTINCT UPPER(TRIM(vin))
FROM vehicle_hydrogen_tank_capacity
WHERE active = 1 AND tank_capacity_l > 0 AND vin IS NOT NULL AND TRIM(vin) <> ''
ORDER BY UPPER(TRIM(vin))`)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var vin string
if err := rows.Scan(&vin); err != nil {
return err
}
fmt.Println(vin)
}
return rows.Err()
}
func printActiveFrameVINs(ctx context.Context, db *sql.DB, cfg config) error {
where := backfillTimePredicates(cfg, cfg.DateFrom, nextDate(cfg.DateTo))
where = append(where,
"parse_status = 'OK'",
"vin IS NOT NULL",
"vin <> ''",
)
if len(cfg.Protocols) > 0 {
quoted := make([]string, 0, len(cfg.Protocols))
for _, protocol := range cfg.Protocols {
quoted = append(quoted, "'"+quote(string(protocol))+"'")
}
where = append(where, "protocol IN ("+strings.Join(quoted, ",")+")")
}
where = append(where, realtimeMileageFramePredicate())
rows, err := db.QueryContext(ctx, fmt.Sprintf(`SELECT DISTINCT vin
FROM %s.raw_frames
WHERE %s`, ident(cfg.TDengineDatabase), strings.Join(where, " AND ")))
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var vin string
if err := rows.Scan(&vin); err != nil {
return err
}
fmt.Println(strings.ToUpper(strings.TrimSpace(vin)))
}
return rows.Err()
}
func backfillTimePredicates(cfg config, eventDateFrom string, eventDateToExclusive string) []string {
where := make([]string, 0, 4)
if !cfg.EventTimeFullScan {
go/vehicle-gateway/cmd/stats-backfill/main_test.go
+45 -18
View File
@@ -4,6 +4,7 @@ import (
"context"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
@@ -189,7 +190,7 @@ func TestDailySourceLastBuildsCandidateKeysBySourceIP(t *testing.T) {
}
}
func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
func TestAggregateFromDailySourceUsesCurrentDayBoundaryAfterOfflineGap(t *testing.T) {
current := dailySourceLast{
VIN: "LMRKH9AC2R1004087",
SourceKey: normalizedSourceKey("YUTONG_MQTT", "", "LMRKH9AC2R1004087", "mqtt://yutong/ytforward/shln/3"),
@@ -212,13 +213,13 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
agg := aggregateFromDailySource("2026-07-08", envelope.ProtocolYutongMQTT, current, previous, true)
if agg.FirstKM != 120672 || agg.LatestKM != 120788 {
if agg.FirstKM != 120778 || agg.LatestKM != 120788 {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if agg.FirstEventTime != previous.TS || agg.LatestEventTime != current.TS {
if agg.FirstEventTime != current.FirstTS || agg.LatestEventTime != current.TS {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonHistorical {
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonCurrentDayFirst {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
if agg.Count != 15 {
@@ -226,7 +227,7 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) {
}
}
func TestAggregateFromDailySourceRejectsHistoricalBaselineJump(t *testing.T) {
func TestAggregateFromDailySourceIgnoresHistoricalBaselineJumpAcrossOfflineGap(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
current := dailySourceLast{
VIN: "LNXNEGRR6SR319464",
@@ -248,10 +249,10 @@ func TestAggregateFromDailySourceRejectsHistoricalBaselineJump(t *testing.T) {
agg := aggregateFromDailySource("2026-07-12", envelope.ProtocolGB32960, current, previous, true)
if agg.FirstKM != previous.TotalKM || agg.LatestKM != current.TotalKM {
if agg.FirstKM != current.FirstTotalKM || agg.LatestKM != current.TotalKM {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if !agg.FirstEventTime.Equal(previous.TS) || !agg.LatestEventTime.Equal(current.TS) {
if !agg.FirstEventTime.Equal(current.FirstTS) || !agg.LatestEventTime.Equal(current.TS) {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityInvalidDelta || agg.QualityReason != "outside_daily_range" {
@@ -285,7 +286,7 @@ func TestAggregateFromDailySourceUsesCurrentDayFirstWhenHistoryIsEmpty(t *testin
}
}
func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.T) {
func TestBuildLastDiffAggregatesStartsFreshBoundaryAfterEmptyDays(t *testing.T) {
tdDB, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
@@ -333,11 +334,11 @@ func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.
if agg == nil {
t.Fatalf("missing day-three aggregate; keys=%v", aggregateKeys(aggregates))
}
if agg.FirstKM != 100 || agg.LatestKM != 120 {
t.Fatalf("km range = %v -> %v, want 100 -> 120", agg.FirstKM, agg.LatestKM)
if agg.FirstKM != 120 || agg.LatestKM != 120 {
t.Fatalf("km range = %v -> %v, want 120 -> 120", agg.FirstKM, agg.LatestKM)
}
if !agg.FirstEventTime.Equal(dayOneTS) || agg.QualityReason != stats.QualityReasonHistorical {
t.Fatalf("baseline = %v reason=%q, want day-one historical baseline", agg.FirstEventTime, agg.QualityReason)
if !agg.FirstEventTime.Equal(dayThreeTS) || agg.QualityReason != stats.QualityReasonCurrentDayFirst {
t.Fatalf("baseline = %v reason=%q, want day-three current-day baseline", agg.FirstEventTime, agg.QualityReason)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
@@ -601,10 +602,10 @@ func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *tes
if agg == nil {
t.Fatal("missing realtime-location fallback aggregate")
}
if agg.FirstKM != 100 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayOneTS) {
t.Fatalf("fallback range = %v@%v -> %v, want 100@day-one -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM)
if agg.FirstKM != 120 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayThreeTS) {
t.Fatalf("fallback range = %v@%v -> %v, want 120@day-three -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM)
}
if agg.QualityReason != "realtime_location_fallback_historical_baseline" {
if agg.QualityReason != "realtime_location_fallback_current_day_first_baseline" {
t.Fatalf("quality reason = %q", agg.QualityReason)
}
if err := mysqlMock.ExpectationsWereMet(); err != nil {
@@ -823,6 +824,24 @@ func TestFieldsForStatsPromotesGB32960FuelCellEngineWorkState(t *testing.T) {
}
}
func TestFieldsForStatsPromotesGB32960VehicleRunningMode(t *testing.T) {
fields := fieldsForStats(envelope.ProtocolGB32960, "LTEST000000000001", `{
"gb32960.vehicle.total_mileage_km": "120.5",
"gb32960.vehicle.running_mode": "1"
}`)
if got := fields[envelope.FieldVehicleRunningMode]; got == nil {
t.Fatalf("fields missing GB32960 vehicle running mode: %#v", fields)
}
active, known := stats.PureHydrogenModeFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
Fields: fields,
})
if !known || active {
t.Fatalf("pure-hydrogen mode = known:%v active:%v, want known inactive", known, active)
}
}
func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
@@ -1028,7 +1047,7 @@ func TestAddSamplesUsesCurrentDayFirstSampleBaseline(t *testing.T) {
}
}
func TestAddSamplesBackfillsOnlyContinuousPureHydrogenIntervals(t *testing.T) {
func TestAddSamplesBackfillsTotalMileageMinusPureElectricIntervals(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
base := stats.MetricSample{
VIN: "LTEST000000000001",
@@ -1061,8 +1080,8 @@ func TestAddSamplesBackfillsOnlyContinuousPureHydrogenIntervals(t *testing.T) {
t.Fatalf("aggregate count = %d, want 1", len(aggregates))
}
for _, agg := range aggregates {
if agg.PureHydrogenMileageKM != 5 {
t.Fatalf("pure-hydrogen mileage = %v, want 5", agg.PureHydrogenMileageKM)
if agg.PureHydrogenMileageKM != 7 {
t.Fatalf("pure-hydrogen mileage = %v, want 7", agg.PureHydrogenMileageKM)
}
if agg.PureHydrogenSampleCount != 2 {
t.Fatalf("pure-hydrogen sample count = %d, want 2", agg.PureHydrogenSampleCount)
@@ -1094,6 +1113,14 @@ func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
}
}
func TestParseVINsNormalizesAndDeduplicates(t *testing.T) {
got := parseVINs(" la9gg68l1pbaf4814, LFV123 ,LA9GG68L1PBAF4814, ")
want := []string{"LA9GG68L1PBAF4814", "LFV123"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("parseVINs() = %v, want %v", got, want)
}
}
func TestBackfillTimePredicatesUsePrimaryTimeForCoarseScanAndEventTimeForBusinessDay(t *testing.T) {
where := strings.Join(backfillTimePredicates(config{}, "2026-07-13", "2026-07-14"), " AND ")
for _, want := range []string{