shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: expand vehicle data platform capabilities

Browse Source
This commit is contained in:
lingniu
2026-07-27 16:46:15 +08:00
parent e3a1f80f86
commit 3c4bece72c
650 changed files with 62155 additions and 2552 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
go/vehicle-gateway/cmd/stats-backfill/main.go
View File

@@ -57,23 +57,27 @@ type rawFrameRow struct {
}
type metricAgg struct {
VIN string
Date string
Protocol envelope.Protocol
FirstKM float64
LatestKM float64
Count int64
SourceKey string
Phone string
DeviceID string
SourceEndpoint string
SourceCode string
PlatformName string
SourceKind string
FirstEventTime time.Time
LatestEventTime time.Time
QualityStatus string
QualityReason string
VIN string
Date string
Protocol envelope.Protocol
FirstKM float64
LatestKM float64
PureHydrogenMileageKM float64
Count int64
PureHydrogenSampleCount int64
SourceKey string
Phone string
DeviceID string
SourceEndpoint string
SourceCode string
PlatformName string
SourceKind string
FirstEventTime time.Time
LatestEventTime time.Time
LatestPureHydrogenActive bool
LatestPureHydrogenModeKnown bool
QualityStatus string
QualityReason string
}
type dailySourceLast struct {
@@ -266,21 +270,24 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
agg, ok := aggregates[key]
if !ok {
aggregates[key] = &metricAgg{
VIN: sample.VIN,
Date: sample.StatDate,
Protocol: sample.Protocol,
FirstKM: sample.TotalMileageKM,
LatestKM: sample.TotalMileageKM,
Count: 1,
SourceKey: sample.SourceKey,
Phone: sample.Phone,
DeviceID: sample.DeviceID,
SourceEndpoint: sample.SourceEndpoint,
PlatformName: sample.PlatformName,
FirstEventTime: sample.EventTime,
LatestEventTime: sample.EventTime,
QualityStatus: stats.QualityOK,
QualityReason: "current_day_first_sample",
VIN: sample.VIN,
Date: sample.StatDate,
Protocol: sample.Protocol,
FirstKM: sample.TotalMileageKM,
LatestKM: sample.TotalMileageKM,
Count: 1,
PureHydrogenSampleCount: activeModeSampleCount(sample),
SourceKey: sample.SourceKey,
Phone: sample.Phone,
DeviceID: sample.DeviceID,
SourceEndpoint: sample.SourceEndpoint,
PlatformName: sample.PlatformName,
FirstEventTime: sample.EventTime,
LatestEventTime: sample.EventTime,
LatestPureHydrogenActive: sample.PureHydrogenActive,
LatestPureHydrogenModeKnown: sample.PureHydrogenModeKnown,
QualityStatus: stats.QualityOK,
QualityReason: "current_day_first_sample",
}
continue
}
@@ -289,8 +296,18 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
agg.FirstEventTime = sample.EventTime
}
if sample.EventTime.After(agg.LatestEventTime) {
agg.PureHydrogenMileageKM += stats.PureHydrogenMileageDelta(
agg.LatestKM,
sample.TotalMileageKM,
agg.LatestPureHydrogenActive,
agg.LatestPureHydrogenModeKnown,
sample.PureHydrogenActive,
sample.PureHydrogenModeKnown,
)
agg.LatestKM = sample.TotalMileageKM
agg.LatestEventTime = sample.EventTime
agg.LatestPureHydrogenActive = sample.PureHydrogenActive
agg.LatestPureHydrogenModeKnown = sample.PureHydrogenModeKnown
agg.SourceEndpoint = sample.SourceEndpoint
if strings.TrimSpace(sample.Phone) != "" {
agg.Phone = sample.Phone
@@ -303,9 +320,17 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
}
}
agg.Count++
agg.PureHydrogenSampleCount += activeModeSampleCount(sample)
}
}
func activeModeSampleCount(sample stats.MetricSample) int64 {
if sample.PureHydrogenModeKnown && sample.PureHydrogenActive {
return 1
}
return 0
}
func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg, batchSize int) (int64, error) {
if len(aggregates) == 0 {
return 0, nil
@@ -338,23 +363,27 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
}
dailyKM := stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM)
candidate := stats.SourceMileageSample{
VIN: agg.VIN,
StatDate: agg.Date,
Protocol: agg.Protocol,
SourceKey: agg.SourceKey,
SourceIP: identity.SourceIP,
SourceEndpoint: agg.SourceEndpoint,
Phone: agg.Phone,
DeviceID: agg.DeviceID,
PlatformName: agg.PlatformName,
FirstTotalKM: agg.FirstKM,
LatestTotalKM: agg.LatestKM,
DailyKM: dailyKM,
SampleCount: agg.Count,
FirstEventTime: agg.FirstEventTime,
LatestEventTime: agg.LatestEventTime,
QualityStatus: agg.QualityStatus,
QualityReason: agg.QualityReason,
VIN: agg.VIN,
StatDate: agg.Date,
Protocol: agg.Protocol,
SourceKey: agg.SourceKey,
SourceIP: identity.SourceIP,
SourceEndpoint: agg.SourceEndpoint,
Phone: agg.Phone,
DeviceID: agg.DeviceID,
PlatformName: agg.PlatformName,
FirstTotalKM: agg.FirstKM,
LatestTotalKM: agg.LatestKM,
DailyKM: dailyKM,
PureHydrogenMileageKM: agg.PureHydrogenMileageKM,
SampleCount: agg.Count,
PureHydrogenSampleCount: agg.PureHydrogenSampleCount,
FirstEventTime: agg.FirstEventTime,
LatestEventTime: agg.LatestEventTime,
LatestPureHydrogenActive: agg.LatestPureHydrogenActive,
LatestPureHydrogenModeKnown: agg.LatestPureHydrogenModeKnown,
QualityStatus: agg.QualityStatus,
QualityReason: agg.QualityReason,
}
if candidate.QualityStatus == "" {
candidate.QualityStatus = stats.QualityOK
@@ -1266,9 +1295,49 @@ func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[str
}
fields[key] = value
}
promoteFuelCellWorkModeForStats(protocol, fields, text)
return fields
}
func promoteFuelCellWorkModeForStats(protocol envelope.Protocol, fields map[string]any, text string) {
if fields == nil {
return
}
if _, exists := fields[envelope.FieldFuelCellWorkMode]; exists {
return
}
var candidates []string
switch protocol {
case envelope.ProtocolGB32960:
candidates = []string{
"gb32960.gd_fc_stack.engine_work_state",
"engine_work_state",
}
case envelope.ProtocolYutongMQTT:
candidates = []string{
"yutong_mqtt.data.triangle_state",
"yutong_mqtt.root.data.triangle_state",
"TRIANGLE_STATE",
"triangle_state",
"triangleState",
}
default:
return
}
for _, key := range candidates {
if value, exists := fields[key]; exists {
fields[envelope.FieldFuelCellWorkMode] = value
return
}
}
for _, key := range candidates {
if value, ok := extractJSONStringField(text, key); ok {
fields[envelope.FieldFuelCellWorkMode] = value
return
}
}
}
func mileageKeys(protocol envelope.Protocol) []string {
switch protocol {
case envelope.ProtocolGB32960:

81
go/vehicle-gateway/cmd/stats-backfill/main_test.go
View File

@@ -763,7 +763,8 @@ func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
fields := fieldsForStats(envelope.ProtocolYutongMQTT, "LMRKH9AC6R1004108", `{
"data": {
"TOTAL_MILEAGE": 65423000,
"METER_SPEED": 12.3
"METER_SPEED": 12.3,
"TRIANGLE_STATE": 11
},
"root": {
"device": "LMRKH9AC6R1004108"
@@ -773,6 +774,9 @@ func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
if got := fields["yutong_mqtt.data.total_mileage"]; got == nil {
t.Fatalf("fields missing raw yutong total mileage: %#v", fields)
}
if got := fields[envelope.FieldFuelCellWorkMode]; got == nil {
t.Fatalf("fields missing raw yutong fuel-cell work mode: %#v", fields)
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
@@ -791,6 +795,32 @@ func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
if samples[0].TotalMileageKM != 65423 {
t.Fatalf("total mileage km = %v, want 65423", samples[0].TotalMileageKM)
}
if !samples[0].PureHydrogenModeKnown || !samples[0].PureHydrogenActive {
t.Fatalf("pure-hydrogen mode = known:%v active:%v, want known active", samples[0].PureHydrogenModeKnown, samples[0].PureHydrogenActive)
}
}
func TestFieldsForStatsPromotesGB32960FuelCellEngineWorkState(t *testing.T) {
fields := fieldsForStats(envelope.ProtocolGB32960, "LTEST000000000001", `{
"data_units": [{
"type": "0x30",
"name": "gd_fc_stack",
"value": {
"summaries": [{"engine_work_state": 2}]
}
}]
}`)
if got := fields[envelope.FieldFuelCellWorkMode]; got == nil {
t.Fatalf("fields missing GB32960 fuel-cell work mode: %#v", fields)
}
active, known := stats.PureHydrogenModeFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
Fields: fields,
})
if !known || !active {
t.Fatalf("pure-hydrogen mode = known:%v active:%v, want known active", known, active)
}
}
func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) {
@@ -863,9 +893,13 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
float64(4100),
float64(4101),
float64(1),
float64(0),
int64(1),
int64(0),
projNow,
projNow,
false,
false,
stats.QualityInvalidDelta,
"outside_daily_range",
).
@@ -994,6 +1028,51 @@ func TestAddSamplesUsesCurrentDayFirstSampleBaseline(t *testing.T) {
}
}
func TestAddSamplesBackfillsOnlyContinuousPureHydrogenIntervals(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
base := stats.MetricSample{
VIN: "LTEST000000000001",
Protocol: envelope.ProtocolGB32960,
StatDate: "2026-07-08",
SourceKey: "GB32960|127.0.0.1",
PureHydrogenModeKnown: true,
}
samples := []stats.MetricSample{
base,
base,
base,
base,
}
samples[0].TotalMileageKM = 100
samples[0].EventTime = time.Date(2026, 7, 8, 8, 0, 0, 0, loc)
samples[1].TotalMileageKM = 102
samples[1].EventTime = time.Date(2026, 7, 8, 9, 0, 0, 0, loc)
samples[1].PureHydrogenActive = true
samples[2].TotalMileageKM = 107
samples[2].EventTime = time.Date(2026, 7, 8, 10, 0, 0, 0, loc)
samples[2].PureHydrogenActive = true
samples[3].TotalMileageKM = 110
samples[3].EventTime = time.Date(2026, 7, 8, 11, 0, 0, 0, loc)
aggregates := map[string]*metricAgg{}
addSamples(aggregates, samples)
if len(aggregates) != 1 {
t.Fatalf("aggregate count = %d, want 1", len(aggregates))
}
for _, agg := range aggregates {
if agg.PureHydrogenMileageKM != 5 {
t.Fatalf("pure-hydrogen mileage = %v, want 5", agg.PureHydrogenMileageKM)
}
if agg.PureHydrogenSampleCount != 2 {
t.Fatalf("pure-hydrogen sample count = %d, want 2", agg.PureHydrogenSampleCount)
}
if !agg.LatestPureHydrogenModeKnown || agg.LatestPureHydrogenActive {
t.Fatalf("latest mode = known:%v active:%v, want known inactive", agg.LatestPureHydrogenModeKnown, agg.LatestPureHydrogenActive)
}
}
}
func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
t.Setenv("BACKFILL_METHOD", "")
t.Setenv("BACKFILL_DAYS_BACK", "")