shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(stats): elect trusted mileage source

Browse Source
This commit is contained in:
lingniu
2026-07-08 13:31:56 +08:00
parent abfed27846
commit a60a628d25
5 changed files with 284 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
go/vehicle-gateway/cmd/stats-backfill/main.go
View File

@@ -49,12 +49,24 @@ type rawFrameRow struct {
}
type metricAgg struct {
VIN string
Date string
Protocol envelope.Protocol
FirstKM float64
LatestKM float64
Count int64
VIN string
Date string
Protocol envelope.Protocol
FirstKM float64
LatestKM float64
Count int64
SourceKey string
Phone string
SourceEndpoint string
}
type dailySourceLast struct {
VIN string
SourceKey string
Phone string
SourceEndpoint string
TS time.Time
TotalKM float64
}
func main() {
@@ -249,7 +261,7 @@ func writeAggregateBatch(ctx context.Context, db *sql.DB, rows []*metricAgg) err
if daily < 0 {
daily = 0
}
placeholders = append(placeholders, "(?,?,?,?,?,?,?)")
placeholders = append(placeholders, "(?,?,?,?,?,?,?,?,?,?)")
args = append(args,
row.VIN,
row.Date,
@@ -257,16 +269,23 @@ func writeAggregateBatch(ctx context.Context, db *sql.DB, rows []*metricAgg) err
daily,
row.FirstKM,
row.LatestKM,
row.SourceKey,
row.Phone,
row.SourceEndpoint,
row.Count,
)
}
sqlText := `INSERT INTO vehicle_daily_mileage
(vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count)
(vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km,
trusted_source_key, trusted_phone, trusted_source_endpoint, sample_count)
VALUES ` + strings.Join(placeholders, ",") + `
ON DUPLICATE KEY UPDATE
daily_mileage_km = VALUES(daily_mileage_km),
first_total_mileage_km = VALUES(first_total_mileage_km),
latest_total_mileage_km = VALUES(latest_total_mileage_km),
trusted_source_key = VALUES(trusted_source_key),
trusted_phone = VALUES(trusted_phone),
trusted_source_endpoint = VALUES(trusted_source_endpoint),
sample_count = VALUES(sample_count),
updated_at = CURRENT_TIMESTAMP`
_, err := db.ExecContext(ctx, sqlText, args...)
@@ -278,11 +297,11 @@ func buildLastDiffAggregates(ctx context.Context, db *sql.DB, cfg config) (map[s
if err != nil {
return nil, err
}
lastByProtocolDate := map[envelope.Protocol]map[string]map[string]float64{}
lastByProtocolDate := map[envelope.Protocol]map[string]map[string][]dailySourceLast{}
for _, protocol := range cfg.Protocols {
lastByProtocolDate[protocol] = map[string]map[string]float64{}
lastByProtocolDate[protocol] = map[string]map[string][]dailySourceLast{}
for _, date := range dates {
rows, err := queryDailyLastRows(ctx, db, cfg, protocol, date)
rows, err := queryDailyLastSourceRows(ctx, db, cfg, protocol, date)
if err != nil {
return nil, err
}
@@ -300,19 +319,22 @@ func buildLastDiffAggregates(ctx context.Context, db *sql.DB, cfg config) (map[s
prevDate := previousDate(date)
current := lastByProtocolDate[protocol][date]
previous := lastByProtocolDate[protocol][prevDate]
for vin, latest := range current {
first, ok := previous[vin]
if !ok || latest < first {
for vin, currentSources := range current {
chosen, ok := chooseTrustedSource(currentSources, previous[vin])
if !ok {
continue
}
key := vin + "|" + date + "|" + string(protocol)
aggregates[key] = &metricAgg{
VIN: vin,
Date: date,
Protocol: protocol,
FirstKM: first,
LatestKM: latest,
Count: 1,
VIN: vin,
Date: date,
Protocol: protocol,
FirstKM: chosen.previous.TotalKM,
LatestKM: chosen.current.TotalKM,
Count: 1,
SourceKey: chosen.current.SourceKey,
Phone: chosen.current.Phone,
SourceEndpoint: chosen.current.SourceEndpoint,
}
}
}
@@ -320,7 +342,38 @@ func buildLastDiffAggregates(ctx context.Context, db *sql.DB, cfg config) (map[s
return aggregates, nil
}
func queryDailyLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string]float64, error) {
type trustedChoice struct {
current dailySourceLast
previous dailySourceLast
}
const maxTrustedDailyMileageKM = 1000
func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast) (trustedChoice, bool) {
previousBySource := map[string]dailySourceLast{}
for _, row := range previous {
previousBySource[row.SourceKey] = row
}
var chosen trustedChoice
var chosenDelta float64
for _, currentRow := range current {
previousRow, ok := previousBySource[currentRow.SourceKey]
if !ok {
continue
}
delta := currentRow.TotalKM - previousRow.TotalKM
if delta < 0 || delta > maxTrustedDailyMileageKM {
continue
}
if chosen.current.SourceKey == "" || delta < chosenDelta {
chosen = trustedChoice{current: currentRow, previous: previousRow}
chosenDelta = delta
}
}
return chosen, chosen.current.SourceKey != ""
}
func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
where := []string{
fmt.Sprintf("ts >= '%s 00:00:00'", quote(date)),
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(date))),
@@ -330,20 +383,23 @@ func queryDailyLastRows(ctx context.Context, db *sql.DB, cfg config, protocol en
fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
realtimeMileageFramePredicate(),
}
sqlText := fmt.Sprintf(`SELECT vin, LAST(parsed_json)
sqlText := fmt.Sprintf(`SELECT vin, phone, source_endpoint, LAST(ts), LAST(parsed_json)
FROM %s.raw_frames
WHERE %s
GROUP BY vin`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
GROUP BY vin, phone, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
rows, err := db.QueryContext(ctx, sqlText)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]float64{}
latestBySource := map[string]dailySourceLast{}
for rows.Next() {
var vin string
var phone string
var sourceEndpoint string
var ts time.Time
var parsedJSON string
if err := rows.Scan(&vin, &parsedJSON); err != nil {
if err := rows.Scan(&vin, &phone, &sourceEndpoint, &ts, &parsedJSON); err != nil {
return nil, err
}
fields := fieldsForStats(protocol, vin, parsedJSON)
@@ -360,9 +416,53 @@ GROUP BY vin`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
if err != nil || len(samples) == 0 {
continue
}
out[strings.TrimSpace(vin)] = samples[0].TotalMileageKM
sourceKey := normalizedSourceKey(phone, sourceEndpoint)
if sourceKey == "" {
continue
}
row := dailySourceLast{
VIN: strings.TrimSpace(vin),
SourceKey: sourceKey,
Phone: strings.TrimSpace(phone),
SourceEndpoint: strings.TrimSpace(sourceEndpoint),
TS: ts,
TotalKM: samples[0].TotalMileageKM,
}
key := row.VIN + "|" + row.SourceKey
if existing, ok := latestBySource[key]; !ok || row.TS.After(existing.TS) {
latestBySource[key] = row
}
}
return out, rows.Err()
if err := rows.Err(); err != nil {
return nil, err
}
out := map[string][]dailySourceLast{}
for _, row := range latestBySource {
out[row.VIN] = append(out[row.VIN], row)
}
return out, nil
}
func normalizedSourceKey(phone string, endpoint string) string {
var parts []string
if phone = strings.TrimSpace(phone); phone != "" {
parts = append(parts, phone)
}
if host := endpointHost(endpoint); host != "" {
parts = append(parts, host)
}
return strings.Join(parts, "@")
}
func endpointHost(endpoint string) string {
endpoint = strings.TrimSpace(endpoint)
if endpoint == "" {
return ""
}
if host, _, ok := strings.Cut(endpoint, ":"); ok {
return strings.TrimSpace(host)
}
return endpoint
}
func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[string]any {