fix(mileage): recover sparse source daily distance
This commit is contained in:
lingniu
@@ -36,7 +36,7 @@ type config struct {
|
||||
Debug bool
|
||||
EventTimeFullScan bool
|
||||
StationaryCarry bool
|
||||
JT808GPSFallback bool
|
||||
GPSFallback bool
|
||||
ProgressEvery int64
|
||||
BaselineLookback int
|
||||
Location *time.Location
|
||||
@@ -151,9 +151,9 @@ func main() {
|
||||
if err != nil {
|
||||
fail("build realtime-location fallback aggregates", err)
|
||||
}
|
||||
gpsFallbacks, err := addJT808GPSCoordinateFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
|
||||
gpsFallbacks, err := addGPSCoordinateFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates)
|
||||
if err != nil {
|
||||
fail("build JT808 GPS-coordinate fallback aggregates", err)
|
||||
fail("build GPS-coordinate fallback aggregates", err)
|
||||
}
|
||||
var written int64
|
||||
var normalized int
|
||||
@@ -516,29 +516,47 @@ type gpsCoordinateBackfillState struct {
|
||||
longGaps int64
|
||||
}
|
||||
|
||||
func addJT808GPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
|
||||
if !cfg.JT808GPSFallback || tdDB == nil || aggregates == nil || !containsProtocol(cfg.Protocols, envelope.ProtocolJT808) {
|
||||
func addGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
|
||||
if !cfg.GPSFallback || tdDB == nil || aggregates == nil {
|
||||
return 0, nil
|
||||
}
|
||||
existing, err := existingJT808OdometerTargets(ctx, mysqlDB, cfg)
|
||||
totalAdded := 0
|
||||
for _, protocol := range cfg.Protocols {
|
||||
added, err := addProtocolGPSCoordinateFallbackAggregates(ctx, mysqlDB, tdDB, cfg, aggregates, protocol)
|
||||
if err != nil {
|
||||
return totalAdded, err
|
||||
}
|
||||
totalAdded += added
|
||||
}
|
||||
return totalAdded, nil
|
||||
}
|
||||
|
||||
func addProtocolGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg, protocol envelope.Protocol) (int, error) {
|
||||
if !supportsBackfillGPSFallback(protocol) {
|
||||
return 0, nil
|
||||
}
|
||||
existing, err := existingPositiveOdometerTargets(ctx, mysqlDB, cfg, protocol)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, agg := range aggregates {
|
||||
if agg != nil && agg.Protocol == envelope.ProtocolJT808 && agg.QualityReason != stats.QualityReasonGPSCoordinate {
|
||||
if agg != nil && agg.Protocol == protocol && agg.QualityReason != stats.QualityReasonGPSCoordinate && agg.LatestKM > agg.FirstKM {
|
||||
existing[agg.VIN+"|"+agg.Date] = struct{}{}
|
||||
}
|
||||
}
|
||||
where := []string{
|
||||
fmt.Sprintf("ts >= '%s 00:00:00'", quote(cfg.DateFrom)),
|
||||
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(cfg.DateTo))),
|
||||
"protocol = 'JT808'",
|
||||
fmt.Sprintf("protocol = '%s'", quote(string(protocol))),
|
||||
"vin IS NOT NULL",
|
||||
"vin <> ''",
|
||||
"longitude BETWEEN -180 AND 180",
|
||||
"latitude BETWEEN -90 AND 90",
|
||||
"NOT (longitude = 0 AND latitude = 0)",
|
||||
}
|
||||
if protocol != envelope.ProtocolJT808 {
|
||||
where = append(where, "speed_kmh > 0")
|
||||
}
|
||||
sqlText := fmt.Sprintf(`SELECT vin, ts, longitude, latitude
|
||||
FROM %s.vehicle_locations
|
||||
WHERE %s
|
||||
@@ -566,7 +584,7 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
current := stats.GPSMileagePoint{
|
||||
VIN: vin,
|
||||
StatDate: date,
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Protocol: protocol,
|
||||
EventTime: eventTime,
|
||||
Longitude: longitude,
|
||||
Latitude: latitude,
|
||||
@@ -612,18 +630,18 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
continue
|
||||
}
|
||||
vin, date := parts[0], parts[1]
|
||||
sourceKey := string(envelope.ProtocolJT808) + ":" + vin + "@GPS_COORDINATE"
|
||||
key := vin + "|" + date + "|" + string(envelope.ProtocolJT808) + "|" + sourceKey
|
||||
sourceKey := string(protocol) + ":" + vin + "@GPS_COORDINATE"
|
||||
key := vin + "|" + date + "|" + string(protocol) + "|" + sourceKey
|
||||
aggregates[key] = &metricAgg{
|
||||
VIN: vin,
|
||||
Date: date,
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Protocol: protocol,
|
||||
FirstKM: 0,
|
||||
LatestKM: state.distanceKM,
|
||||
Count: state.pointCount,
|
||||
SourceKey: sourceKey,
|
||||
SourceEndpoint: "gps-coordinate",
|
||||
PlatformName: "JT808 GPS轨迹估算",
|
||||
PlatformName: backfillGPSPlatformName(protocol),
|
||||
SourceKind: "UNKNOWN",
|
||||
FirstEventTime: state.firstEventTime,
|
||||
LatestEventTime: state.latestEventTime,
|
||||
@@ -633,23 +651,44 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
added++
|
||||
}
|
||||
if len(states) > 0 {
|
||||
slog.Info("JT808 GPS-coordinate fallbacks loaded", "states", len(states), "added", added)
|
||||
slog.Info("GPS-coordinate fallbacks loaded", "protocol", protocol, "states", len(states), "added", added)
|
||||
}
|
||||
return added, nil
|
||||
}
|
||||
|
||||
func existingJT808OdometerTargets(ctx context.Context, db *sql.DB, cfg config) (map[string]struct{}, error) {
|
||||
func supportsBackfillGPSFallback(protocol envelope.Protocol) bool {
|
||||
switch protocol {
|
||||
case envelope.ProtocolGB32960, envelope.ProtocolJT808, envelope.ProtocolYutongMQTT:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func backfillGPSPlatformName(protocol envelope.Protocol) string {
|
||||
switch protocol {
|
||||
case envelope.ProtocolGB32960:
|
||||
return "GB32960 GPS轨迹估算"
|
||||
case envelope.ProtocolYutongMQTT:
|
||||
return "宇通 GPS轨迹估算"
|
||||
default:
|
||||
return "JT808 GPS轨迹估算"
|
||||
}
|
||||
}
|
||||
|
||||
func existingPositiveOdometerTargets(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol) (map[string]struct{}, error) {
|
||||
result := map[string]struct{}{}
|
||||
if db == nil {
|
||||
return result, nil
|
||||
}
|
||||
rows, err := db.QueryContext(ctx, `SELECT DISTINCT vin, DATE_FORMAT(stat_date, '%Y-%m-%d')
|
||||
FROM vehicle_daily_mileage_source
|
||||
WHERE protocol = 'JT808'
|
||||
WHERE protocol = ?
|
||||
AND stat_date >= ? AND stat_date <= ?
|
||||
AND quality_status = ?
|
||||
AND COALESCE(quality_reason, '') <> ?
|
||||
AND latest_total_mileage_km IS NOT NULL`, cfg.DateFrom, cfg.DateTo, stats.QualityOK, stats.QualityReasonGPSCoordinate)
|
||||
AND daily_mileage_km > 0
|
||||
AND latest_total_mileage_km IS NOT NULL`, string(protocol), cfg.DateFrom, cfg.DateTo, stats.QualityOK, stats.QualityReasonGPSCoordinate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1184,7 +1223,7 @@ func loadConfig() (config, error) {
|
||||
Debug: envBool("BACKFILL_DEBUG", false),
|
||||
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
|
||||
StationaryCarry: envBool("BACKFILL_STATIONARY_CARRY_FORWARD", false),
|
||||
JT808GPSFallback: envBool("BACKFILL_JT808_GPS_FALLBACK", true),
|
||||
GPSFallback: envBool("BACKFILL_GPS_FALLBACK", envBool("BACKFILL_JT808_GPS_FALLBACK", true)),
|
||||
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
|
||||
BaselineLookback: envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
|
||||
Location: loc,
|
||||
|
||||
Reference in New Issue
Block a user