shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(stats): reconcile stale gps mileage candidates

Browse Source
This commit is contained in:
lingniu
2026-07-20 03:51:18 +08:00
parent 5ba636419b
commit 758da369d8
6 changed files with 172 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
go/vehicle-gateway/cmd/stats-backfill/main.go
View File

@@ -155,6 +155,10 @@ func main() {
if err != nil {
fail("build GPS-coordinate fallback aggregates", err)
}
subthresholdFound, subthresholdDeleted, err := cleanupSubthresholdGPSMileageCandidates(ctx, mysqlDB, cfg, !cfg.DryRun)
if err != nil {
fail("cleanup sub-threshold GPS mileage candidates", err)
}
var written int64
var normalized int
if !cfg.DryRun {
@@ -167,7 +171,7 @@ func main() {
fail("normalize historical platform sources", err)
}
}
slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "jt808GPSFallbacks", gpsFallbacks, "written", written, "platformSourcesNormalized", normalized)
slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "jt808GPSFallbacks", gpsFallbacks, "subthresholdGPSCandidatesFound", subthresholdFound, "subthresholdGPSCandidatesDeleted", subthresholdDeleted, "written", written, "platformSourcesNormalized", normalized)
return
}
@@ -516,6 +520,100 @@ type gpsCoordinateBackfillState struct {
longGaps int64
}
type subthresholdGPSMileageTarget struct {
vin string
statDate string
protocol envelope.Protocol
rowCount int64
}
func cleanupSubthresholdGPSMileageCandidates(ctx context.Context, db *sql.DB, cfg config, apply bool) (int64, int64, error) {
if db == nil || len(cfg.Protocols) == 0 {
return 0, 0, nil
}
placeholders := make([]string, 0, len(cfg.Protocols))
args := []any{
cfg.DateFrom,
cfg.DateTo,
stats.QualityReasonGPSCoordinate,
stats.GPSMinimumDailyDistanceKM,
}
for _, protocol := range cfg.Protocols {
placeholders = append(placeholders, "?")
args = append(args, string(protocol))
}
query := `SELECT vin, DATE_FORMAT(stat_date, '%Y-%m-%d'), protocol, COUNT(*)
FROM vehicle_daily_mileage_source
WHERE stat_date >= ? AND stat_date <= ?
AND quality_reason = ?
AND daily_mileage_km >= 0 AND daily_mileage_km < ?
AND protocol IN (` + strings.Join(placeholders, ",") + `)
GROUP BY vin, stat_date, protocol
ORDER BY stat_date, protocol, vin`
rows, err := db.QueryContext(ctx, query, args...)
if err != nil {
return 0, 0, err
}
targets := make([]subthresholdGPSMileageTarget, 0)
var found int64
for rows.Next() {
var target subthresholdGPSMileageTarget
var protocol string
if err := rows.Scan(&target.vin, &target.statDate, &protocol, &target.rowCount); err != nil {
_ = rows.Close()
return found, 0, err
}
target.protocol = envelope.Protocol(strings.TrimSpace(protocol))
targets = append(targets, target)
found += target.rowCount
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return found, 0, err
}
if err := rows.Close(); err != nil {
return found, 0, err
}
if !apply {
return found, 0, nil
}
var deleted int64
for _, target := range targets {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return found, deleted, err
}
result, err := tx.ExecContext(ctx, `DELETE FROM vehicle_daily_mileage_source
WHERE vin = ? AND stat_date = ? AND protocol = ?
AND quality_reason = ?
AND daily_mileage_km >= 0 AND daily_mileage_km < ?`,
target.vin,
target.statDate,
string(target.protocol),
stats.QualityReasonGPSCoordinate,
stats.GPSMinimumDailyDistanceKM,
)
if err == nil {
err = stats.ProjectDailyMileage(ctx, tx, target.vin, target.statDate, target.protocol)
}
if err != nil {
_ = tx.Rollback()
return found, deleted, err
}
affected, err := result.RowsAffected()
if err != nil {
_ = tx.Rollback()
return found, deleted, err
}
if err := tx.Commit(); err != nil {
return found, deleted, err
}
deleted += affected
}
return found, deleted, nil
}
func addGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
if !cfg.GPSFallback || tdDB == nil || aggregates == nil {
return 0, nil

67
go/vehicle-gateway/cmd/stats-backfill/main_test.go
View File

@@ -50,6 +50,73 @@ func TestChooseTrustedSourceKeepsContinuingSourceAndRejectsNewJump(t *testing.T)
}
}
func TestCleanupSubthresholdGPSMileageCandidatesDryRunReportsWithoutDeleting(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT vin, DATE_FORMAT").
WithArgs("2026-07-20", "2026-07-20", stats.QualityReasonGPSCoordinate, stats.GPSMinimumDailyDistanceKM, "JT808").
WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol", "row_count"}).
AddRow("LKLG7C4E9NA774739", "2026-07-20", "JT808", 1))
found, deleted, err := cleanupSubthresholdGPSMileageCandidates(context.Background(), db, config{
DateFrom: "2026-07-20",
DateTo: "2026-07-20",
Protocols: []envelope.Protocol{envelope.ProtocolJT808},
}, false)
if err != nil {
t.Fatalf("cleanupSubthresholdGPSMileageCandidates() error = %v", err)
}
if found != 1 || deleted != 0 {
t.Fatalf("cleanup result found=%d deleted=%d", found, deleted)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet SQL expectations: %v", err)
}
}
func TestCleanupSubthresholdGPSMileageCandidatesDeletesAndReprojectsAtomically(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT vin, DATE_FORMAT").
WithArgs("2026-07-20", "2026-07-20", stats.QualityReasonGPSCoordinate, stats.GPSMinimumDailyDistanceKM, "JT808").
WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol", "row_count"}).
AddRow("LKLG7C4E9NA774739", "2026-07-20", "JT808", 1))
mock.ExpectBegin()
mock.ExpectExec("DELETE FROM vehicle_daily_mileage_source").
WithArgs("LKLG7C4E9NA774739", "2026-07-20", "JT808", stats.QualityReasonGPSCoordinate, stats.GPSMinimumDailyDistanceKM).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("INSERT INTO vehicle_daily_mileage").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("UPDATE vehicle_daily_mileage_source").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("DELETE FROM vehicle_daily_mileage").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
found, deleted, err := cleanupSubthresholdGPSMileageCandidates(context.Background(), db, config{
DateFrom: "2026-07-20",
DateTo: "2026-07-20",
Protocols: []envelope.Protocol{envelope.ProtocolJT808},
}, true)
if err != nil {
t.Fatalf("cleanupSubthresholdGPSMileageCandidates() error = %v", err)
}
if found != 1 || deleted != 1 {
t.Fatalf("cleanup result found=%d deleted=%d", found, deleted)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet SQL expectations: %v", err)
}
}
func TestChooseTrustedSourceAcceptsSmallNegativeMileageJitter(t *testing.T) {
previous := []dailySourceLast{{
VIN: "LB9A32A28R0LS1574",

4
go/vehicle-gateway/internal/stats/gps_mileage.go
View File

@@ -17,7 +17,7 @@ const (
gpsCoordinateSourceSuffix = "#GPS_COORDINATE"
gpsMaxSegmentGap = 10 * time.Minute
gpsMaxImpliedSpeedKMH = 220.0
gpsMinimumDailyDistanceKM = 0.1
GPSMinimumDailyDistanceKM = 0.1
)
const GPSMileageStateTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_gps_mileage_state (
@@ -266,7 +266,7 @@ func AccumulateGPSMileage(ctx context.Context, exec Execer, point GPSMileagePoin
func GPSMileageCandidateEligible(distanceKM float64, usableSegmentCount int64) bool {
return usableSegmentCount > 0 &&
isFiniteNonNegative(distanceKM) &&
distanceKM >= gpsMinimumDailyDistanceKM
distanceKM >= GPSMinimumDailyDistanceKM
}
// AccumulateJT808GPSMileage remains as a compatibility wrapper for callers and

2
go/vehicle-gateway/internal/stats/gps_mileage_test.go
View File

@@ -299,7 +299,7 @@ func TestAccumulateGPSMileageKeepsSubThresholdDistanceAsStateOnly(t *testing.T)
if err != nil {
t.Fatalf("AccumulateGPSMileage() error = %v", err)
}
if recovered || state.DailyMileageKM <= 0 || state.DailyMileageKM >= gpsMinimumDailyDistanceKM {
if recovered || state.DailyMileageKM <= 0 || state.DailyMileageKM >= GPSMinimumDailyDistanceKM {
t.Fatalf("sub-threshold state = %+v recovered=%v", state, recovered)
}
if err := mock.ExpectationsWereMet(); err != nil {