fix(stats): reconcile stale gps mileage candidates

2026-07-20 03:51:18 +08:00
parent 5ba636419b
commit 758da369d8
6 changed files with 172 additions and 5 deletions
--- a/go/vehicle-gateway/cmd/stats-backfill/main.go
+++ b/go/vehicle-gateway/cmd/stats-backfill/main.go
@@ -155,6 +155,10 @@ func main() {
 		if err != nil {
 			fail("build GPS-coordinate fallback aggregates", err)
 		}
+		subthresholdFound, subthresholdDeleted, err := cleanupSubthresholdGPSMileageCandidates(ctx, mysqlDB, cfg, !cfg.DryRun)
+		if err != nil {
+			fail("cleanup sub-threshold GPS mileage candidates", err)
+		}
 		var written int64
 		var normalized int
 		if !cfg.DryRun {
@@ -167,7 +171,7 @@ func main() {
 				fail("normalize historical platform sources", err)
 			}
 		}
-		slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "jt808GPSFallbacks", gpsFallbacks, "written", written, "platformSourcesNormalized", normalized)
+		slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "jt808GPSFallbacks", gpsFallbacks, "subthresholdGPSCandidatesFound", subthresholdFound, "subthresholdGPSCandidatesDeleted", subthresholdDeleted, "written", written, "platformSourcesNormalized", normalized)
 		return
 	}

@@ -516,6 +520,100 @@ type gpsCoordinateBackfillState struct {
 	longGaps        int64
 }

+type subthresholdGPSMileageTarget struct {
+	vin      string
+	statDate string
+	protocol envelope.Protocol
+	rowCount int64
+}
+
+func cleanupSubthresholdGPSMileageCandidates(ctx context.Context, db *sql.DB, cfg config, apply bool) (int64, int64, error) {
+	if db == nil || len(cfg.Protocols) == 0 {
+		return 0, 0, nil
+	}
+	placeholders := make([]string, 0, len(cfg.Protocols))
+	args := []any{
+		cfg.DateFrom,
+		cfg.DateTo,
+		stats.QualityReasonGPSCoordinate,
+		stats.GPSMinimumDailyDistanceKM,
+	}
+	for _, protocol := range cfg.Protocols {
+		placeholders = append(placeholders, "?")
+		args = append(args, string(protocol))
+	}
+	query := `SELECT vin, DATE_FORMAT(stat_date, '%Y-%m-%d'), protocol, COUNT(*)
+FROM vehicle_daily_mileage_source
+WHERE stat_date >= ? AND stat_date <= ?
+  AND quality_reason = ?
+  AND daily_mileage_km >= 0 AND daily_mileage_km < ?
+  AND protocol IN (` + strings.Join(placeholders, ",") + `)
+GROUP BY vin, stat_date, protocol
+ORDER BY stat_date, protocol, vin`
+	rows, err := db.QueryContext(ctx, query, args...)
+	if err != nil {
+		return 0, 0, err
+	}
+	targets := make([]subthresholdGPSMileageTarget, 0)
+	var found int64
+	for rows.Next() {
+		var target subthresholdGPSMileageTarget
+		var protocol string
+		if err := rows.Scan(&target.vin, &target.statDate, &protocol, &target.rowCount); err != nil {
+			_ = rows.Close()
+			return found, 0, err
+		}
+		target.protocol = envelope.Protocol(strings.TrimSpace(protocol))
+		targets = append(targets, target)
+		found += target.rowCount
+	}
+	if err := rows.Err(); err != nil {
+		_ = rows.Close()
+		return found, 0, err
+	}
+	if err := rows.Close(); err != nil {
+		return found, 0, err
+	}
+	if !apply {
+		return found, 0, nil
+	}
+
+	var deleted int64
+	for _, target := range targets {
+		tx, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			return found, deleted, err
+		}
+		result, err := tx.ExecContext(ctx, `DELETE FROM vehicle_daily_mileage_source
+WHERE vin = ? AND stat_date = ? AND protocol = ?
+  AND quality_reason = ?
+  AND daily_mileage_km >= 0 AND daily_mileage_km < ?`,
+			target.vin,
+			target.statDate,
+			string(target.protocol),
+			stats.QualityReasonGPSCoordinate,
+			stats.GPSMinimumDailyDistanceKM,
+		)
+		if err == nil {
+			err = stats.ProjectDailyMileage(ctx, tx, target.vin, target.statDate, target.protocol)
+		}
+		if err != nil {
+			_ = tx.Rollback()
+			return found, deleted, err
+		}
+		affected, err := result.RowsAffected()
+		if err != nil {
+			_ = tx.Rollback()
+			return found, deleted, err
+		}
+		if err := tx.Commit(); err != nil {
+			return found, deleted, err
+		}
+		deleted += affected
+	}
+	return found, deleted, nil
+}
+
 func addGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
 	if !cfg.GPSFallback || tdDB == nil || aggregates == nil {
 		return 0, nil
--- a/go/vehicle-gateway/cmd/stats-backfill/main_test.go
+++ b/go/vehicle-gateway/cmd/stats-backfill/main_test.go
@@ -50,6 +50,73 @@ func TestChooseTrustedSourceKeepsContinuingSourceAndRejectsNewJump(t *testing.T)
 	}
 }

+func TestCleanupSubthresholdGPSMileageCandidatesDryRunReportsWithoutDeleting(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("sqlmock.New() error = %v", err)
+	}
+	defer db.Close()
+
+	mock.ExpectQuery("SELECT vin, DATE_FORMAT").
+		WithArgs("2026-07-20", "2026-07-20", stats.QualityReasonGPSCoordinate, stats.GPSMinimumDailyDistanceKM, "JT808").
+		WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol", "row_count"}).
+			AddRow("LKLG7C4E9NA774739", "2026-07-20", "JT808", 1))
+
+	found, deleted, err := cleanupSubthresholdGPSMileageCandidates(context.Background(), db, config{
+		DateFrom:  "2026-07-20",
+		DateTo:    "2026-07-20",
+		Protocols: []envelope.Protocol{envelope.ProtocolJT808},
+	}, false)
+	if err != nil {
+		t.Fatalf("cleanupSubthresholdGPSMileageCandidates() error = %v", err)
+	}
+	if found != 1 || deleted != 0 {
+		t.Fatalf("cleanup result found=%d deleted=%d", found, deleted)
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("unmet SQL expectations: %v", err)
+	}
+}
+
+func TestCleanupSubthresholdGPSMileageCandidatesDeletesAndReprojectsAtomically(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("sqlmock.New() error = %v", err)
+	}
+	defer db.Close()
+
+	mock.ExpectQuery("SELECT vin, DATE_FORMAT").
+		WithArgs("2026-07-20", "2026-07-20", stats.QualityReasonGPSCoordinate, stats.GPSMinimumDailyDistanceKM, "JT808").
+		WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol", "row_count"}).
+			AddRow("LKLG7C4E9NA774739", "2026-07-20", "JT808", 1))
+	mock.ExpectBegin()
+	mock.ExpectExec("DELETE FROM vehicle_daily_mileage_source").
+		WithArgs("LKLG7C4E9NA774739", "2026-07-20", "JT808", stats.QualityReasonGPSCoordinate, stats.GPSMinimumDailyDistanceKM).
+		WillReturnResult(sqlmock.NewResult(0, 1))
+	mock.ExpectExec("INSERT INTO vehicle_daily_mileage").
+		WillReturnResult(sqlmock.NewResult(0, 0))
+	mock.ExpectExec("UPDATE vehicle_daily_mileage_source").
+		WillReturnResult(sqlmock.NewResult(0, 0))
+	mock.ExpectExec("DELETE FROM vehicle_daily_mileage").
+		WillReturnResult(sqlmock.NewResult(0, 1))
+	mock.ExpectCommit()
+
+	found, deleted, err := cleanupSubthresholdGPSMileageCandidates(context.Background(), db, config{
+		DateFrom:  "2026-07-20",
+		DateTo:    "2026-07-20",
+		Protocols: []envelope.Protocol{envelope.ProtocolJT808},
+	}, true)
+	if err != nil {
+		t.Fatalf("cleanupSubthresholdGPSMileageCandidates() error = %v", err)
+	}
+	if found != 1 || deleted != 1 {
+		t.Fatalf("cleanup result found=%d deleted=%d", found, deleted)
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("unmet SQL expectations: %v", err)
+	}
+}
+
 func TestChooseTrustedSourceAcceptsSmallNegativeMileageJitter(t *testing.T) {
 	previous := []dailySourceLast{{
 		VIN:            "LB9A32A28R0LS1574",