feat: productionize raw history ingestion

2026-06-30 23:52:16 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
@@ -23,6 +23,9 @@ import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
 import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -36,7 +39,11 @@ import java.nio.file.Path;
 import java.time.Duration;
 import java.time.ZoneId;

-@AutoConfiguration
+@AutoConfiguration(after = {
+        DataSourceAutoConfiguration.class,
+        JdbcTemplateAutoConfiguration.class,
+        RedisAutoConfiguration.class
+})
@EnableConfigurationProperties(VehicleStatProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.vehicle-stat", name = "enabled", havingValue = "true")
 public class VehicleStatAutoConfiguration {
@@ -4,13 +4,16 @@ import org.springframework.jdbc.core.JdbcTemplate;

 import java.sql.Timestamp;
 import java.time.Instant;
+import java.util.Locale;
+import java.util.Objects;

 public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageRepository {

    private final JdbcTemplate jdbc;

    public JdbcJt808DailyMileageRepository(JdbcTemplate jdbc) {
-        this.jdbc = jdbc;
+        this.jdbc = Objects.requireNonNull(jdbc, "jdbc must not be null");
+        initializeSchema();
    }

    @Override
@@ -18,18 +21,23 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
        jdbc.update("""
                        insert into vehicle_daily_mileage_jt808 (
                          stat_date, vehicle_key, vin, phone, first_event_time, last_event_time,
-                          gps_mileage_km, speed_integral_km, odometer_mileage_km,
+                          daily_mileage_km, mileage_source, gps_mileage_km, speed_integral_km,
+                          odometer_mileage_km, first_total_mileage_km, last_total_mileage_km,
                          accepted_points, bad_jump_segments, long_gap_segments, out_of_order_points,
                          odometer_anomalies, data_quality
-                        ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                        on duplicate key update
                          vin = values(vin),
                          phone = values(phone),
                          first_event_time = values(first_event_time),
                          last_event_time = values(last_event_time),
+                          daily_mileage_km = values(daily_mileage_km),
+                          mileage_source = values(mileage_source),
                          gps_mileage_km = values(gps_mileage_km),
                          speed_integral_km = values(speed_integral_km),
                          odometer_mileage_km = values(odometer_mileage_km),
+                          first_total_mileage_km = values(first_total_mileage_km),
+                          last_total_mileage_km = values(last_total_mileage_km),
                          accepted_points = values(accepted_points),
                          bad_jump_segments = values(bad_jump_segments),
                          long_gap_segments = values(long_gap_segments),
@@ -44,9 +52,13 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
                result.phone(),
                timestamp(result.firstEventTime()),
                timestamp(result.lastEventTime()),
+                result.dailyMileageKm(),
+                result.mileageSource(),
                result.gpsMileageKm(),
                result.speedIntegralKm(),
                result.odometerMileageKm(),
+                result.firstTotalMileageKm(),
+                result.lastTotalMileageKm(),
                result.acceptedPoints(),
                result.badJumpSegments(),
                result.longGapSegments(),
@@ -55,6 +67,59 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
                result.dataQuality());
    }

+    private void initializeSchema() {
+        jdbc.execute("""
+                create table if not exists vehicle_daily_mileage_jt808 (
+                  stat_date date not null,
+                  vehicle_key varchar(64) not null,
+                  vin varchar(32) null,
+                  phone varchar(32) null,
+                  first_event_time timestamp null,
+                  last_event_time timestamp null,
+                  daily_mileage_km decimal(12,3) not null default 0,
+                  mileage_source varchar(32) not null default 'INSUFFICIENT',
+                  gps_mileage_km decimal(12,3) not null,
+                  speed_integral_km decimal(12,3) not null,
+                  odometer_mileage_km decimal(12,3) null,
+                  first_total_mileage_km decimal(12,3) null,
+                  last_total_mileage_km decimal(12,3) null,
+                  accepted_points int not null,
+                  bad_jump_segments int not null,
+                  long_gap_segments int not null,
+                  out_of_order_points int not null,
+                  odometer_anomalies int not null,
+                  data_quality varchar(16) not null,
+                  updated_at timestamp not null default current_timestamp on update current_timestamp,
+                  primary key (stat_date, vehicle_key),
+                  key idx_vehicle_daily_mileage_jt808_vehicle_date (vehicle_key, stat_date),
+                  key idx_vehicle_daily_mileage_jt808_vin_date (vin, stat_date),
+                  key idx_vehicle_daily_mileage_jt808_phone_date (phone, stat_date)
+                )
+                """);
+        ensureColumn("daily_mileage_km", "daily_mileage_km decimal(12,3) not null default 0");
+        ensureColumn("mileage_source", "mileage_source varchar(32) not null default 'INSUFFICIENT'");
+        ensureColumn("first_total_mileage_km", "first_total_mileage_km decimal(12,3) null");
+        ensureColumn("last_total_mileage_km", "last_total_mileage_km decimal(12,3) null");
+    }
+
+    private void ensureColumn(String columnName, String definition) {
+        if (!columnExists(columnName)) {
+            jdbc.execute("alter table vehicle_daily_mileage_jt808 add column " + definition);
+        }
+    }
+
+    private boolean columnExists(String columnName) {
+        Integer count = jdbc.queryForObject("""
+                select count(*)
+                from information_schema.columns
+                where lower(table_name) = ?
+                  and lower(column_name) = ?
+                """, Integer.class,
+                "vehicle_daily_mileage_jt808",
+                columnName.toLowerCase(Locale.ROOT));
+        return count != null && count > 0;
+    }
+
    private static Timestamp timestamp(Instant instant) {
        return instant == null ? null : Timestamp.from(instant);
    }
@@ -10,9 +10,13 @@ public record Jt808DailyMileageResult(
        String phone,
        Instant firstEventTime,
        Instant lastEventTime,
+        double dailyMileageKm,
+        String mileageSource,
        double gpsMileageKm,
        double speedIntegralKm,
        Double odometerMileageKm,
+        Double firstTotalMileageKm,
+        Double lastTotalMileageKm,
        int acceptedPoints,
        int badJumpSegments,
        int longGapSegments,
@@ -12,6 +12,7 @@ public final class Jt808DailyMileageState {
    private Instant firstEventTime;
    private Instant lastEventTime;
    private Jt808LocationPoint lastPoint;
+    private double firstTotalMileageKm = Double.NaN;
    private double lastTotalMileageKm = Double.NaN;
    private double gpsMileageKm;
    private double speedIntegralKm;
@@ -47,6 +48,7 @@ public final class Jt808DailyMileageState {
            Instant firstEventTime,
            Instant lastEventTime,
            Jt808LocationPoint lastPoint,
+            double firstTotalMileageKm,
            double lastTotalMileageKm,
            double gpsMileageKm,
            double speedIntegralKm,
@@ -60,6 +62,7 @@ public final class Jt808DailyMileageState {
        state.firstEventTime = firstEventTime;
        state.lastEventTime = lastEventTime;
        state.lastPoint = lastPoint;
+        state.firstTotalMileageKm = firstTotalMileageKm;
        state.lastTotalMileageKm = lastTotalMileageKm;
        state.gpsMileageKm = gpsMileageKm;
        state.speedIntegralKm = speedIntegralKm;
@@ -82,7 +85,7 @@ public final class Jt808DailyMileageState {
        }
        if (lastPoint == null) {
            firstEventTime = point.eventTime();
-            updateOdometer(point);
+            updateOdometer(point, calculator);
            accept(point);
            return;
        }
@@ -100,7 +103,7 @@ public final class Jt808DailyMileageState {
            gpsMileageKm += segment.gpsKm();
            speedIntegralKm += segment.speedIntegralKm();
        }
-        updateOdometer(point);
+        updateOdometer(point, calculator);
        accept(point);
    }

@@ -112,9 +115,13 @@ public final class Jt808DailyMileageState {
                phone,
                firstEventTime,
                lastEventTime,
+                dailyMileageKm(),
+                mileageSource(),
                gpsMileageKm,
                speedIntegralKm,
                odometerMileageKm > 0.0 ? odometerMileageKm : null,
+                nullable(firstTotalMileageKm),
+                nullable(lastTotalMileageKm),
                acceptedPoints,
                badJumpSegments,
                longGapSegments,
@@ -129,11 +136,14 @@ public final class Jt808DailyMileageState {
        acceptedPoints++;
    }

-    private void updateOdometer(Jt808LocationPoint point) {
+    private void updateOdometer(Jt808LocationPoint point, Jt808GpsMileageCalculator calculator) {
        if (point.totalMileageKm() == null || !Double.isFinite(point.totalMileageKm())) {
            return;
        }
        double current = point.totalMileageKm();
+        if (!Double.isFinite(firstTotalMileageKm)) {
+            firstTotalMileageKm = current;
+        }
        if (!Double.isFinite(lastTotalMileageKm)) {
            lastTotalMileageKm = current;
            return;
@@ -143,15 +153,56 @@ public final class Jt808DailyMileageState {
            lastTotalMileageKm = current;
            return;
        }
-        odometerMileageKm += current - lastTotalMileageKm;
+        double delta = current - lastTotalMileageKm;
+        if (lastPoint != null && !calculator.plausibleOdometerDelta(lastPoint, point, delta)) {
+            odometerAnomalies++;
+            lastTotalMileageKm = current;
+            return;
+        }
+        odometerMileageKm += delta;
        lastTotalMileageKm = current;
    }

+    private double dailyMileageKm() {
+        if (hasUsableTotalMileage()) {
+            return odometerMileageKm;
+        }
+        if (gpsMileageKm > 0.0) {
+            return gpsMileageKm;
+        }
+        return 0.0;
+    }
+
+    private String mileageSource() {
+        if (hasUsableTotalMileage()) {
+            return odometerAnomalies > 0 ? "JT808_TOTAL_MILEAGE_PARTIAL" : "JT808_TOTAL_MILEAGE";
+        }
+        if (gpsMileageKm > 0.0) {
+            return "GPS_DISTANCE";
+        }
+        return "INSUFFICIENT";
+    }
+
+    private boolean hasUsableTotalMileage() {
+        if (odometerMileageKm > 0.0) {
+            return true;
+        }
+        return acceptedPoints >= 2
+                && odometerAnomalies == 0
+                && Double.isFinite(firstTotalMileageKm)
+                && Double.isFinite(lastTotalMileageKm)
+                && lastTotalMileageKm >= firstTotalMileageKm;
+    }
+
+    private static Double nullable(double value) {
+        return Double.isFinite(value) ? value : null;
+    }
+
    private String dataQuality() {
        if (acceptedPoints < 2) {
            return "PARTIAL";
        }
-        if (badJumpSegments > 0 || longGapSegments > 0 || vehicleKey.startsWith("jt808:")) {
+        if (badJumpSegments > 0 || longGapSegments > 0 || odometerAnomalies > 0 || vehicleKey.startsWith("jt808:")) {
            return "PARTIAL";
        }
        return "GOOD";
@@ -201,6 +252,10 @@ public final class Jt808DailyMileageState {
        return lastTotalMileageKm;
    }

+    public double firstTotalMileageKm() {
+        return firstTotalMileageKm;
+    }
+
    public int acceptedPoints() {
        return acceptedPoints;
    }
@@ -5,6 +5,7 @@ import java.time.Duration;
 public final class Jt808GpsMileageCalculator {

    private static final double EARTH_RADIUS_KM = 6371.0088;
+    private static final double ODOMETER_DELTA_TOLERANCE_KM = 0.1;

    private final Duration maxSegmentGap;
    private final double maxImpliedSpeedKmh;
@@ -61,6 +62,20 @@ public final class Jt808GpsMileageCalculator {
        return new SegmentResult(gpsKm, speedIntegralKm, true, false, false, false);
    }

+    public boolean plausibleOdometerDelta(Jt808LocationPoint previous,
+                                          Jt808LocationPoint current,
+                                          double deltaKm) {
+        if (previous == null || current == null || !Double.isFinite(deltaKm) || deltaKm < 0.0) {
+            return false;
+        }
+        Duration elapsed = Duration.between(previous.eventTime(), current.eventTime());
+        if (elapsed.isZero() || elapsed.isNegative()) {
+            return false;
+        }
+        double hours = elapsed.toMillis() / 3_600_000.0;
+        return deltaKm <= maxImpliedSpeedKmh * hours + ODOMETER_DELTA_TOLERANCE_KM;
+    }
+
    private static double haversineKm(double lon1, double lat1, double lon2, double lat2) {
        double latRad1 = Math.toRadians(lat1);
        double latRad2 = Math.toRadians(lat2);
@@ -13,7 +13,8 @@ public final class Jt808LocationPointExtractor {
    public Optional<Jt808LocationPoint> extract(VehicleEnvelope envelope) {
        if (envelope == null
                || !"JT808".equalsIgnoreCase(envelope.getSource())
-                || !envelope.hasTelemetrySnapshot()) {
+                || !envelope.hasTelemetrySnapshot()
+                || !"LOCATION".equalsIgnoreCase(envelope.getTelemetrySnapshot().getEventType())) {
            return Optional.empty();
        }
        Map<String, String> fields = fields(envelope);
@@ -64,7 +64,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
            String firstEventTime,
            String lastEventTime,
            PointSnapshot lastPoint,
-            double lastTotalMileageKm,
+            Double firstTotalMileageKm,
+            Double lastTotalMileageKm,
            double gpsMileageKm,
            double speedIntegralKm,
            double odometerMileageKm,
@@ -83,7 +84,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
                    format(state.firstEventTime()),
                    format(state.lastEventTime()),
                    PointSnapshot.from(state.lastPoint()),
-                    state.lastTotalMileageKm(),
+                    finiteOrNull(state.firstTotalMileageKm()),
+                    finiteOrNull(state.lastTotalMileageKm()),
                    state.gpsMileageKm(),
                    state.speedIntegralKm(),
                    state.odometerMileageKm(),
@@ -103,7 +105,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
                    parseInstant(firstEventTime),
                    parseInstant(lastEventTime),
                    lastPoint == null ? null : lastPoint.toPoint(),
-                    lastTotalMileageKm,
+                    nanIfNull(firstTotalMileageKm),
+                    nanIfNull(lastTotalMileageKm),
                    gpsMileageKm,
                    speedIntegralKm,
                    odometerMileageKm,
@@ -164,4 +167,12 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
    private static Instant parseInstant(String value) {
        return value == null || value.isBlank() ? null : Instant.parse(value);
    }
+
+    private static Double finiteOrNull(double value) {
+        return Double.isFinite(value) ? value : null;
+    }
+
+    private static double nanIfNull(Double value) {
+        return value == null ? Double.NaN : value;
+    }
 }