From 3e19c729159ae9ab2a5f834ff86205a186ceebab Mon Sep 17 00:00:00 2001 From: lingniu Date: Wed, 1 Jul 2026 06:25:33 +0800 Subject: [PATCH] refactor: store jt808 daily mileage as single metric --- .../operations/jt808-daily-mileage-runbook.md | 6 +- docs/target-architecture.md | 8 +- .../JdbcVehicleStatMetricRepository.java | 131 +++++++++++++----- .../JdbcVehicleStatMetricRepositoryTest.java | 37 +++-- 4 files changed, 127 insertions(+), 55 deletions(-) diff --git a/docs/operations/jt808-daily-mileage-runbook.md b/docs/operations/jt808-daily-mileage-runbook.md index cb38b5dd..5420e1ef 100644 --- a/docs/operations/jt808-daily-mileage-runbook.md +++ b/docs/operations/jt808-daily-mileage-runbook.md @@ -6,7 +6,7 @@ The analytics app can calculate daily mileage from JT808 telemetry only. It cons ## Storage -There is no separate JT808 daily-mileage table and no Redis mileage state. The current day start total, latest total, and derived daily mileage are all stored in the common JDBC/MySQL metric table `vehicle_stat_metric`. +There is no separate JT808 daily-mileage table and no Redis mileage state. The derived value is stored as one `daily_mileage_km` metric row in the common JDBC/MySQL table `vehicle_stat_metric`; the first and latest GPS total mileage values are kept on that same row as calculation source columns. The JT808 daily-mileage value is calculated from the GPS total mileage reported in location additional information: @@ -15,7 +15,7 @@ daily_mileage_km = last_total_mileage_km - first_total_mileage_km calculation_method = JT808_TOTAL_MILEAGE_DIFF ``` -The first valid JT808 location point for a vehicle and local day stores the day start total and writes `daily_mileage_km=0.0`. Later ordered points update the latest total and the derived `daily_mileage_km`. If a later total mileage is lower than the stored start or latest total, the update is ignored and the previous metric remains. +The first valid JT808 location point for a vehicle and local day stores the day start total on the metric row and writes `daily_mileage_km=0.0`. Later ordered points update the latest total on the same row and rewrite the derived `daily_mileage_km`. If a later total mileage is lower than the stored latest total, the previous latest total remains. ## Runtime Settings @@ -29,4 +29,4 @@ MYSQL_USERNAME= MYSQL_PASSWORD= ``` -Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `vehicle_stat_metric` rows, so no separate mileage state store is required. +Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `daily_mileage_km` metric row, so no separate mileage state store is required. diff --git a/docs/target-architecture.md b/docs/target-architecture.md index 1e0f97c3..44d46692 100644 --- a/docs/target-architecture.md +++ b/docs/target-architecture.md @@ -125,12 +125,12 @@ calculation_method = JT808_TOTAL_MILEAGE_DIFF ``` - Store the metric in `vehicle_stat_metric` with - `metric_key = daily_mileage_km`. + `metric_key = daily_mileage_km`; the first and latest GPS total mileage used + for the subtraction stay on the same metric row as calculation source columns. There is no JT808-specific daily mileage table and no in-memory production -state-store mode. Daily start total, latest total, and derived mileage are all -stored in `vehicle_stat_metric`, so restart recovery does not require a separate -Redis mileage state. +state-store mode. Restart recovery reads the same metric row, so it does not +require a separate Redis mileage state. ### Latest State diff --git a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepository.java b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepository.java index 86f6b194..c28852c0 100644 --- a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepository.java +++ b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepository.java @@ -1,9 +1,12 @@ package com.lingniu.ingest.vehiclestat; import org.springframework.dao.DuplicateKeyException; +import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcTemplate; import java.sql.Date; +import java.sql.ResultSet; +import java.sql.SQLException; import java.time.LocalDate; import java.util.List; import java.util.Optional; @@ -15,6 +18,8 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit private static final String DAILY_MILEAGE_UNIT = "km"; private static final String DAILY_MILEAGE_START_TOTAL_KEY = "daily_mileage_start_total_km"; private static final String DAILY_MILEAGE_LATEST_TOTAL_KEY = "daily_mileage_latest_total_km"; + private static final String FIRST_TOTAL_COLUMN = "first_total_mileage_km"; + private static final String LATEST_TOTAL_COLUMN = "latest_total_mileage_km"; private final JdbcTemplate jdbcTemplate; @@ -26,17 +31,6 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit ensureSchema(); } - private void saveDailyMileageMetric(VehicleDailyStatResult result) { - if (result == null || result.dailyMileageKm().isEmpty()) { - return; - } - String vin = clean(result.vin()); - Date statDate = Date.valueOf(result.statDate()); - double value = result.dailyMileageKm().getAsDouble(); - String strategy = result.dailyMileageStrategy().name(); - upsertMetric(vin, statDate, DAILY_MILEAGE_KEY, value, DAILY_MILEAGE_UNIT, strategy); - } - @Override public Optional findDailyStat(String vin, LocalDate statDate) { String normalizedVin = clean(vin); @@ -63,34 +57,48 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit String normalizedVin = clean(vin); Date date = Date.valueOf(statDate); String strategy = DailyMileageStrategy.JT808_TOTAL_MILEAGE_DIFF.name(); - OptionalDouble existingMin = findMetric(normalizedVin, date, DAILY_MILEAGE_START_TOTAL_KEY); - double minTotalMileage = existingMin.isPresent() - ? Math.min(existingMin.getAsDouble(), totalMileageKm) - : totalMileageKm; - if (existingMin.isEmpty() || Double.compare(minTotalMileage, existingMin.getAsDouble()) != 0) { - upsertMetric(normalizedVin, date, DAILY_MILEAGE_START_TOTAL_KEY, minTotalMileage, - DAILY_MILEAGE_UNIT, strategy); - } - - OptionalDouble existingMax = findMetric(normalizedVin, date, DAILY_MILEAGE_LATEST_TOTAL_KEY); - double maxTotalMileage = existingMax.isPresent() - ? Math.max(existingMax.getAsDouble(), totalMileageKm) - : totalMileageKm; - if (existingMax.isEmpty() || Double.compare(maxTotalMileage, existingMax.getAsDouble()) != 0) { - upsertMetric(normalizedVin, date, DAILY_MILEAGE_LATEST_TOTAL_KEY, maxTotalMileage, - DAILY_MILEAGE_UNIT, strategy); - } - - double dailyMileageKm = maxTotalMileage - minTotalMileage; + DailyMileageState existingState = findDailyMileageState(normalizedVin, date) + .orElseGet(() -> legacyDailyMileageState(normalizedVin, date).orElse(null)); + double firstTotalMileage = existingState == null || !Double.isFinite(existingState.firstTotalMileageKm()) + ? totalMileageKm + : existingState.firstTotalMileageKm(); + double latestTotalMileage = existingState == null || !Double.isFinite(existingState.latestTotalMileageKm()) + ? totalMileageKm + : Math.max(existingState.latestTotalMileageKm(), totalMileageKm); + double dailyMileageKm = latestTotalMileage - firstTotalMileage; + upsertDailyMileageMetric(normalizedVin, date, dailyMileageKm, DAILY_MILEAGE_UNIT, strategy, + firstTotalMileage, latestTotalMileage); + deleteLegacyMileageStateMetrics(normalizedVin, date); VehicleDailyStatResult result = new VehicleDailyStatResult( normalizedVin, statDate, OptionalDouble.of(dailyMileageKm), DailyMileageStrategy.JT808_TOTAL_MILEAGE_DIFF); - saveDailyMileageMetric(result); return Optional.of(result); } + private Optional findDailyMileageState(String vin, Date statDate) { + List rows = jdbcTemplate.query(""" + SELECT first_total_mileage_km, latest_total_mileage_km + FROM vehicle_stat_metric + WHERE vin = ? AND stat_date = ? AND metric_key = ? + """, (rs, rowNum) -> new DailyMileageState( + nullableDouble(rs, FIRST_TOTAL_COLUMN), + nullableDouble(rs, LATEST_TOTAL_COLUMN)), vin, statDate, DAILY_MILEAGE_KEY); + return rows.stream() + .filter(DailyMileageState::hasCompleteState) + .findFirst(); + } + + private Optional legacyDailyMileageState(String vin, Date statDate) { + OptionalDouble start = findMetric(vin, statDate, DAILY_MILEAGE_START_TOTAL_KEY); + OptionalDouble latest = findMetric(vin, statDate, DAILY_MILEAGE_LATEST_TOTAL_KEY); + if (start.isEmpty() || latest.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new DailyMileageState(start.getAsDouble(), latest.getAsDouble())); + } + private OptionalDouble findMetric(String vin, Date statDate, String metricKey) { List rows = jdbcTemplate.query(""" SELECT metric_value @@ -105,25 +113,38 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit return OptionalDouble.of(rows.getFirst()); } - private void upsertMetric(String vin, Date statDate, String metricKey, double value, String unit, String strategy) { + private void upsertDailyMileageMetric(String vin, Date statDate, double value, String unit, String strategy, + double firstTotalMileageKm, double latestTotalMileageKm) { try { jdbcTemplate.update(""" INSERT INTO vehicle_stat_metric - (vin, stat_date, metric_key, metric_value, metric_unit, calculation_method) - VALUES (?, ?, ?, ?, ?, ?) - """, vin, statDate, metricKey, value, unit, strategy); + (vin, stat_date, metric_key, metric_value, metric_unit, calculation_method, + first_total_mileage_km, latest_total_mileage_km) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, vin, statDate, DAILY_MILEAGE_KEY, value, unit, strategy, + firstTotalMileageKm, latestTotalMileageKm); } catch (DuplicateKeyException ex) { jdbcTemplate.update(""" UPDATE vehicle_stat_metric SET metric_value = ?, metric_unit = ?, calculation_method = ?, + first_total_mileage_km = ?, + latest_total_mileage_km = ?, updated_at = CURRENT_TIMESTAMP WHERE vin = ? AND stat_date = ? AND metric_key = ? - """, value, unit, strategy, vin, statDate, metricKey); + """, value, unit, strategy, firstTotalMileageKm, latestTotalMileageKm, + vin, statDate, DAILY_MILEAGE_KEY); } } + private void deleteLegacyMileageStateMetrics(String vin, Date statDate) { + jdbcTemplate.update(""" + DELETE FROM vehicle_stat_metric + WHERE vin = ? AND stat_date = ? AND metric_key IN (?, ?) + """, vin, statDate, DAILY_MILEAGE_START_TOTAL_KEY, DAILY_MILEAGE_LATEST_TOTAL_KEY); + } + private void ensureSchema() { jdbcTemplate.execute(""" CREATE TABLE IF NOT EXISTS vehicle_stat_metric ( @@ -133,11 +154,43 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit metric_value DECIMAL(18,6) NULL, metric_unit VARCHAR(16) NOT NULL, calculation_method VARCHAR(64) NOT NULL, + first_total_mileage_km DECIMAL(18,6) NULL, + latest_total_mileage_km DECIMAL(18,6) NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (vin, stat_date, metric_key) ) """); + ensureColumn(FIRST_TOTAL_COLUMN); + ensureColumn(LATEST_TOTAL_COLUMN); + } + + private void ensureColumn(String columnName) { + if (columnExists(columnName)) { + return; + } + jdbcTemplate.execute("ALTER TABLE vehicle_stat_metric ADD COLUMN " + columnName + " DECIMAL(18,6) NULL"); + } + + private boolean columnExists(String columnName) { + ConnectionCallback callback = connection -> { + try (ResultSet columns = connection.getMetaData().getColumns( + connection.getCatalog(), null, "vehicle_stat_metric", columnName)) { + if (columns.next()) { + return true; + } + } + try (ResultSet columns = connection.getMetaData().getColumns( + connection.getCatalog(), null, "VEHICLE_STAT_METRIC", columnName.toUpperCase())) { + return columns.next(); + } + }; + return Boolean.TRUE.equals(jdbcTemplate.execute(callback)); + } + + private static double nullableDouble(ResultSet rs, String columnName) throws SQLException { + double value = rs.getDouble(columnName); + return rs.wasNull() ? Double.NaN : value; } private static String clean(String value) { @@ -146,4 +199,10 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit } return value.trim(); } + + private record DailyMileageState(double firstTotalMileageKm, double latestTotalMileageKm) { + boolean hasCompleteState() { + return Double.isFinite(firstTotalMileageKm) && Double.isFinite(latestTotalMileageKm); + } + } } diff --git a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepositoryTest.java b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepositoryTest.java index fb10ca0d..d233bc03 100644 --- a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepositoryTest.java +++ b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepositoryTest.java @@ -27,13 +27,23 @@ class JdbcVehicleStatMetricRepositoryTest { .extracting(result -> result.dailyMileageKm().orElseThrow()) .isEqualTo(15.5); assertThat(jdbcTemplate.queryForObject("SELECT COUNT(*) FROM vehicle_stat_metric", Integer.class)) - .isEqualTo(3); + .isOne(); assertThat(jdbcTemplate.queryForObject( "SELECT metric_key FROM vehicle_stat_metric WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km'", String.class)).isEqualTo("daily_mileage_km"); assertThat(jdbcTemplate.queryForObject( "SELECT calculation_method FROM vehicle_stat_metric WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km'", String.class)).isEqualTo("JT808_TOTAL_MILEAGE_DIFF"); + assertThat(jdbcTemplate.queryForObject(""" + SELECT first_total_mileage_km + FROM vehicle_stat_metric + WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km' + """, Double.class)).isEqualTo(100.0); + assertThat(jdbcTemplate.queryForObject(""" + SELECT latest_total_mileage_km + FROM vehicle_stat_metric + WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km' + """, Double.class)).isEqualTo(115.5); assertThat(jdbcTemplate.queryForObject(""" SELECT COUNT(*) FROM information_schema.tables @@ -58,7 +68,7 @@ class JdbcVehicleStatMetricRepositoryTest { assertThat(second.dailyMileageKm()).hasValue(8.25); assertThat(jdbcTemplate.queryForObject( "SELECT COUNT(*) FROM vehicle_stat_metric WHERE vin = 'VIN001' AND stat_date = DATE '2026-07-01'", - Integer.class)).isEqualTo(3); + Integer.class)).isOne(); assertThat(jdbcTemplate.queryForObject(""" SELECT metric_value FROM vehicle_stat_metric @@ -70,30 +80,33 @@ class JdbcVehicleStatMetricRepositoryTest { } @Test - void recalculatesJt808MileageFromDailyMinAndMaxTotalsWhenSamplesArriveOutOfOrder() { + void recalculatesJt808MileageFromSingleMetricRowAfterRestart() { JdbcTemplate jdbcTemplate = new JdbcTemplate(new DriverManagerDataSource( "jdbc:h2:mem:vehicle_stat_metric_jt808_out_of_order;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1", "sa", "")); JdbcVehicleStatMetricRepository repository = new JdbcVehicleStatMetricRepository(jdbcTemplate); - repository.recordDailyMileageSample("VIN001", LocalDate.of(2026, 7, 1), 1008.75).orElseThrow(); - VehicleDailyStatResult lowerSample = repository.recordDailyMileageSample( - "VIN001", LocalDate.of(2026, 7, 1), 1000.5).orElseThrow(); - VehicleDailyStatResult higherSample = repository.recordDailyMileageSample( + repository.recordDailyMileageSample("VIN001", LocalDate.of(2026, 7, 1), 1000.5).orElseThrow(); + JdbcVehicleStatMetricRepository restartedRepository = new JdbcVehicleStatMetricRepository(jdbcTemplate); + VehicleDailyStatResult higherSample = restartedRepository.recordDailyMileageSample( "VIN001", LocalDate.of(2026, 7, 1), 1010.0).orElseThrow(); - assertThat(lowerSample.dailyMileageKm()).hasValue(8.25); assertThat(higherSample.dailyMileageKm()).hasValue(9.5); assertThat(jdbcTemplate.queryForObject(""" - SELECT metric_value + SELECT COUNT(*) FROM vehicle_stat_metric - WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_start_total_km' + WHERE vin = 'VIN001' AND stat_date = DATE '2026-07-01' + """, Integer.class)).isOne(); + assertThat(jdbcTemplate.queryForObject(""" + SELECT first_total_mileage_km + FROM vehicle_stat_metric + WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km' """, Double.class)).isEqualTo(1000.5); assertThat(jdbcTemplate.queryForObject(""" - SELECT metric_value + SELECT latest_total_mileage_km FROM vehicle_stat_metric - WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_latest_total_km' + WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km' """, Double.class)).isEqualTo(1010.0); assertThat(jdbcTemplate.queryForObject(""" SELECT metric_value