shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: store jt808 daily mileage as single metric

Browse Source
This commit is contained in:
lingniu
2026-07-01 06:25:33 +08:00
parent 3d4676adf3
commit 3e19c72915
4 changed files with 127 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docs/operations/jt808-daily-mileage-runbook.md
View File

@@ -6,7 +6,7 @@ The analytics app can calculate daily mileage from JT808 telemetry only. It cons
## Storage
There is no separate JT808 daily-mileage table and no Redis mileage state. The current day start total, latest total, and derived daily mileage are all stored in the common JDBC/MySQL metric table `vehicle_stat_metric`.
There is no separate JT808 daily-mileage table and no Redis mileage state. The derived value is stored as one `daily_mileage_km` metric row in the common JDBC/MySQL table `vehicle_stat_metric`; the first and latest GPS total mileage values are kept on that same row as calculation source columns.
The JT808 daily-mileage value is calculated from the GPS total mileage reported in location additional information:
@@ -15,7 +15,7 @@ daily_mileage_km = last_total_mileage_km - first_total_mileage_km
calculation_method = JT808_TOTAL_MILEAGE_DIFF
```
The first valid JT808 location point for a vehicle and local day stores the day start total and writes `daily_mileage_km=0.0`. Later ordered points update the latest total and the derived `daily_mileage_km`. If a later total mileage is lower than the stored start or latest total, the update is ignored and the previous metric remains.
The first valid JT808 location point for a vehicle and local day stores the day start total on the metric row and writes `daily_mileage_km=0.0`. Later ordered points update the latest total on the same row and rewrite the derived `daily_mileage_km`. If a later total mileage is lower than the stored latest total, the previous latest total remains.
## Runtime Settings
@@ -29,4 +29,4 @@ MYSQL_USERNAME=<user>
MYSQL_PASSWORD=<password>
```
Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `vehicle_stat_metric` rows, so no separate mileage state store is required.
Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `daily_mileage_km` metric row, so no separate mileage state store is required.

8
docs/target-architecture.md
View File

@@ -125,12 +125,12 @@ calculation_method = JT808_TOTAL_MILEAGE_DIFF
```
- Store the metric in `vehicle_stat_metric` with
`metric_key = daily_mileage_km`.
`metric_key = daily_mileage_km`; the first and latest GPS total mileage used
for the subtraction stay on the same metric row as calculation source columns.
There is no JT808-specific daily mileage table and no in-memory production
state-store mode. Daily start total, latest total, and derived mileage are all
stored in `vehicle_stat_metric`, so restart recovery does not require a separate
Redis mileage state.
state-store mode. Restart recovery reads the same metric row, so it does not
require a separate Redis mileage state.
### Latest State

131
modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepository.java
View File

@@ -1,9 +1,12 @@
package com.lingniu.ingest.vehiclestat;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.jdbc.core.ConnectionCallback;
import org.springframework.jdbc.core.JdbcTemplate;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDate;
import java.util.List;
import java.util.Optional;
@@ -15,6 +18,8 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit
private static final String DAILY_MILEAGE_UNIT = "km";
private static final String DAILY_MILEAGE_START_TOTAL_KEY = "daily_mileage_start_total_km";
private static final String DAILY_MILEAGE_LATEST_TOTAL_KEY = "daily_mileage_latest_total_km";
private static final String FIRST_TOTAL_COLUMN = "first_total_mileage_km";
private static final String LATEST_TOTAL_COLUMN = "latest_total_mileage_km";
private final JdbcTemplate jdbcTemplate;
@@ -26,17 +31,6 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit
ensureSchema();
}
private void saveDailyMileageMetric(VehicleDailyStatResult result) {
if (result == null || result.dailyMileageKm().isEmpty()) {
return;
}
String vin = clean(result.vin());
Date statDate = Date.valueOf(result.statDate());
double value = result.dailyMileageKm().getAsDouble();
String strategy = result.dailyMileageStrategy().name();
upsertMetric(vin, statDate, DAILY_MILEAGE_KEY, value, DAILY_MILEAGE_UNIT, strategy);
}
@Override
public Optional<VehicleDailyStatResult> findDailyStat(String vin, LocalDate statDate) {
String normalizedVin = clean(vin);
@@ -63,34 +57,48 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit
String normalizedVin = clean(vin);
Date date = Date.valueOf(statDate);
String strategy = DailyMileageStrategy.JT808_TOTAL_MILEAGE_DIFF.name();
OptionalDouble existingMin = findMetric(normalizedVin, date, DAILY_MILEAGE_START_TOTAL_KEY);
double minTotalMileage = existingMin.isPresent()
? Math.min(existingMin.getAsDouble(), totalMileageKm)
: totalMileageKm;
if (existingMin.isEmpty() || Double.compare(minTotalMileage, existingMin.getAsDouble()) != 0) {
upsertMetric(normalizedVin, date, DAILY_MILEAGE_START_TOTAL_KEY, minTotalMileage,
DAILY_MILEAGE_UNIT, strategy);
}
OptionalDouble existingMax = findMetric(normalizedVin, date, DAILY_MILEAGE_LATEST_TOTAL_KEY);
double maxTotalMileage = existingMax.isPresent()
? Math.max(existingMax.getAsDouble(), totalMileageKm)
: totalMileageKm;
if (existingMax.isEmpty() || Double.compare(maxTotalMileage, existingMax.getAsDouble()) != 0) {
upsertMetric(normalizedVin, date, DAILY_MILEAGE_LATEST_TOTAL_KEY, maxTotalMileage,
DAILY_MILEAGE_UNIT, strategy);
}
double dailyMileageKm = maxTotalMileage - minTotalMileage;
DailyMileageState existingState = findDailyMileageState(normalizedVin, date)
.orElseGet(() -> legacyDailyMileageState(normalizedVin, date).orElse(null));
double firstTotalMileage = existingState == null || !Double.isFinite(existingState.firstTotalMileageKm())
? totalMileageKm
: existingState.firstTotalMileageKm();
double latestTotalMileage = existingState == null || !Double.isFinite(existingState.latestTotalMileageKm())
? totalMileageKm
: Math.max(existingState.latestTotalMileageKm(), totalMileageKm);
double dailyMileageKm = latestTotalMileage - firstTotalMileage;
upsertDailyMileageMetric(normalizedVin, date, dailyMileageKm, DAILY_MILEAGE_UNIT, strategy,
firstTotalMileage, latestTotalMileage);
deleteLegacyMileageStateMetrics(normalizedVin, date);
VehicleDailyStatResult result = new VehicleDailyStatResult(
normalizedVin,
statDate,
OptionalDouble.of(dailyMileageKm),
DailyMileageStrategy.JT808_TOTAL_MILEAGE_DIFF);
saveDailyMileageMetric(result);
return Optional.of(result);
}
private Optional<DailyMileageState> findDailyMileageState(String vin, Date statDate) {
List<DailyMileageState> rows = jdbcTemplate.query("""
SELECT first_total_mileage_km, latest_total_mileage_km
FROM vehicle_stat_metric
WHERE vin = ? AND stat_date = ? AND metric_key = ?
""", (rs, rowNum) -> new DailyMileageState(
nullableDouble(rs, FIRST_TOTAL_COLUMN),
nullableDouble(rs, LATEST_TOTAL_COLUMN)), vin, statDate, DAILY_MILEAGE_KEY);
return rows.stream()
.filter(DailyMileageState::hasCompleteState)
.findFirst();
}
private Optional<DailyMileageState> legacyDailyMileageState(String vin, Date statDate) {
OptionalDouble start = findMetric(vin, statDate, DAILY_MILEAGE_START_TOTAL_KEY);
OptionalDouble latest = findMetric(vin, statDate, DAILY_MILEAGE_LATEST_TOTAL_KEY);
if (start.isEmpty() || latest.isEmpty()) {
return Optional.empty();
}
return Optional.of(new DailyMileageState(start.getAsDouble(), latest.getAsDouble()));
}
private OptionalDouble findMetric(String vin, Date statDate, String metricKey) {
List<Double> rows = jdbcTemplate.query("""
SELECT metric_value
@@ -105,25 +113,38 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit
return OptionalDouble.of(rows.getFirst());
}
private void upsertMetric(String vin, Date statDate, String metricKey, double value, String unit, String strategy) {
private void upsertDailyMileageMetric(String vin, Date statDate, double value, String unit, String strategy,
double firstTotalMileageKm, double latestTotalMileageKm) {
try {
jdbcTemplate.update("""
INSERT INTO vehicle_stat_metric
(vin, stat_date, metric_key, metric_value, metric_unit, calculation_method)
VALUES (?, ?, ?, ?, ?, ?)
""", vin, statDate, metricKey, value, unit, strategy);
(vin, stat_date, metric_key, metric_value, metric_unit, calculation_method,
first_total_mileage_km, latest_total_mileage_km)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", vin, statDate, DAILY_MILEAGE_KEY, value, unit, strategy,
firstTotalMileageKm, latestTotalMileageKm);
} catch (DuplicateKeyException ex) {
jdbcTemplate.update("""
UPDATE vehicle_stat_metric
SET metric_value = ?,
metric_unit = ?,
calculation_method = ?,
first_total_mileage_km = ?,
latest_total_mileage_km = ?,
updated_at = CURRENT_TIMESTAMP
WHERE vin = ? AND stat_date = ? AND metric_key = ?
""", value, unit, strategy, vin, statDate, metricKey);
""", value, unit, strategy, firstTotalMileageKm, latestTotalMileageKm,
vin, statDate, DAILY_MILEAGE_KEY);
}
}
private void deleteLegacyMileageStateMetrics(String vin, Date statDate) {
jdbcTemplate.update("""
DELETE FROM vehicle_stat_metric
WHERE vin = ? AND stat_date = ? AND metric_key IN (?, ?)
""", vin, statDate, DAILY_MILEAGE_START_TOTAL_KEY, DAILY_MILEAGE_LATEST_TOTAL_KEY);
}
private void ensureSchema() {
jdbcTemplate.execute("""
CREATE TABLE IF NOT EXISTS vehicle_stat_metric (
@@ -133,11 +154,43 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit
metric_value DECIMAL(18,6) NULL,
metric_unit VARCHAR(16) NOT NULL,
calculation_method VARCHAR(64) NOT NULL,
first_total_mileage_km DECIMAL(18,6) NULL,
latest_total_mileage_km DECIMAL(18,6) NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (vin, stat_date, metric_key)
)
""");
ensureColumn(FIRST_TOTAL_COLUMN);
ensureColumn(LATEST_TOTAL_COLUMN);
}
private void ensureColumn(String columnName) {
if (columnExists(columnName)) {
return;
}
jdbcTemplate.execute("ALTER TABLE vehicle_stat_metric ADD COLUMN " + columnName + " DECIMAL(18,6) NULL");
}
private boolean columnExists(String columnName) {
ConnectionCallback<Boolean> callback = connection -> {
try (ResultSet columns = connection.getMetaData().getColumns(
connection.getCatalog(), null, "vehicle_stat_metric", columnName)) {
if (columns.next()) {
return true;
}
}
try (ResultSet columns = connection.getMetaData().getColumns(
connection.getCatalog(), null, "VEHICLE_STAT_METRIC", columnName.toUpperCase())) {
return columns.next();
}
};
return Boolean.TRUE.equals(jdbcTemplate.execute(callback));
}
private static double nullableDouble(ResultSet rs, String columnName) throws SQLException {
double value = rs.getDouble(columnName);
return rs.wasNull() ? Double.NaN : value;
}
private static String clean(String value) {
@@ -146,4 +199,10 @@ public final class JdbcVehicleStatMetricRepository implements VehicleStatReposit
}
return value.trim();
}
private record DailyMileageState(double firstTotalMileageKm, double latestTotalMileageKm) {
boolean hasCompleteState() {
return Double.isFinite(firstTotalMileageKm) && Double.isFinite(latestTotalMileageKm);
}
}
}

37
modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/JdbcVehicleStatMetricRepositoryTest.java
View File

@@ -27,13 +27,23 @@ class JdbcVehicleStatMetricRepositoryTest {
.extracting(result -> result.dailyMileageKm().orElseThrow())
.isEqualTo(15.5);
assertThat(jdbcTemplate.queryForObject("SELECT COUNT(*) FROM vehicle_stat_metric", Integer.class))
.isEqualTo(3);
.isOne();
assertThat(jdbcTemplate.queryForObject(
"SELECT metric_key FROM vehicle_stat_metric WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km'",
String.class)).isEqualTo("daily_mileage_km");
assertThat(jdbcTemplate.queryForObject(
"SELECT calculation_method FROM vehicle_stat_metric WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km'",
String.class)).isEqualTo("JT808_TOTAL_MILEAGE_DIFF");
assertThat(jdbcTemplate.queryForObject("""
SELECT first_total_mileage_km
FROM vehicle_stat_metric
WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km'
""", Double.class)).isEqualTo(100.0);
assertThat(jdbcTemplate.queryForObject("""
SELECT latest_total_mileage_km
FROM vehicle_stat_metric
WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km'
""", Double.class)).isEqualTo(115.5);
assertThat(jdbcTemplate.queryForObject("""
SELECT COUNT(*)
FROM information_schema.tables
@@ -58,7 +68,7 @@ class JdbcVehicleStatMetricRepositoryTest {
assertThat(second.dailyMileageKm()).hasValue(8.25);
assertThat(jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM vehicle_stat_metric WHERE vin = 'VIN001' AND stat_date = DATE '2026-07-01'",
Integer.class)).isEqualTo(3);
Integer.class)).isOne();
assertThat(jdbcTemplate.queryForObject("""
SELECT metric_value
FROM vehicle_stat_metric
@@ -70,30 +80,33 @@ class JdbcVehicleStatMetricRepositoryTest {
}
@Test
void recalculatesJt808MileageFromDailyMinAndMaxTotalsWhenSamplesArriveOutOfOrder() {
void recalculatesJt808MileageFromSingleMetricRowAfterRestart() {
JdbcTemplate jdbcTemplate = new JdbcTemplate(new DriverManagerDataSource(
"jdbc:h2:mem:vehicle_stat_metric_jt808_out_of_order;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1",
"sa",
""));
JdbcVehicleStatMetricRepository repository = new JdbcVehicleStatMetricRepository(jdbcTemplate);
repository.recordDailyMileageSample("VIN001", LocalDate.of(2026, 7, 1), 1008.75).orElseThrow();
VehicleDailyStatResult lowerSample = repository.recordDailyMileageSample(
"VIN001", LocalDate.of(2026, 7, 1), 1000.5).orElseThrow();
VehicleDailyStatResult higherSample = repository.recordDailyMileageSample(
repository.recordDailyMileageSample("VIN001", LocalDate.of(2026, 7, 1), 1000.5).orElseThrow();
JdbcVehicleStatMetricRepository restartedRepository = new JdbcVehicleStatMetricRepository(jdbcTemplate);
VehicleDailyStatResult higherSample = restartedRepository.recordDailyMileageSample(
"VIN001", LocalDate.of(2026, 7, 1), 1010.0).orElseThrow();
assertThat(lowerSample.dailyMileageKm()).hasValue(8.25);
assertThat(higherSample.dailyMileageKm()).hasValue(9.5);
assertThat(jdbcTemplate.queryForObject("""
SELECT metric_value
SELECT COUNT(*)
FROM vehicle_stat_metric
WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_start_total_km'
WHERE vin = 'VIN001' AND stat_date = DATE '2026-07-01'
""", Integer.class)).isOne();
assertThat(jdbcTemplate.queryForObject("""
SELECT first_total_mileage_km
FROM vehicle_stat_metric
WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km'
""", Double.class)).isEqualTo(1000.5);
assertThat(jdbcTemplate.queryForObject("""
SELECT metric_value
SELECT latest_total_mileage_km
FROM vehicle_stat_metric
WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_latest_total_km'
WHERE vin = 'VIN001' AND metric_key = 'daily_mileage_km'
""", Double.class)).isEqualTo(1010.0);
assertThat(jdbcTemplate.queryForObject("""
SELECT metric_value