feat: productionize raw history ingestion

This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions

View File

@@ -36,6 +36,10 @@
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>

View File

@@ -12,6 +12,10 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
@@ -27,6 +31,8 @@ import java.util.Map;
@RequestMapping("/api/event-history")
public class EventHistoryController {
private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai");
private final EventFileStore store;
public EventHistoryController(EventFileStore store) {
@@ -40,8 +46,13 @@ public class EventHistoryController {
@RequestParam String dateTo,
@RequestParam(defaultValue = "ASC") EventFileQuery.Order order,
@RequestParam(defaultValue = "100") int limit,
@RequestParam(required = false) String vin) throws IOException {
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin).stream()
@RequestParam(required = false) String vin,
@RequestParam(required = false) String eventType,
@RequestParam(required = false) String cursorEventTime,
@RequestParam(required = false) String cursorIngestTime,
@RequestParam(required = false) String cursorEventId) throws IOException {
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin, eventType,
cursorEventTime, cursorIngestTime, cursorEventId).stream()
.map(RecordResponse::from)
.toList();
}
@@ -51,7 +62,11 @@ public class EventHistoryController {
String dateTo,
EventFileQuery.Order order,
int limit,
String vin) throws IOException {
String vin,
String eventType,
String cursorEventTime,
String cursorIngestTime,
String cursorEventId) throws IOException {
QueryTimeRange range = QueryTimeRange.parse(dateFrom, dateTo);
return store.query(new EventFileQuery(
protocol,
@@ -62,7 +77,22 @@ public class EventHistoryController {
order,
limit,
vin,
null));
eventType,
instantParam(cursorEventTime),
instantParam(cursorIngestTime),
cursorEventId));
}
private static Instant instantParam(String raw) {
if (raw == null || raw.isBlank()) {
return null;
}
String value = raw.trim();
try {
return OffsetDateTime.parse(value).toInstant();
} catch (RuntimeException ignored) {
return LocalDateTime.parse(value).atZone(DEFAULT_ZONE).toInstant();
}
}
public record RecordResponse(

View File

@@ -1,8 +1,8 @@
package com.lingniu.ingest.eventhistory;
import com.google.protobuf.InvalidProtocolBufferException;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
@@ -20,7 +20,7 @@ import java.util.List;
* <p>当前 32960 本机运行配置没有启用 Kafka consumer这个类保留给“其他服务把 envelope
* 写回历史库”的解耦部署方式。失败时返回结构化结果,由 Kafka worker 决定是否进 DLQ。
*/
public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor {
private final EventFileStore store;
private final TelemetryEnvelopeRecordMapper mapper;
@@ -33,7 +33,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter) {
this(mapper, tdengineWriter, true);
this(mapper, tdengineWriter, false);
}
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
@@ -45,7 +45,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
public EventHistoryEnvelopeIngestor(EventFileStore store,
TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter) {
this(store, mapper, tdengineWriter, true, true);
this(store, mapper, tdengineWriter, true, false);
}
public EventHistoryEnvelopeIngestor(EventFileStore store,

View File

@@ -1,7 +1,5 @@
package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
@@ -32,8 +30,6 @@ import java.util.List;
@Tag(name = "jt-808-location-controller", description = "JT808 位置历史分页查询接口。")
public final class Jt808LocationHistoryController {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final TdengineHistoryReader reader;
public Jt808LocationHistoryController(TdengineHistoryReader reader) {
@@ -127,8 +123,7 @@ public final class Jt808LocationHistoryController {
long alarmFlag,
long statusFlag,
Double totalMileageKm,
String rawUri,
String metadataJson) {
String rawUri) {
private static LocationResponse from(TdengineLocationRow row) {
return new LocationResponse(
@@ -147,23 +142,7 @@ public final class Jt808LocationHistoryController {
row.alarmFlag(),
row.statusFlag(),
row.totalMileageKm(),
rawUri(row),
row.metadataJson());
}
private static String rawUri(TdengineLocationRow row) {
if (row.rawUri() != null && !row.rawUri().isBlank()) {
return row.rawUri();
}
String metadataJson = row.metadataJson();
if (metadataJson == null || metadataJson.isBlank()) {
return row.rawUri();
}
try {
return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
} catch (JsonProcessingException ignored) {
return row.rawUri();
}
row.rawUri());
}
}
}

View File

@@ -1,7 +1,5 @@
package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
@@ -32,8 +30,6 @@ import java.util.Locale;
@Tag(name = "location-history-controller", description = "通用位置历史分页查询接口。")
public final class LocationHistoryController {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final TdengineHistoryReader reader;
public LocationHistoryController(TdengineHistoryReader reader) {
@@ -161,7 +157,6 @@ public final class LocationHistoryController {
long statusFlag,
Double totalMileageKm,
String rawUri,
String metadataJson,
String protocol) {
private static LocationResponse from(TdengineLocationRow row) {
@@ -181,24 +176,8 @@ public final class LocationHistoryController {
row.alarmFlag(),
row.statusFlag(),
row.totalMileageKm(),
rawUri(row),
row.metadataJson(),
row.rawUri(),
row.protocol());
}
private static String rawUri(TdengineLocationRow row) {
if (row.rawUri() != null && !row.rawUri().isBlank()) {
return row.rawUri();
}
String metadataJson = row.metadataJson();
if (metadataJson == null || metadataJson.isBlank()) {
return row.rawUri();
}
try {
return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
} catch (JsonProcessingException ignored) {
return row.rawUri();
}
}
}
}

View File

@@ -184,6 +184,7 @@ public final class RawFrameHistoryController {
String parseError,
String peer,
String metadataJson,
String parsedJson,
Map<String, Object> metadata,
Map<String, Object> parsedFields,
String protocol,
@@ -209,8 +210,9 @@ public final class RawFrameHistoryController {
row.parseError(),
row.peer(),
row.metadataJson(),
row.parsedJson(),
metadata,
parsedFields(reader, row, metadata, includeParsedFields),
parsedFields(reader, row, metadata, parsed(row.parsedJson()), includeParsedFields),
row.protocol(),
row.vehicleKey(),
row.vin(),
@@ -231,18 +233,23 @@ public final class RawFrameHistoryController {
private static Map<String, Object> parsedFields(TdengineHistoryReader reader,
TdengineRawFrameRow row,
Map<String, Object> metadata,
Map<String, Object> parsed,
boolean includeParsedFields) throws IOException {
Map<String, Object> out = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : metadata.entrySet()) {
String key = entry.getKey();
if (key == null || key.isBlank()) {
continue;
}
if (key.startsWith("raw.") || key.startsWith("jt808.")) {
out.put(key, entry.getValue());
} else {
out.put("raw." + key, entry.getValue());
if (parsed.isEmpty()) {
for (Map.Entry<String, Object> entry : metadata.entrySet()) {
String key = entry.getKey();
if (key == null || key.isBlank()) {
continue;
}
if (key.startsWith("raw.") || key.startsWith("jt808.")) {
out.put(key, entry.getValue());
} else {
out.put("raw." + key, entry.getValue());
}
}
} else {
out.putAll(parsed);
}
if (!includeParsedFields) {
return Map.copyOf(out);
@@ -262,6 +269,17 @@ public final class RawFrameHistoryController {
return Map.copyOf(out);
}
private static Map<String, Object> parsed(String parsedJson) {
if (parsedJson == null || parsedJson.isBlank()) {
return Map.of();
}
try {
return OBJECT_MAPPER.readValue(parsedJson, MAP_TYPE);
} catch (Exception ignored) {
return Map.of("_raw", parsedJson);
}
}
private static void addLocation(Map<String, Object> out, TdengineLocationRow row) {
out.put("location.longitude", row.longitude());
out.put("location.latitude", row.latitude());

View File

@@ -150,6 +150,7 @@ public final class TelemetryEnvelopeRecordMapper {
payload.put("infoType", metadata.get("infoType"));
}
payload.put("rawSizeBytes", rawArchive.getSizeBytes());
putParsedJson(payload, rawArchive.getParsedJson());
payload.put("metadata", metadata);
try {
return OBJECT_MAPPER.writeValueAsString(payload);
@@ -157,4 +158,16 @@ public final class TelemetryEnvelopeRecordMapper {
throw new IllegalArgumentException("failed to serialize raw_archive", e);
}
}
private static void putParsedJson(Map<String, Object> payload, String parsedJson) {
if (parsedJson == null || parsedJson.isBlank()) {
return;
}
try {
payload.put("parsed", OBJECT_MAPPER.readTree(parsedJson));
} catch (JsonProcessingException ex) {
payload.put("parsed", parsedJson);
payload.put("parsedJsonError", ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
}
}
}

View File

@@ -26,7 +26,11 @@ class EventHistoryControllerTest {
"2026-06-22",
EventFileQuery.Order.DESC,
50,
"VIN001");
"VIN001",
"RAW_ARCHIVE",
"2026-06-22T08:00:00Z",
"2026-06-22T08:00:01Z",
"event-0");
assertThat(records).extracting(EventHistoryController.RecordResponse::eventId).containsExactly("event-1");
assertThat(records.getFirst().eventTime()).isEqualTo("2026-06-22T08:00:00Z");
@@ -37,6 +41,10 @@ class EventHistoryControllerTest {
assertThat(store.lastQuery.order()).isEqualTo(EventFileQuery.Order.DESC);
assertThat(store.lastQuery.limit()).isEqualTo(50);
assertThat(store.lastQuery.vin()).isEqualTo("VIN001");
assertThat(store.lastQuery.eventType()).isEqualTo("RAW_ARCHIVE");
assertThat(store.lastQuery.cursorEventTime()).isEqualTo(Instant.parse("2026-06-22T08:00:00Z"));
assertThat(store.lastQuery.cursorIngestTime()).isEqualTo(Instant.parse("2026-06-22T08:00:01Z"));
assertThat(store.lastQuery.cursorEventId()).isEqualTo("event-0");
}
private static EventFileRecord record(String eventId) {

View File

@@ -126,7 +126,7 @@ class EventHistoryEnvelopeIngestorTest {
}
@Test
void tryIngestWritesRawLocationAndTelemetryFactsToTdengineWhenWriterExists() throws Exception {
void tryIngestWritesRawAndLocationFactsToTdengineWhenWriterExists() throws Exception {
CapturingStore store = new CapturingStore();
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
@@ -180,10 +180,7 @@ class EventHistoryEnvelopeIngestorTest {
.extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1");
assertThat(tdengineWriter.locations.getFirst().longitude()).isEqualTo(113.12);
assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
assertThat(tdengineWriter.telemetryFields.getFirst().valueDouble()).isEqualTo(42.5);
assertThat(tdengineWriter.telemetryFields).isEmpty();
}
@Test
@@ -200,16 +197,14 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1");
assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
assertThat(tdengineWriter.telemetryFields).isEmpty();
}
@Test
void tryIngestCanSkipTelemetryFieldFactsForHighThroughputRuntime() {
void tryIngestCanWriteTelemetryFieldFactsWhenExplicitlyEnabled() {
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
new TelemetryEnvelopeRecordMapper(), tdengineWriter, false);
new TelemetryEnvelopeRecordMapper(), tdengineWriter, true);
EnvelopeIngestResult result = ingestor.tryIngest(
jt808LocationEnvelope("jt808-location-1", "frame-jt808-1", "013800000001").toByteArray());
@@ -219,7 +214,9 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1");
assertThat(tdengineWriter.telemetryFields).isEmpty();
assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
}
@Test
@@ -249,9 +246,7 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1", "frame-jt808-2");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1", "jt808-location-2");
assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh", "location.speedKmh");
assertThat(tdengineWriter.telemetryFields).isEmpty();
}
private static VehicleEnvelope envelope(String eventId) {

View File

@@ -314,6 +314,7 @@ class Gb32960DecodedFrameServiceTest {
"",
"127.0.0.1:32960",
"{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960",
vin,
vin,
@@ -360,6 +361,7 @@ class Gb32960DecodedFrameServiceTest {
"",
"8.134.95.166:37302",
"{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960",
"unknown:GB32960:platform-login",
"",
@@ -409,6 +411,7 @@ class Gb32960DecodedFrameServiceTest {
"",
"127.0.0.1:32960",
"{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960",
vin,
vin,

View File

@@ -51,13 +51,12 @@ class Jt808LocationHistoryControllerTest {
}
@Test
void fallsBackToRawArchiveUriInMetadataWhenLocationRawUriIsBlank() throws Exception {
void returnsLocationRawUriFromCoreColumn() throws Exception {
CapturingReader reader = new CapturingReader(new TdenginePage<>(
List.of(row(
"fact-blank-raw",
"2026-06-29T05:00:01Z",
"",
"{\"rawArchiveUri\":\"archive://jt808/metadata-frame.bin\"}")),
"archive://jt808/core-frame.bin")),
Optional.empty()));
Jt808LocationHistoryController controller = new Jt808LocationHistoryController(reader);
@@ -71,7 +70,7 @@ class Jt808LocationHistoryControllerTest {
null);
assertThat(response.items().getFirst().rawUri())
.isEqualTo("archive://jt808/metadata-frame.bin");
.isEqualTo("archive://jt808/core-frame.bin");
}
@Test
@@ -92,10 +91,10 @@ class Jt808LocationHistoryControllerTest {
}
private static TdengineLocationRow row(String factId, String ts) {
return row(factId, ts, "archive://jt808/frame-1.bin", "{\"messageId\":\"512\"}");
return row(factId, ts, "archive://jt808/frame-1.bin");
}
private static TdengineLocationRow row(String factId, String ts, String rawUri, String metadataJson) {
private static TdengineLocationRow row(String factId, String ts, String rawUri) {
Instant instant = Instant.parse(ts);
return new TdengineLocationRow(
instant,
@@ -111,7 +110,6 @@ class Jt808LocationHistoryControllerTest {
3L,
null,
rawUri,
metadataJson,
"JT808",
"jt808:g7gps",
"",

View File

@@ -96,6 +96,7 @@ class Jt808RawFrameHistoryControllerTest {
"",
"222.66.200.68:41000",
"{\"jt808.register.deviceId\":\"9963320\",\"jt808.register.plate\":\"沪A61559F\"}",
"",
"JT808",
"jt808:13079963320",
"unknown",

View File

@@ -82,7 +82,6 @@ class LocationHistoryControllerTest {
1,
null,
"",
"{\"rawArchiveUri\":\"archive://raw.bin\"}",
"MQTT_YUTONG",
"LMRKH9AC2R1004087",
"LMRKH9AC2R1004087",

View File

@@ -91,7 +91,6 @@ class RawFrameHistoryControllerTest {
524290,
14504.446,
raw.rawUri(),
"{\"rawArchiveUri\":\"" + raw.rawUri() + "\"}",
"JT808",
"VIN-JT808-1",
"VIN-JT808-1",
@@ -183,6 +182,7 @@ class RawFrameHistoryControllerTest {
"xinda",
"{\"rawArchiveUri\":\"archive://2026/06/29/XINDA_PUSH/unknown/" + frameId
+ ".bin\",\"plateNo\":\"粤BD12345\",\"rawJson\":\"{\\\"carName\\\":\\\"粤BD12345\\\"}\"}",
"",
"XINDA_PUSH",
"LB9A32A24P0LS1270",
"LB9A32A24P0LS1270",

View File

@@ -2,9 +2,11 @@ package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@@ -12,6 +14,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
class TelemetryEnvelopeRecordMapperTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void mapsFullFieldEnvelopeToEventFileRecord() {
VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
@@ -91,4 +95,33 @@ class TelemetryEnvelopeRecordMapperTest {
.containsEntry("tenant", "ln")
.containsEntry("originalSource", "FUTURE_OEM");
}
@Test
void rawArchiveRecordStoresParsedJsonInPayloadOnly() throws Exception {
VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
.setSchemaVersion("1.0")
.setEventId("raw-808-1")
.setTraceId("trace-raw-1")
.setVin("VIN808")
.setSource("JT808")
.setEventTimeMs(1_782_112_400_000L)
.setIngestTimeMs(1_782_112_401_000L)
.putMetadata("phone", "13307795425")
.setRawArchive(RawArchiveRef.newBuilder()
.setUri("archive://2026/06/30/JT808/VIN808/raw-808-1.bin")
.setSizeBytes(64)
.setParsedJson("{\"messageId\":\"0x0200\",\"body\":{\"speedKmh\":60.5}}")
.build())
.build();
EventFileRecord record = new TelemetryEnvelopeRecordMapper().toRecord(envelope);
assertThat(record.eventType()).isEqualTo("RAW_ARCHIVE");
assertThat(record.metadata())
.containsEntry("phone", "13307795425")
.doesNotContainKey("rawArchiveParsedJson");
var payload = objectMapper.readTree(record.payloadJson());
assertThat(payload.path("parsed").path("messageId").asText()).isEqualTo("0x0200");
assertThat(payload.path("parsed").path("body").path("speedKmh").asDouble()).isEqualTo(60.5);
}
}

View File

@@ -35,11 +35,14 @@ class EventHistoryAutoConfigurationTest {
@Test
void createsHistoryBeansWhenEnabled() {
contextRunner
.withPropertyValues("lingniu.ingest.event-history.enabled=true")
.withPropertyValues(
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=true")
.run(context -> {
assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class);
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context).hasSingleBean(EnvelopeConsumerProcessor.class);
assertThat(context.getBeansOfType(EnvelopeConsumerProcessor.class))
.containsOnlyKeys("eventHistoryEnvelopeConsumerProcessor");
assertThat(context).hasSingleBean(EventHistoryController.class);
assertThat(context).doesNotHaveBean(Gb32960DecodedFrameService.class);
assertThat(context).doesNotHaveBean(Gb32960FrameController.class);