feat: productionize raw history ingestion

2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
--- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java
+++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java
@@ -12,6 +12,10 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;

 import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
 import java.util.List;
 import java.util.Map;

@@ -27,6 +31,8 @@ import java.util.Map;
@RequestMapping("/api/event-history")
 public class EventHistoryController {

+    private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai");
+
    private final EventFileStore store;

    public EventHistoryController(EventFileStore store) {
@@ -40,8 +46,13 @@ public class EventHistoryController {
            @RequestParam String dateTo,
            @RequestParam(defaultValue = "ASC") EventFileQuery.Order order,
            @RequestParam(defaultValue = "100") int limit,
-            @RequestParam(required = false) String vin) throws IOException {
-        return queryRecords(protocol, dateFrom, dateTo, order, limit, vin).stream()
+            @RequestParam(required = false) String vin,
+            @RequestParam(required = false) String eventType,
+            @RequestParam(required = false) String cursorEventTime,
+            @RequestParam(required = false) String cursorIngestTime,
+            @RequestParam(required = false) String cursorEventId) throws IOException {
+        return queryRecords(protocol, dateFrom, dateTo, order, limit, vin, eventType,
+                        cursorEventTime, cursorIngestTime, cursorEventId).stream()
                .map(RecordResponse::from)
                .toList();
    }
@@ -51,7 +62,11 @@ public class EventHistoryController {
                                               String dateTo,
                                               EventFileQuery.Order order,
                                               int limit,
-                                               String vin) throws IOException {
+                                               String vin,
+                                               String eventType,
+                                               String cursorEventTime,
+                                               String cursorIngestTime,
+                                               String cursorEventId) throws IOException {
        QueryTimeRange range = QueryTimeRange.parse(dateFrom, dateTo);
        return store.query(new EventFileQuery(
                protocol,
@@ -62,7 +77,22 @@ public class EventHistoryController {
                order,
                limit,
                vin,
-                null));
+                eventType,
+                instantParam(cursorEventTime),
+                instantParam(cursorIngestTime),
+                cursorEventId));
+    }
+
+    private static Instant instantParam(String raw) {
+        if (raw == null || raw.isBlank()) {
+            return null;
+        }
+        String value = raw.trim();
+        try {
+            return OffsetDateTime.parse(value).toInstant();
+        } catch (RuntimeException ignored) {
+            return LocalDateTime.parse(value).atZone(DEFAULT_ZONE).toInstant();
+        }
    }

    public record RecordResponse(
--- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java
+++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java
@@ -1,8 +1,8 @@
 package com.lingniu.ingest.eventhistory;

 import com.google.protobuf.InvalidProtocolBufferException;
+import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
 import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
-import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
 import com.lingniu.ingest.eventfilestore.EventFileRecord;
 import com.lingniu.ingest.eventfilestore.EventFileStore;
 import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
@@ -20,7 +20,7 @@ import java.util.List;
 * <p>当前 32960 本机运行配置没有启用 Kafka consumer；这个类保留给“其他服务把 envelope
 * 写回历史库”的解耦部署方式。失败时返回结构化结果，由 Kafka worker 决定是否进 DLQ。
 */
-public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
+public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor {

    private final EventFileStore store;
    private final TelemetryEnvelopeRecordMapper mapper;
@@ -33,7 +33,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {

    public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
                                        TdengineHistoryWriter tdengineWriter) {
-        this(mapper, tdengineWriter, true);
+        this(mapper, tdengineWriter, false);
    }

    public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
@@ -45,7 +45,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
    public EventHistoryEnvelopeIngestor(EventFileStore store,
                                        TelemetryEnvelopeRecordMapper mapper,
                                        TdengineHistoryWriter tdengineWriter) {
-        this(store, mapper, tdengineWriter, true, true);
+        this(store, mapper, tdengineWriter, true, false);
    }

    public EventHistoryEnvelopeIngestor(EventFileStore store,
--- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryController.java
+++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryController.java
@@ -1,7 +1,5 @@
 package com.lingniu.ingest.eventhistory;

-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
 import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
 import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
@@ -32,8 +30,6 @@ import java.util.List;
@Tag(name = "jt-808-location-controller", description = "JT808 位置历史分页查询接口。")
 public final class Jt808LocationHistoryController {

-    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
    private final TdengineHistoryReader reader;

    public Jt808LocationHistoryController(TdengineHistoryReader reader) {
@@ -127,8 +123,7 @@ public final class Jt808LocationHistoryController {
            long alarmFlag,
            long statusFlag,
            Double totalMileageKm,
-            String rawUri,
-            String metadataJson) {
+            String rawUri) {

        private static LocationResponse from(TdengineLocationRow row) {
            return new LocationResponse(
@@ -147,23 +142,7 @@ public final class Jt808LocationHistoryController {
                    row.alarmFlag(),
                    row.statusFlag(),
                    row.totalMileageKm(),
-                    rawUri(row),
-                    row.metadataJson());
-        }
-
-        private static String rawUri(TdengineLocationRow row) {
-            if (row.rawUri() != null && !row.rawUri().isBlank()) {
-                return row.rawUri();
-            }
-            String metadataJson = row.metadataJson();
-            if (metadataJson == null || metadataJson.isBlank()) {
-                return row.rawUri();
-            }
-            try {
-                return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
-            } catch (JsonProcessingException ignored) {
-                return row.rawUri();
-            }
+                    row.rawUri());
        }
    }
 }
--- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/LocationHistoryController.java
+++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/LocationHistoryController.java
@@ -1,7 +1,5 @@
 package com.lingniu.ingest.eventhistory;

-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
 import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
 import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
@@ -32,8 +30,6 @@ import java.util.Locale;
@Tag(name = "location-history-controller", description = "通用位置历史分页查询接口。")
 public final class LocationHistoryController {

-    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
    private final TdengineHistoryReader reader;

    public LocationHistoryController(TdengineHistoryReader reader) {
@@ -161,7 +157,6 @@ public final class LocationHistoryController {
            long statusFlag,
            Double totalMileageKm,
            String rawUri,
-            String metadataJson,
            String protocol) {

        private static LocationResponse from(TdengineLocationRow row) {
@@ -181,24 +176,8 @@ public final class LocationHistoryController {
                    row.alarmFlag(),
                    row.statusFlag(),
                    row.totalMileageKm(),
-                    rawUri(row),
-                    row.metadataJson(),
+                    row.rawUri(),
                    row.protocol());
        }
-
-        private static String rawUri(TdengineLocationRow row) {
-            if (row.rawUri() != null && !row.rawUri().isBlank()) {
-                return row.rawUri();
-            }
-            String metadataJson = row.metadataJson();
-            if (metadataJson == null || metadataJson.isBlank()) {
-                return row.rawUri();
-            }
-            try {
-                return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
-            } catch (JsonProcessingException ignored) {
-                return row.rawUri();
-            }
-        }
    }
 }
--- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java
+++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java
@@ -184,6 +184,7 @@ public final class RawFrameHistoryController {
            String parseError,
            String peer,
            String metadataJson,
+            String parsedJson,
            Map<String, Object> metadata,
            Map<String, Object> parsedFields,
            String protocol,
@@ -209,8 +210,9 @@ public final class RawFrameHistoryController {
                    row.parseError(),
                    row.peer(),
                    row.metadataJson(),
+                    row.parsedJson(),
                    metadata,
-                    parsedFields(reader, row, metadata, includeParsedFields),
+                    parsedFields(reader, row, metadata, parsed(row.parsedJson()), includeParsedFields),
                    row.protocol(),
                    row.vehicleKey(),
                    row.vin(),
@@ -231,18 +233,23 @@ public final class RawFrameHistoryController {
        private static Map<String, Object> parsedFields(TdengineHistoryReader reader,
                                                        TdengineRawFrameRow row,
                                                        Map<String, Object> metadata,
+                                                        Map<String, Object> parsed,
                                                        boolean includeParsedFields) throws IOException {
            Map<String, Object> out = new LinkedHashMap<>();
-            for (Map.Entry<String, Object> entry : metadata.entrySet()) {
-                String key = entry.getKey();
-                if (key == null || key.isBlank()) {
-                    continue;
-                }
-                if (key.startsWith("raw.") || key.startsWith("jt808.")) {
-                    out.put(key, entry.getValue());
-                } else {
-                    out.put("raw." + key, entry.getValue());
+            if (parsed.isEmpty()) {
+                for (Map.Entry<String, Object> entry : metadata.entrySet()) {
+                    String key = entry.getKey();
+                    if (key == null || key.isBlank()) {
+                        continue;
+                    }
+                    if (key.startsWith("raw.") || key.startsWith("jt808.")) {
+                        out.put(key, entry.getValue());
+                    } else {
+                        out.put("raw." + key, entry.getValue());
+                    }
                }
+            } else {
+                out.putAll(parsed);
            }
            if (!includeParsedFields) {
                return Map.copyOf(out);
@@ -262,6 +269,17 @@ public final class RawFrameHistoryController {
            return Map.copyOf(out);
        }

+        private static Map<String, Object> parsed(String parsedJson) {
+            if (parsedJson == null || parsedJson.isBlank()) {
+                return Map.of();
+            }
+            try {
+                return OBJECT_MAPPER.readValue(parsedJson, MAP_TYPE);
+            } catch (Exception ignored) {
+                return Map.of("_raw", parsedJson);
+            }
+        }
+
        private static void addLocation(Map<String, Object> out, TdengineLocationRow row) {
            out.put("location.longitude", row.longitude());
            out.put("location.latitude", row.latitude());
--- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java
+++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java
@@ -150,6 +150,7 @@ public final class TelemetryEnvelopeRecordMapper {
            payload.put("infoType", metadata.get("infoType"));
        }
        payload.put("rawSizeBytes", rawArchive.getSizeBytes());
+        putParsedJson(payload, rawArchive.getParsedJson());
        payload.put("metadata", metadata);
        try {
            return OBJECT_MAPPER.writeValueAsString(payload);
@@ -157,4 +158,16 @@ public final class TelemetryEnvelopeRecordMapper {
            throw new IllegalArgumentException("failed to serialize raw_archive", e);
        }
    }
+
+    private static void putParsedJson(Map<String, Object> payload, String parsedJson) {
+        if (parsedJson == null || parsedJson.isBlank()) {
+            return;
+        }
+        try {
+            payload.put("parsed", OBJECT_MAPPER.readTree(parsedJson));
+        } catch (JsonProcessingException ex) {
+            payload.put("parsed", parsedJson);
+            payload.put("parsedJsonError", ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
+        }
+    }
 }