shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ingest raw archive envelopes in history

Browse Source
This commit is contained in:
lingniu
2026-06-23 16:58:45 +08:00
parent 66807fdf3d
commit 6b82144f3c
3 changed files with 110 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java
View File

@@ -1,7 +1,11 @@
package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
@@ -21,20 +25,24 @@ import java.util.Map;
public final class TelemetryEnvelopeRecordMapper {
private static final JsonFormat.Printer JSON_PRINTER = JsonFormat.printer();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public EventFileRecord toRecord(VehicleEnvelope envelope) {
if (envelope == null) {
throw new IllegalArgumentException("envelope must not be null");
}
if (!envelope.hasTelemetrySnapshot()) {
throw new IllegalArgumentException("envelope telemetry_snapshot is required");
if (envelope.hasTelemetrySnapshot()) {
return telemetryRecord(envelope);
}
if (envelope.hasRawArchive()) {
return rawArchiveRecord(envelope);
}
throw new IllegalArgumentException("envelope telemetry_snapshot or raw_archive is required");
}
private static EventFileRecord telemetryRecord(VehicleEnvelope envelope) {
ProtocolId protocol = protocol(envelope.getSource());
Map<String, String> metadata = new LinkedHashMap<>(envelope.getMetadataMap());
if (protocol == ProtocolId.UNKNOWN && !envelope.getSource().isBlank()) {
// 保留未知 source 原文，避免历史库标准化成 UNKNOWN 后丢失排障线索。
metadata.putIfAbsent("originalSource", envelope.getSource());
}
Map<String, String> metadata = metadata(envelope, protocol);
return new EventFileRecord(
envelope.getEventId(),
protocol,
@@ -48,6 +56,40 @@ public final class TelemetryEnvelopeRecordMapper {
);
}
private static EventFileRecord rawArchiveRecord(VehicleEnvelope envelope) {
ProtocolId protocol = protocol(envelope.getSource());
RawArchiveRef rawArchive = envelope.getRawArchive();
Map<String, String> metadata = metadata(envelope, protocol);
String rawArchiveUri = rawArchive.getUri();
metadata.putIfAbsent(RawArchiveKeys.META_EVENT_ID, envelope.getEventId());
if (!rawArchiveUri.isBlank()) {
metadata.putIfAbsent(RawArchiveKeys.META_URI, rawArchiveUri);
}
return new EventFileRecord(
envelope.getEventId(),
protocol,
"RAW_ARCHIVE",
envelope.getVin(),
Instant.ofEpochMilli(envelope.getEventTimeMs()),
Instant.ofEpochMilli(envelope.getIngestTimeMs()),
rawArchiveUri,
metadata,
rawArchiveJson(envelope, metadata)
);
}
private static Map<String, String> metadata(VehicleEnvelope envelope, ProtocolId protocol) {
Map<String, String> metadata = new LinkedHashMap<>(envelope.getMetadataMap());
if (!envelope.getProtocolVersion().isBlank()) {
metadata.putIfAbsent("protocolVersion", envelope.getProtocolVersion());
}
if (protocol == ProtocolId.UNKNOWN && !envelope.getSource().isBlank()) {
// 保留未知 source 原文，避免历史库标准化成 UNKNOWN 后丢失排障线索。
metadata.putIfAbsent("originalSource", envelope.getSource());
}
return metadata;
}
private static ProtocolId protocol(String source) {
if (source == null || source.isBlank()) {
return ProtocolId.UNKNOWN;
@@ -66,4 +108,26 @@ public final class TelemetryEnvelopeRecordMapper {
throw new IllegalArgumentException("failed to serialize telemetry_snapshot", e);
}
}
private static String rawArchiveJson(VehicleEnvelope envelope, Map<String, String> metadata) {
Map<String, Object> payload = new LinkedHashMap<>();
RawArchiveRef rawArchive = envelope.getRawArchive();
payload.put("eventId", envelope.getEventId());
payload.put("vin", envelope.getVin());
payload.put("protocol", protocol(envelope.getSource()).name());
if (!envelope.getSource().isBlank()) {
payload.put("source", envelope.getSource());
}
payload.put("eventType", "RAW_ARCHIVE");
payload.put("eventTime", Instant.ofEpochMilli(envelope.getEventTimeMs()).toString());
payload.put("ingestTime", Instant.ofEpochMilli(envelope.getIngestTimeMs()).toString());
payload.put("rawArchiveUri", rawArchive.getUri());
payload.put("rawSizeBytes", rawArchive.getSizeBytes());
payload.put("metadata", metadata);
try {
return OBJECT_MAPPER.writeValueAsString(payload);
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("failed to serialize raw_archive", e);
}
}
}