shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove event file store contracts

Browse Source
This commit is contained in:
lingniu
2026-07-01 11:18:02 +08:00
parent 7ce64e9963
commit 986fc2fac3
20 changed files with 151 additions and 1315 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java
View File

@@ -1,122 +0,0 @@
package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
/**
* 通用历史事件查询接口。
*
* <p>这是跨协议的低层记录查询，返回的是 EventFileStore 中的 envelope/snapshot 记录。
* GB32960 业务查询应优先使用 {@link Gb32960FrameController}，因为它会回读 RAW 并按协议展开全部字段。
*/
@RestController
@ConditionalOnProperty(prefix = "lingniu.ingest.event-history", name = "enabled", havingValue = "true")
@ConditionalOnBean(EventFileStore.class)
@RequestMapping("/api/event-history")
public class EventHistoryController {
private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai");
private final EventFileStore store;
public EventHistoryController(EventFileStore store) {
this.store = store;
}
@GetMapping("/records")
public List<RecordResponse> query(
@RequestParam ProtocolId protocol,
@RequestParam String dateFrom,
@RequestParam String dateTo,
@RequestParam(defaultValue = "ASC") EventFileQuery.Order order,
@RequestParam(defaultValue = "100") int limit,
@RequestParam(required = false) String vin,
@RequestParam(required = false) String eventType,
@RequestParam(required = false) String cursorEventTime,
@RequestParam(required = false) String cursorIngestTime,
@RequestParam(required = false) String cursorEventId) throws IOException {
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin, eventType,
cursorEventTime, cursorIngestTime, cursorEventId).stream()
.map(RecordResponse::from)
.toList();
}
private List<EventFileRecord> queryRecords(ProtocolId protocol,
String dateFrom,
String dateTo,
EventFileQuery.Order order,
int limit,
String vin,
String eventType,
String cursorEventTime,
String cursorIngestTime,
String cursorEventId) throws IOException {
QueryTimeRange range = QueryTimeRange.parse(dateFrom, dateTo);
return store.query(new EventFileQuery(
protocol,
range.dateFrom(),
range.dateTo(),
range.eventTimeFrom(),
range.eventTimeTo(),
order,
limit,
vin,
eventType,
instantParam(cursorEventTime),
instantParam(cursorIngestTime),
cursorEventId));
}
private static Instant instantParam(String raw) {
if (raw == null || raw.isBlank()) {
return null;
}
String value = raw.trim();
try {
return OffsetDateTime.parse(value).toInstant();
} catch (RuntimeException ignored) {
return LocalDateTime.parse(value).atZone(DEFAULT_ZONE).toInstant();
}
}
public record RecordResponse(
String eventId,
ProtocolId protocol,
String eventType,
String vin,
String eventTime,
String ingestTime,
String rawArchiveUri,
Map<String, String> metadata,
String payloadJson) {
private static RecordResponse from(EventFileRecord record) {
return new RecordResponse(
record.eventId(),
record.protocol(),
record.eventType(),
record.vin(),
record.eventTime().toString(),
record.ingestTime().toString(),
record.rawArchiveUri(),
record.metadata(),
record.payloadJson());
}
}
}

64
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java
View File

@@ -3,8 +3,6 @@ package com.lingniu.ingest.eventhistory;
import com.google.protobuf.InvalidProtocolBufferException;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
@@ -16,8 +14,7 @@ import java.util.ArrayList;
import java.util.List;
/**
* Consumer-side entry point that stores one Kafka envelope value into the
* historical detail file store.
* Consumer-side entry point that stores Kafka envelope values into TDengine history tables.
*
* <p>当前 32960 本机运行配置没有启用 Kafka consumer；这个类保留给“其他服务把 envelope
* 写回历史库”的解耦部署方式。失败时返回结构化结果，由 Kafka worker 决定是否进 DLQ。
@@ -26,62 +23,24 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor
private static final Logger log = LoggerFactory.getLogger(EventHistoryEnvelopeIngestor.class);
private final EventFileStore store;
private final TelemetryEnvelopeRecordMapper mapper;
private final TdengineHistoryWriter tdengineWriter;
private final boolean telemetryFieldsEnabled;
public EventHistoryEnvelopeIngestor(EventFileStore store, TelemetryEnvelopeRecordMapper mapper) {
this(store, mapper, null);
}
public EventHistoryEnvelopeIngestor(TdengineHistoryWriter tdengineWriter) {
this(tdengineWriter, false);
}
public EventHistoryEnvelopeIngestor(TdengineHistoryWriter tdengineWriter,
boolean telemetryFieldsEnabled) {
this(null, null, tdengineWriter, false, telemetryFieldsEnabled);
}
public EventHistoryEnvelopeIngestor(EventFileStore store,
TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter) {
this(store, mapper, tdengineWriter, true, false);
}
public EventHistoryEnvelopeIngestor(EventFileStore store,
TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter,
boolean telemetryFieldsEnabled) {
this(store, mapper, tdengineWriter, true, telemetryFieldsEnabled);
}
private EventHistoryEnvelopeIngestor(EventFileStore store,
TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter,
boolean requireStore,
boolean telemetryFieldsEnabled) {
if (store == null) {
if (requireStore) {
throw new IllegalArgumentException("store must not be null");
}
if (tdengineWriter == null) {
throw new IllegalArgumentException("tdengineWriter must not be null");
}
if (store != null && mapper == null) {
throw new IllegalArgumentException("mapper must not be null");
}
this.store = store;
this.mapper = mapper;
this.tdengineWriter = tdengineWriter;
this.telemetryFieldsEnabled = telemetryFieldsEnabled;
}
public void ingest(byte[] kafkaValue) throws IOException {
VehicleEnvelope envelope = parse(kafkaValue);
if (store != null) {
EventFileRecord record = mapper.toRecord(envelope);
store.append(record);
}
writeTdengineFacts(List.of(envelope));
}
@@ -101,9 +60,8 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor
VehicleEnvelope envelope = null;
try {
envelope = parse(kafkaValue);
EventFileRecord record = store == null ? null : mapper.toRecord(envelope);
results.add(null);
valid.add(new BatchEntry(results.size() - 1, envelope, record));
valid.add(new BatchEntry(results.size() - 1, envelope));
} catch (IllegalArgumentException ex) {
results.add(envelope == null
? EnvelopeIngestResult.invalid(ex.getMessage())
@@ -114,9 +72,6 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor
return List.copyOf(results);
}
try {
if (store != null) {
store.appendAll(valid.stream().map(BatchEntry::record).toList());
}
writeTdengineFacts(valid.stream().map(BatchEntry::envelope).toList());
for (BatchEntry entry : valid) {
results.set(entry.index(), EnvelopeIngestResult.stored(
@@ -124,7 +79,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor
entry.vin()));
}
} catch (IOException ex) {
log.error("event history TDengine/file-store ingest failed batchSize={} validSize={} firstEventId={} firstVin={}",
log.error("event history TDengine ingest failed batchSize={} validSize={} firstEventId={} firstVin={}",
kafkaValues.size(),
valid.size(),
valid.getFirst().envelope().getEventId(),
@@ -140,13 +95,13 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor
return List.copyOf(results);
}
private record BatchEntry(int index, VehicleEnvelope envelope, EventFileRecord record) {
private record BatchEntry(int index, VehicleEnvelope envelope) {
private String eventId() {
return record == null ? envelope.getEventId() : record.eventId();
return envelope.getEventId();
}
private String vin() {
return record == null ? envelope.getVin() : record.vin();
return envelope.getVin();
}
}
@@ -162,9 +117,6 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor
}
private void writeTdengineFacts(List<VehicleEnvelope> envelopes) throws IOException {
if (tdengineWriter == null) {
return;
}
var rawFrames = envelopes.stream()
.flatMap(envelope -> TdengineEnvelopeRows.rawFrame(envelope).stream())
.toList();

181
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/Gb32960DecodedFrameService.java
View File

@@ -2,10 +2,6 @@ package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.model.Gb32960Message;
import com.lingniu.ingest.protocol.gb32960.model.InfoBlock;
@@ -38,8 +34,8 @@ import java.util.Map;
/**
* GB32960 RAW 历史帧查询和业务快照组装服务。
*
* <p>历史库里保存的是 RAW_ARCHIVE 索引，不保存完整解码后的宽表。查询时先从
* {@link EventFileStore} 找到 archive:// URI，再读取本地 RAW .bin，用当前协议解析器即时解码。
* <p>TDengine raw_frames 保存 archive:// URI。查询时先从 TDengine 找到 RAW 索引，
* 再读取本地 RAW .bin，用当前协议解析器即时解码。
* 这种设计让写入路径保持轻量，也允许后续修复解析器后直接重放历史 RAW 得到新字段。
*
* <p>snapshot 的聚合单位是 {@code vin + eventTime}。同一时刻可能有多个 32960 子包，
@@ -50,37 +46,20 @@ public final class Gb32960DecodedFrameService {
private static final Logger log = LoggerFactory.getLogger(Gb32960DecodedFrameService.class);
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {};
private static final int OUTPUT_DOUBLE_SCALE = 6;
private static final String RAW_ARCHIVE_EVENT_TYPE = "RAW_ARCHIVE";
private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai");
private final EventFileStore store;
private final TdengineHistoryReader tdengineReader;
private final Gb32960MessageDecoder decoder;
private final Path archiveRoot;
private final ObjectMapper objectMapper;
public Gb32960DecodedFrameService(EventFileStore store,
Gb32960MessageDecoder decoder,
Path archiveRoot,
ObjectMapper objectMapper) {
this(store, null, decoder, archiveRoot, objectMapper);
}
public Gb32960DecodedFrameService(TdengineHistoryReader tdengineReader,
Gb32960MessageDecoder decoder,
Path archiveRoot,
ObjectMapper objectMapper) {
this(null, tdengineReader, decoder, archiveRoot, objectMapper);
}
public Gb32960DecodedFrameService(EventFileStore store,
TdengineHistoryReader tdengineReader,
Gb32960MessageDecoder decoder,
Path archiveRoot,
ObjectMapper objectMapper) {
if (store == null && tdengineReader == null) {
if (tdengineReader == null) {
log.info("gb32960 decoded frame service has no history index backend; rawUri direct lookup remains available");
} else if (store == null) {
} else {
log.info("gb32960 decoded frame service using TDengine raw_frames index");
}
if (decoder == null) {
@@ -89,7 +68,6 @@ public final class Gb32960DecodedFrameService {
if (archiveRoot == null) {
throw new IllegalArgumentException("archiveRoot must not be null");
}
this.store = store;
this.tdengineReader = tdengineReader;
this.decoder = decoder;
this.archiveRoot = archiveRoot.toAbsolutePath().normalize();
@@ -98,7 +76,7 @@ public final class Gb32960DecodedFrameService {
public List<DecodedFrame> query(LocalDate dateFrom,
LocalDate dateTo,
EventFileQuery.Order order,
HistoryQueryOrder order,
int limit,
String vin,
String platformAccount) throws IOException {
@@ -109,39 +87,28 @@ public final class Gb32960DecodedFrameService {
LocalDate dateTo,
java.time.Instant eventTimeFrom,
java.time.Instant eventTimeTo,
EventFileQuery.Order order,
HistoryQueryOrder order,
int limit,
String vin,
String platformAccount) throws IOException {
int frameLimit = Math.max(1, Math.min(limit, 1000));
List<DecodedFrame> fromEventStore = store == null ? List.of() : queryFromEventStore(
dateFrom, dateTo, eventTimeFrom, eventTimeTo, order, frameLimit, vin, platformAccount);
if (!fromEventStore.isEmpty() || tdengineReader == null) {
return fromEventStore;
if (tdengineReader == null) {
return List.of();
}
return queryFromTdengine(dateFrom, dateTo, eventTimeFrom, eventTimeTo, order, frameLimit, vin, platformAccount);
}
public DecodedFramePage queryPage(QueryTimeRange range,
EventFileQuery.Order order,
HistoryQueryOrder order,
int limit,
String vin,
String platformAccount,
TdenginePageCursor cursor) throws IOException {
int frameLimit = Math.max(1, Math.min(limit, 1000));
if (tdengineReader != null && vin != null && !vin.isBlank()) {
return queryPageFromTdengine(
range.dateFrom(),
range.dateTo(),
range.eventTimeFrom(),
range.eventTimeTo(),
order,
frameLimit,
vin,
platformAccount,
cursor);
if (tdengineReader == null || vin == null || vin.isBlank()) {
return new DecodedFramePage(List.of(), null);
}
List<DecodedFrame> items = query(
return queryPageFromTdengine(
range.dateFrom(),
range.dateTo(),
range.eventTimeFrom(),
@@ -149,55 +116,15 @@ public final class Gb32960DecodedFrameService {
order,
frameLimit,
vin,
platformAccount);
return new DecodedFramePage(items, null);
}
private List<DecodedFrame> queryFromEventStore(LocalDate dateFrom,
LocalDate dateTo,
java.time.Instant eventTimeFrom,
java.time.Instant eventTimeTo,
EventFileQuery.Order order,
int frameLimit,
String vin,
String platformAccount) throws IOException {
// 只查 RAW_ARCHIVE：REALTIME/LOCATION 等派生事件不再作为 32960 历史查询的数据源。
List<EventFileRecord> records = store.query(new EventFileQuery(
ProtocolId.GB32960,
dateFrom,
dateTo,
eventTimeFrom,
eventTimeTo,
order == null ? EventFileQuery.Order.DESC : order,
frameLimit,
vin,
RAW_ARCHIVE_EVENT_TYPE));
List<DecodedFrame> out = new ArrayList<>();
LinkedHashSet<String> seenUris = new LinkedHashSet<>();
for (EventFileRecord record : records) {
String rawArchiveUri = rawArchiveUri(record);
// 同一个 RAW 可能因为索引重建或事件补偿被看到多次，按 URI 去重保证返回帧唯一。
if (rawArchiveUri == null || rawArchiveUri.isBlank() || !seenUris.add(rawArchiveUri)) {
continue;
}
DecodedFrame frame = decodeIfAvailable(record, rawArchiveUri, platformAccount);
if (frame == null) {
continue;
}
out.add(frame);
if (out.size() >= frameLimit) {
break;
}
}
return out;
platformAccount,
cursor);
}
private List<DecodedFrame> queryFromTdengine(LocalDate dateFrom,
LocalDate dateTo,
java.time.Instant eventTimeFrom,
java.time.Instant eventTimeTo,
EventFileQuery.Order order,
HistoryQueryOrder order,
int frameLimit,
String vin,
String platformAccount) throws IOException {
@@ -229,7 +156,7 @@ public final class Gb32960DecodedFrameService {
LocalDate dateTo,
java.time.Instant eventTimeFrom,
java.time.Instant eventTimeTo,
EventFileQuery.Order order,
HistoryQueryOrder order,
int frameLimit,
String vin,
String platformAccount,
@@ -261,7 +188,7 @@ public final class Gb32960DecodedFrameService {
}
public List<LogicalSnapshot> snapshots(QueryTimeRange range,
EventFileQuery.Order order,
HistoryQueryOrder order,
int limit,
String vin,
String platformAccount) throws IOException {
@@ -270,7 +197,7 @@ public final class Gb32960DecodedFrameService {
}
public List<FieldSnapshot> snapshotFields(QueryTimeRange range,
EventFileQuery.Order order,
HistoryQueryOrder order,
int limit,
String vin,
String platformAccount,
@@ -286,7 +213,7 @@ public final class Gb32960DecodedFrameService {
}
public String snapshotFieldsCsv(QueryTimeRange range,
EventFileQuery.Order order,
HistoryQueryOrder order,
int limit,
String vin,
String platformAccount,
@@ -324,18 +251,12 @@ public final class Gb32960DecodedFrameService {
if (rawArchiveUri == null || rawArchiveUri.isBlank()) {
return null;
}
if (store != null) {
EventFileRecord record = store.findByRawArchiveUri(rawArchiveUri);
if (record != null) {
return decode(record, rawArchiveUri(record), platformAccount);
}
}
return decodeWithoutIndex(rawArchiveUri, platformAccount);
}
public List<LogicalSnapshot> snapshots(LocalDate dateFrom,
LocalDate dateTo,
EventFileQuery.Order order,
HistoryQueryOrder order,
int limit,
String vin,
String platformAccount) throws IOException {
@@ -349,7 +270,7 @@ public final class Gb32960DecodedFrameService {
LocalDate dateTo,
java.time.Instant eventTimeFrom,
java.time.Instant eventTimeTo,
EventFileQuery.Order order,
HistoryQueryOrder order,
int limit,
String vin,
String platformAccount) throws IOException {
@@ -378,19 +299,6 @@ public final class Gb32960DecodedFrameService {
return out;
}
private DecodedFrame decodeIfAvailable(EventFileRecord record,
String rawArchiveUri,
String platformAccount) throws IOException {
try {
return decode(record, rawArchiveUri, platformAccount);
} catch (NoSuchFileException e) {
// 索引存在但 RAW 文件被人工清理时不中断整次查询，返回仍可用的其他帧。
log.warn("skip gb32960 frame because raw archive is missing eventId={} rawArchiveUri={}",
record.eventId(), rawArchiveUri);
return null;
}
}
private DecodedFrame decodeIfAvailable(TdengineRawFrameRow row, String platformAccount) throws IOException {
try {
return decode(row, platformAccount);
@@ -401,29 +309,6 @@ public final class Gb32960DecodedFrameService {
}
}
private DecodedFrame decode(EventFileRecord record, String rawArchiveUri, String platformAccount) throws IOException {
Path rawPath = archivePath(rawArchiveUri);
byte[] rawBytes = Files.readAllBytes(rawPath);
Gb32960Message message = decoder.decode(ByteBuffer.wrap(rawBytes), platformAccount(record, platformAccount));
List<Map<String, Object>> blocks = new ArrayList<>();
for (InfoBlock block : message.infoBlocks()) {
blocks.add(blockJson(block));
}
return new DecodedFrame(
record.eventId(),
message.header().vin(),
message.header().command().name(),
message.header().responseFlag().name(),
message.header().protocolVersion().name(),
message.header().encryptType().name(),
message.header().dataLength(),
message.header().eventTime() == null ? null : message.header().eventTime().toString(),
record.ingestTime().toString(),
rawArchiveUri,
rawBytes.length,
blocks);
}
private DecodedFrame decode(TdengineRawFrameRow row, String platformAccount) throws IOException {
Path rawPath = archivePath(row.rawUri());
byte[] rawBytes = Files.readAllBytes(rawPath);
@@ -618,26 +503,6 @@ public final class Gb32960DecodedFrameService {
return path;
}
private static String rawArchiveUri(EventFileRecord record) {
if (record.rawArchiveUri() != null && !record.rawArchiveUri().isBlank()) {
return record.rawArchiveUri();
}
if (record.metadata() == null) {
return "";
}
return record.metadata().getOrDefault("rawArchiveUri", "");
}
private static String platformAccount(EventFileRecord record, String override) {
if (override != null && !override.isBlank()) {
return override;
}
if (record.metadata() == null) {
return null;
}
return record.metadata().get("platformAccount");
}
private String platformAccount(TdengineRawFrameRow row, String override) {
if (override != null && !override.isBlank()) {
return override;
@@ -663,8 +528,8 @@ public final class Gb32960DecodedFrameService {
return inclusive.plusMillis(1);
}
private static TdengineQueryOrder tdengineOrder(EventFileQuery.Order order) {
return order == EventFileQuery.Order.ASC ? TdengineQueryOrder.ASC : TdengineQueryOrder.DESC;
private static TdengineQueryOrder tdengineOrder(HistoryQueryOrder order) {
return order == HistoryQueryOrder.ASC ? TdengineQueryOrder.ASC : TdengineQueryOrder.DESC;
}
private static String rawArchiveEventId(String rawArchiveUri) {

13
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/Gb32960FrameController.java
View File

@@ -1,6 +1,5 @@
package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.tdenginehistory.TdenginePageCursor;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
@@ -23,7 +22,7 @@ import java.util.List;
*
* <p>这个 Controller 只做参数校验、时间范围解析和 Swagger 说明，真正的 RAW 读取、
* 协议解码、snapshot 合并、字段投影都放在 {@link Gb32960DecodedFrameService}。
* 这样可以保证接口层足够薄，后续如果要把查询服务拆出去，只需要迁移 service 和 store。
* 这样可以保证接口层足够薄，后续如果要把查询服务拆出去，只需要迁移 service。
*/
@RestController
@ConditionalOnProperty(prefix = "lingniu.ingest.event-history", name = "enabled", havingValue = "true")
@@ -52,7 +51,7 @@ public final class Gb32960FrameController {
@Parameter(description = "结束时间，支持日期或精确到秒的时间；无时区时按东八区解析。", example = "2026-06-23T10:49:00")
@RequestParam String dateTo,
@Parameter(description = "排序方向。", example = "DESC")
@RequestParam(defaultValue = "DESC") EventFileQuery.Order order,
@RequestParam(defaultValue = "DESC") HistoryQueryOrder order,
@Parameter(description = "返回帧数量上限。", example = "10")
@RequestParam(defaultValue = "10") int limit,
@Parameter(description = "车辆 VIN；用于缩小到单车查询。", example = "LB9A32A20P0LS1257")
@@ -82,7 +81,7 @@ public final class Gb32960FrameController {
@Parameter(description = "结束时间，支持日期或精确到秒的时间；无时区时按东八区解析。", example = "2026-06-23T10:49:00")
@RequestParam String dateTo,
@Parameter(description = "排序方向。", example = "DESC")
@RequestParam(defaultValue = "DESC") EventFileQuery.Order order,
@RequestParam(defaultValue = "DESC") HistoryQueryOrder order,
@Parameter(description = "返回帧数量上限，最大 1000。", example = "100")
@RequestParam(defaultValue = "100") int limit,
@Parameter(description = "车辆 VIN，必填；用于命中 TDengine raw child table。", required = true, example = "LB9A32A20P0LS1257")
@@ -139,7 +138,7 @@ public final class Gb32960FrameController {
@Parameter(description = "结束时间，支持日期或精确到秒的时间；无时区时按东八区解析。", example = "2026-06-23T10:49:00")
@RequestParam String dateTo,
@Parameter(description = "排序方向。", example = "DESC")
@RequestParam(defaultValue = "DESC") EventFileQuery.Order order,
@RequestParam(defaultValue = "DESC") HistoryQueryOrder order,
@Parameter(description = "返回 snapshot 数量上限。", example = "10")
@RequestParam(defaultValue = "10") int limit,
@Parameter(description = "车辆 VIN，必填；snapshot 查询只支持单车，避免跨车合并和低效扫描。", required = true, example = "LB9A32A20P0LS1257")
@@ -163,7 +162,7 @@ public final class Gb32960FrameController {
@Parameter(description = "结束时间，支持日期或精确到秒的时间；无时区时按东八区解析。", example = "2026-06-23T10:49:00")
@RequestParam String dateTo,
@Parameter(description = "排序方向。", example = "DESC")
@RequestParam(defaultValue = "DESC") EventFileQuery.Order order,
@RequestParam(defaultValue = "DESC") HistoryQueryOrder order,
@Parameter(description = "返回 snapshot 数量上限。", example = "10")
@RequestParam(defaultValue = "10") int limit,
@Parameter(description = "车辆 VIN，必填；字段查询只支持单车。", required = true, example = "LB9A32A20P0LS1257")
@@ -189,7 +188,7 @@ public final class Gb32960FrameController {
@Parameter(description = "结束时间，支持日期或精确到秒的时间；无时区时按东八区解析。", example = "2026-06-23T10:49:00")
@RequestParam String dateTo,
@Parameter(description = "排序方向。", example = "DESC")
@RequestParam(defaultValue = "DESC") EventFileQuery.Order order,
@RequestParam(defaultValue = "DESC") HistoryQueryOrder order,
@Parameter(description = "返回 snapshot 数量上限。", example = "1000")
@RequestParam(defaultValue = "1000") int limit,
@Parameter(description = "车辆 VIN，必填；字段导出只支持单车。", required = true, example = "LB9A32A20P0LS1257")

6
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/HistoryQueryOrder.java Normal file
View File

@@ -0,0 +1,6 @@
package com.lingniu.ingest.eventhistory;
public enum HistoryQueryOrder {
ASC,
DESC
}

173
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java
View File

@@ -1,173 +0,0 @@
package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* Maps Kafka protobuf envelopes into records stored by the historical detail
* file store.
*
* <p>这是跨协议 Kafka envelope 回灌历史库的通用适配器。它保存的是 protobuf
* telemetry_snapshot 的 JSON 视图，不负责 GB32960 原始包解码；32960 全字段 snapshot
* 查询仍以 rawArchiveUri 指向的 .bin 为准。
*/
public final class TelemetryEnvelopeRecordMapper {
private static final JsonFormat.Printer JSON_PRINTER = JsonFormat.printer();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public EventFileRecord toRecord(VehicleEnvelope envelope) {
if (envelope == null) {
throw new IllegalArgumentException("envelope must not be null");
}
if (envelope.hasTelemetrySnapshot()) {
return telemetryRecord(envelope);
}
if (envelope.hasRawArchive()) {
return rawArchiveRecord(envelope);
}
throw new IllegalArgumentException("envelope telemetry_snapshot or raw_archive is required");
}
private static EventFileRecord telemetryRecord(VehicleEnvelope envelope) {
ProtocolId protocol = protocol(envelope.getSource());
Map<String, String> metadata = metadata(envelope, protocol);
return new EventFileRecord(
envelope.getEventId(),
protocol,
envelope.getTelemetrySnapshot().getEventType(),
envelope.getVin(),
Instant.ofEpochMilli(envelope.getEventTimeMs()),
Instant.ofEpochMilli(envelope.getIngestTimeMs()),
envelope.getTelemetrySnapshot().getRawArchiveUri(),
metadata,
snapshotJson(envelope)
);
}
private static EventFileRecord rawArchiveRecord(VehicleEnvelope envelope) {
ProtocolId protocol = protocol(envelope.getSource());
RawArchiveRef rawArchive = envelope.getRawArchive();
Map<String, String> metadata = metadata(envelope, protocol);
String rawArchiveKey = rawArchiveKey(rawArchive.getUri(), metadata);
String rawArchiveUri = rawArchive.getUri().isBlank() && !rawArchiveKey.isBlank()
? RawArchiveKeys.logicalUri(rawArchiveKey)
: rawArchive.getUri();
metadata.putIfAbsent(RawArchiveKeys.META_EVENT_ID, envelope.getEventId());
if (!rawArchiveKey.isBlank()) {
metadata.putIfAbsent(RawArchiveKeys.META_KEY, rawArchiveKey);
}
if (!rawArchiveUri.isBlank()) {
metadata.putIfAbsent(RawArchiveKeys.META_URI, rawArchiveUri);
}
return new EventFileRecord(
envelope.getEventId(),
protocol,
"RAW_ARCHIVE",
envelope.getVin(),
Instant.ofEpochMilli(envelope.getEventTimeMs()),
Instant.ofEpochMilli(envelope.getIngestTimeMs()),
rawArchiveUri,
metadata,
rawArchiveJson(envelope, metadata, rawArchiveKey, rawArchiveUri)
);
}
private static String rawArchiveKey(String rawArchiveUri, Map<String, String> metadata) {
String key = metadata.getOrDefault(RawArchiveKeys.META_KEY, "");
if (!key.isBlank()) {
return key;
}
if (rawArchiveUri != null && rawArchiveUri.startsWith("archive://")) {
return rawArchiveUri.substring("archive://".length());
}
return "";
}
private static Map<String, String> metadata(VehicleEnvelope envelope, ProtocolId protocol) {
Map<String, String> metadata = new LinkedHashMap<>(envelope.getMetadataMap());
if (!envelope.getProtocolVersion().isBlank()) {
metadata.putIfAbsent("protocolVersion", envelope.getProtocolVersion());
}
if (protocol == ProtocolId.UNKNOWN && !envelope.getSource().isBlank()) {
// 保留未知 source 原文，避免历史库标准化成 UNKNOWN 后丢失排障线索。
metadata.putIfAbsent("originalSource", envelope.getSource());
}
return metadata;
}
private static ProtocolId protocol(String source) {
if (source == null || source.isBlank()) {
return ProtocolId.UNKNOWN;
}
try {
return ProtocolId.valueOf(source);
} catch (IllegalArgumentException ex) {
return ProtocolId.UNKNOWN;
}
}
private static String snapshotJson(VehicleEnvelope envelope) {
try {
return JSON_PRINTER.print(envelope.getTelemetrySnapshot());
} catch (InvalidProtocolBufferException e) {
throw new IllegalArgumentException("failed to serialize telemetry_snapshot", e);
}
}
private static String rawArchiveJson(VehicleEnvelope envelope,
Map<String, String> metadata,
String rawArchiveKey,
String rawArchiveUri) {
Map<String, Object> payload = new LinkedHashMap<>();
RawArchiveRef rawArchive = envelope.getRawArchive();
payload.put("eventId", envelope.getEventId());
payload.put("vin", envelope.getVin());
payload.put("protocol", protocol(envelope.getSource()).name());
if (!envelope.getSource().isBlank()) {
payload.put("source", envelope.getSource());
}
payload.put("eventType", "RAW_ARCHIVE");
payload.put("eventTime", Instant.ofEpochMilli(envelope.getEventTimeMs()).toString());
payload.put("ingestTime", Instant.ofEpochMilli(envelope.getIngestTimeMs()).toString());
payload.put("rawArchiveUri", rawArchiveUri);
payload.put("rawArchiveKey", rawArchiveKey);
if (metadata.containsKey("command")) {
payload.put("command", metadata.get("command"));
}
if (metadata.containsKey("infoType")) {
payload.put("infoType", metadata.get("infoType"));
}
payload.put("rawSizeBytes", rawArchive.getSizeBytes());
putParsedJson(payload, rawArchive.getParsedJson());
payload.put("metadata", metadata);
try {
return OBJECT_MAPPER.writeValueAsString(payload);
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("failed to serialize raw_archive", e);
}
}
private static void putParsedJson(Map<String, Object> payload, String parsedJson) {
if (parsedJson == null || parsedJson.isBlank()) {
return;
}
try {
payload.put("parsed", OBJECT_MAPPER.readTree(parsedJson));
} catch (JsonProcessingException ex) {
payload.put("parsed", parsedJson);
payload.put("parsedJsonError", ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
}
}
}

35
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/config/EventHistoryAutoConfiguration.java
View File

@@ -2,8 +2,6 @@ package com.lingniu.ingest.eventhistory.config;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.eventhistory.EventHistoryController;
import com.lingniu.ingest.eventhistory.EventHistoryEnvelopeIngestor;
import com.lingniu.ingest.eventhistory.Gb32960DecodedFrameService;
import com.lingniu.ingest.eventhistory.Gb32960FrameController;
@@ -12,7 +10,6 @@ import com.lingniu.ingest.eventhistory.Jt808RawFrameHistoryController;
import com.lingniu.ingest.eventhistory.LocationHistoryController;
import com.lingniu.ingest.eventhistory.RawFrameHistoryController;
import com.lingniu.ingest.eventhistory.TelemetryFieldHistoryController;
import com.lingniu.ingest.eventhistory.TelemetryEnvelopeRecordMapper;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
@@ -37,7 +34,7 @@ import java.nio.file.Path;
* <p>该模块有两类入口：
* <ul>
* <li>TDengine HTTP 查询入口：查询位置、RAW 帧和按需解码的 GB32960 帧。
* <li>兼容 EventFileStore 入口：仅在旧索引显式启用时创建旧记录查询和旧写入路径。
* <li>Kafka consumer 入口：消费 envelope 并写入 TDengine 历史表。
* </ul>
*
* <p>当前生产 history app 以 TDengine 为准；Kafka consumer 是否启动还取决于
@@ -52,22 +49,6 @@ import java.nio.file.Path;
@ConditionalOnProperty(prefix = "lingniu.ingest.event-history", name = "enabled", havingValue = "true")
public class EventHistoryAutoConfiguration {
@Bean
@ConditionalOnBean(EventFileStore.class)
@ConditionalOnMissingBean
public TelemetryEnvelopeRecordMapper telemetryEnvelopeRecordMapper() {
return new TelemetryEnvelopeRecordMapper();
}
@Bean
@ConditionalOnBean(EventFileStore.class)
@ConditionalOnMissingBean(value = {TdengineHistoryWriter.class, EventHistoryEnvelopeIngestor.class})
public EventHistoryEnvelopeIngestor eventHistoryEnvelopeIngestor(EventFileStore store,
TelemetryEnvelopeRecordMapper mapper,
ObjectProvider<TdengineHistoryWriter> writer) {
return new EventHistoryEnvelopeIngestor(store, mapper, writer.getIfAvailable());
}
@Bean
@ConditionalOnBean(TdengineHistoryWriter.class)
@ConditionalOnMissingBean
@@ -94,24 +75,14 @@ public class EventHistoryAutoConfiguration {
return new EnvelopeConsumerProcessor("event-history-raw", ingestor, deadLetterSink);
}
@Bean
@ConditionalOnBean(EventFileStore.class)
@ConditionalOnMissingBean
public EventHistoryController eventHistoryController(EventFileStore store) {
return new EventHistoryController(store);
}
@Bean
@ConditionalOnBean(Gb32960MessageDecoder.class)
@ConditionalOnMissingBean
public Gb32960DecodedFrameService gb32960DecodedFrameService(ObjectProvider<EventFileStore> store,
ObjectProvider<TdengineHistoryReader> reader,
public Gb32960DecodedFrameService gb32960DecodedFrameService(ObjectProvider<TdengineHistoryReader> reader,
Gb32960MessageDecoder decoder,
@Value("${lingniu.ingest.event-history.archive-path:${SINK_ARCHIVE_PATH:./archive/}}")
String archivePath) {
// 优先使用 EventFileStore 索引；TDengine-only 高吞吐运行时可直接从 raw_frames 找 rawUri。
return new Gb32960DecodedFrameService(store.getIfAvailable(), reader.getIfAvailable(),
decoder, archiveRoot(archivePath), null);
return new Gb32960DecodedFrameService(reader.getIfAvailable(), decoder, archiveRoot(archivePath), null);
}
@Bean