feat: productionize raw history ingestion
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
package com.lingniu.ingest.api.consumer;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Optional extension for consumers that can persist a Kafka poll batch in one
|
||||
* storage operation.
|
||||
*/
|
||||
public interface EnvelopeBatchIngestor extends EnvelopeIngestor {
|
||||
|
||||
List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues);
|
||||
}
|
||||
@@ -70,6 +70,32 @@ public final class EnvelopeConsumerProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
public List<EnvelopeIngestResult> processBatch(List<EnvelopeConsumerRecord> records) {
|
||||
if (records == null || records.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
if (!(ingestor instanceof EnvelopeBatchIngestor batchIngestor)) {
|
||||
return records.stream().map(this::process).toList();
|
||||
}
|
||||
List<byte[]> payloads = records.stream()
|
||||
.map(EnvelopeConsumerRecord::payload)
|
||||
.toList();
|
||||
List<EnvelopeIngestResult> results = batchIngestor.tryIngestAll(payloads);
|
||||
if (results.size() != records.size()) {
|
||||
throw new IllegalStateException("batch ingestor returned " + results.size()
|
||||
+ " results for " + records.size() + " records");
|
||||
}
|
||||
List<EnvelopeIngestResult> copy = new ArrayList<>(results.size());
|
||||
for (int i = 0; i < records.size(); i++) {
|
||||
EnvelopeIngestResult result = results.get(i);
|
||||
if (DEAD_LETTER_STATUSES.contains(result.status())) {
|
||||
deadLetterSink.publish(toDeadLetter(records.get(i), result));
|
||||
}
|
||||
copy.add(result);
|
||||
}
|
||||
return List.copyOf(copy);
|
||||
}
|
||||
|
||||
private EnvelopeDeadLetterRecord toDeadLetter(EnvelopeConsumerRecord record, EnvelopeIngestResult result) {
|
||||
return new EnvelopeDeadLetterRecord(
|
||||
service,
|
||||
|
||||
@@ -19,6 +19,7 @@ public final class RawArchiveKeys {
|
||||
public static final String META_EVENT_ID = "rawArchiveEventId";
|
||||
public static final String META_KEY = "rawArchiveKey";
|
||||
public static final String META_URI = "rawArchiveUri";
|
||||
public static final String META_PARSED_JSON = "rawArchiveParsedJson";
|
||||
|
||||
private static final DateTimeFormatter DATE_KEY =
|
||||
DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneId.of("Asia/Shanghai"));
|
||||
|
||||
@@ -150,6 +150,7 @@ public sealed interface VehicleEvent
|
||||
* @param command 协议主命令码(如 32960 0x02/0x03 等),冗余在事件里便于按命令分片归档
|
||||
* @param infoType 协议子类型(可为 0),同上
|
||||
* @param rawBytes 原始入站字节,不可为 null
|
||||
* @param parsedJson 已解析结构化 JSON。只用于 RAW 明细查询,不应复制到位置/字段等派生表。
|
||||
*/
|
||||
record RawArchive(
|
||||
String eventId,
|
||||
@@ -161,6 +162,25 @@ public sealed interface VehicleEvent
|
||||
Map<String, String> metadata,
|
||||
int command,
|
||||
int infoType,
|
||||
byte[] rawBytes
|
||||
) implements VehicleEvent {}
|
||||
byte[] rawBytes,
|
||||
String parsedJson
|
||||
) implements VehicleEvent {
|
||||
public RawArchive(String eventId,
|
||||
String vin,
|
||||
ProtocolId source,
|
||||
Instant eventTime,
|
||||
Instant ingestTime,
|
||||
String traceId,
|
||||
Map<String, String> metadata,
|
||||
int command,
|
||||
int infoType,
|
||||
byte[] rawBytes) {
|
||||
this(eventId, vin, source, eventTime, ingestTime, traceId, metadata,
|
||||
command, infoType, rawBytes, "");
|
||||
}
|
||||
|
||||
public RawArchive {
|
||||
parsedJson = parsedJson == null ? "" : parsedJson;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user