feat: productionize raw history ingestion

This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions

View File

@@ -0,0 +1,12 @@
package com.lingniu.ingest.api.consumer;
import java.util.List;
/**
* Optional extension for consumers that can persist a Kafka poll batch in one
* storage operation.
*/
public interface EnvelopeBatchIngestor extends EnvelopeIngestor {
List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues);
}

View File

@@ -70,6 +70,32 @@ public final class EnvelopeConsumerProcessor {
}
}
public List<EnvelopeIngestResult> processBatch(List<EnvelopeConsumerRecord> records) {
if (records == null || records.isEmpty()) {
return List.of();
}
if (!(ingestor instanceof EnvelopeBatchIngestor batchIngestor)) {
return records.stream().map(this::process).toList();
}
List<byte[]> payloads = records.stream()
.map(EnvelopeConsumerRecord::payload)
.toList();
List<EnvelopeIngestResult> results = batchIngestor.tryIngestAll(payloads);
if (results.size() != records.size()) {
throw new IllegalStateException("batch ingestor returned " + results.size()
+ " results for " + records.size() + " records");
}
List<EnvelopeIngestResult> copy = new ArrayList<>(results.size());
for (int i = 0; i < records.size(); i++) {
EnvelopeIngestResult result = results.get(i);
if (DEAD_LETTER_STATUSES.contains(result.status())) {
deadLetterSink.publish(toDeadLetter(records.get(i), result));
}
copy.add(result);
}
return List.copyOf(copy);
}
private EnvelopeDeadLetterRecord toDeadLetter(EnvelopeConsumerRecord record, EnvelopeIngestResult result) {
return new EnvelopeDeadLetterRecord(
service,

View File

@@ -19,6 +19,7 @@ public final class RawArchiveKeys {
public static final String META_EVENT_ID = "rawArchiveEventId";
public static final String META_KEY = "rawArchiveKey";
public static final String META_URI = "rawArchiveUri";
public static final String META_PARSED_JSON = "rawArchiveParsedJson";
private static final DateTimeFormatter DATE_KEY =
DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneId.of("Asia/Shanghai"));

View File

@@ -150,6 +150,7 @@ public sealed interface VehicleEvent
* @param command 协议主命令码(如 32960 0x02/0x03 等),冗余在事件里便于按命令分片归档
* @param infoType 协议子类型(可为 0同上
* @param rawBytes 原始入站字节,不可为 null
* @param parsedJson 已解析结构化 JSON。只用于 RAW 明细查询,不应复制到位置/字段等派生表。
*/
record RawArchive(
String eventId,
@@ -161,6 +162,25 @@ public sealed interface VehicleEvent
Map<String, String> metadata,
int command,
int infoType,
byte[] rawBytes
) implements VehicleEvent {}
byte[] rawBytes,
String parsedJson
) implements VehicleEvent {
public RawArchive(String eventId,
String vin,
ProtocolId source,
Instant eventTime,
Instant ingestTime,
String traceId,
Map<String, String> metadata,
int command,
int infoType,
byte[] rawBytes) {
this(eventId, vin, source, eventTime, ingestTime, traceId, metadata,
command, infoType, rawBytes, "");
}
public RawArchive {
parsedJson = parsedJson == null ? "" : parsedJson;
}
}
}