shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: keep jt808 raw history consumer current

Browse Source
This commit is contained in:
lingniu
2026-06-29 15:56:41 +08:00
parent 57ef82f563
commit 98b37e71e9
13 changed files with 274 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java
View File

@@ -10,6 +10,7 @@ import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
@@ -47,30 +48,57 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
VehicleEnvelope envelope = parse(kafkaValue);
EventFileRecord record = mapper.toRecord(envelope);
store.append(record);
writeTdengineFacts(envelope);
writeTdengineFacts(List.of(envelope));
}
@Override
public EnvelopeIngestResult tryIngest(byte[] kafkaValue) {
VehicleEnvelope envelope = null;
try {
envelope = parse(kafkaValue);
EventFileRecord record = mapper.toRecord(envelope);
store.append(record);
writeTdengineFacts(envelope);
return EnvelopeIngestResult.stored(record.eventId(), record.vin());
} catch (IllegalArgumentException ex) {
// 协议不可解析或缺少必要字段属于不可重试问题，避免 Kafka consumer 无限重放。
return envelope == null
? EnvelopeIngestResult.invalid(ex.getMessage())
: EnvelopeIngestResult.skipped(envelope.getEventId(), envelope.getVin(), ex.getMessage());
} catch (IOException ex) {
// 文件库写入失败可能是临时 I/O 问题，交给上层 worker 按失败处理。
return EnvelopeIngestResult.failed(
envelope == null ? "" : envelope.getEventId(),
envelope == null ? "" : envelope.getVin(),
ex.getMessage());
return tryIngestAll(List.of(kafkaValue)).getFirst();
}
@Override
public List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues) {
if (kafkaValues == null || kafkaValues.isEmpty()) {
return List.of();
}
List<EnvelopeIngestResult> results = new ArrayList<>(kafkaValues.size());
List<BatchEntry> valid = new ArrayList<>(kafkaValues.size());
for (byte[] kafkaValue : kafkaValues) {
VehicleEnvelope envelope = null;
try {
envelope = parse(kafkaValue);
EventFileRecord record = mapper.toRecord(envelope);
results.add(null);
valid.add(new BatchEntry(results.size() - 1, envelope, record));
} catch (IllegalArgumentException ex) {
results.add(envelope == null
? EnvelopeIngestResult.invalid(ex.getMessage())
: EnvelopeIngestResult.skipped(envelope.getEventId(), envelope.getVin(), ex.getMessage()));
}
}
if (valid.isEmpty()) {
return List.copyOf(results);
}
try {
store.appendAll(valid.stream().map(BatchEntry::record).toList());
writeTdengineFacts(valid.stream().map(BatchEntry::envelope).toList());
for (BatchEntry entry : valid) {
results.set(entry.index(), EnvelopeIngestResult.stored(
entry.record().eventId(),
entry.record().vin()));
}
} catch (IOException ex) {
for (BatchEntry entry : valid) {
results.set(entry.index(), EnvelopeIngestResult.failed(
entry.envelope().getEventId(),
entry.envelope().getVin(),
ex.getMessage()));
}
}
return List.copyOf(results);
}
private record BatchEntry(int index, VehicleEnvelope envelope, EventFileRecord record) {
}
private static VehicleEnvelope parse(byte[] kafkaValue) {
@@ -84,19 +112,25 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
}
}
private void writeTdengineFacts(VehicleEnvelope envelope) throws IOException {
private void writeTdengineFacts(List<VehicleEnvelope> envelopes) throws IOException {
if (tdengineWriter == null) {
return;
}
var rawFrame = TdengineEnvelopeRows.rawFrame(envelope);
if (rawFrame.isPresent()) {
tdengineWriter.appendRawFrames(List.of(rawFrame.get()));
var rawFrames = envelopes.stream()
.flatMap(envelope -> TdengineEnvelopeRows.rawFrame(envelope).stream())
.toList();
if (!rawFrames.isEmpty()) {
tdengineWriter.appendRawFrames(rawFrames);
}
var location = TdengineEnvelopeRows.location(envelope);
if (location.isPresent()) {
tdengineWriter.appendLocations(List.of(location.get()));
var locations = envelopes.stream()
.flatMap(envelope -> TdengineEnvelopeRows.location(envelope).stream())
.toList();
if (!locations.isEmpty()) {
tdengineWriter.appendLocations(locations);
}
var telemetryFields = TdengineEnvelopeRows.telemetryFields(envelope);
var telemetryFields = envelopes.stream()
.flatMap(envelope -> TdengineEnvelopeRows.telemetryFields(envelope).stream())
.toList();
if (!telemetryFields.isEmpty()) {
tdengineWriter.appendTelemetryFields(telemetryFields);
}