fix: keep jt808 raw history consumer current
This commit is contained in:
lingniu
@@ -10,6 +10,7 @@ import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
@@ -47,30 +48,57 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
VehicleEnvelope envelope = parse(kafkaValue);
|
||||
EventFileRecord record = mapper.toRecord(envelope);
|
||||
store.append(record);
|
||||
writeTdengineFacts(envelope);
|
||||
writeTdengineFacts(List.of(envelope));
|
||||
}
|
||||
|
||||
@Override
|
||||
public EnvelopeIngestResult tryIngest(byte[] kafkaValue) {
|
||||
VehicleEnvelope envelope = null;
|
||||
try {
|
||||
envelope = parse(kafkaValue);
|
||||
EventFileRecord record = mapper.toRecord(envelope);
|
||||
store.append(record);
|
||||
writeTdengineFacts(envelope);
|
||||
return EnvelopeIngestResult.stored(record.eventId(), record.vin());
|
||||
} catch (IllegalArgumentException ex) {
|
||||
// 协议不可解析或缺少必要字段属于不可重试问题，避免 Kafka consumer 无限重放。
|
||||
return envelope == null
|
||||
? EnvelopeIngestResult.invalid(ex.getMessage())
|
||||
: EnvelopeIngestResult.skipped(envelope.getEventId(), envelope.getVin(), ex.getMessage());
|
||||
} catch (IOException ex) {
|
||||
// 文件库写入失败可能是临时 I/O 问题，交给上层 worker 按失败处理。
|
||||
return EnvelopeIngestResult.failed(
|
||||
envelope == null ? "" : envelope.getEventId(),
|
||||
envelope == null ? "" : envelope.getVin(),
|
||||
ex.getMessage());
|
||||
return tryIngestAll(List.of(kafkaValue)).getFirst();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues) {
|
||||
if (kafkaValues == null || kafkaValues.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<EnvelopeIngestResult> results = new ArrayList<>(kafkaValues.size());
|
||||
List<BatchEntry> valid = new ArrayList<>(kafkaValues.size());
|
||||
for (byte[] kafkaValue : kafkaValues) {
|
||||
VehicleEnvelope envelope = null;
|
||||
try {
|
||||
envelope = parse(kafkaValue);
|
||||
EventFileRecord record = mapper.toRecord(envelope);
|
||||
results.add(null);
|
||||
valid.add(new BatchEntry(results.size() - 1, envelope, record));
|
||||
} catch (IllegalArgumentException ex) {
|
||||
results.add(envelope == null
|
||||
? EnvelopeIngestResult.invalid(ex.getMessage())
|
||||
: EnvelopeIngestResult.skipped(envelope.getEventId(), envelope.getVin(), ex.getMessage()));
|
||||
}
|
||||
}
|
||||
if (valid.isEmpty()) {
|
||||
return List.copyOf(results);
|
||||
}
|
||||
try {
|
||||
store.appendAll(valid.stream().map(BatchEntry::record).toList());
|
||||
writeTdengineFacts(valid.stream().map(BatchEntry::envelope).toList());
|
||||
for (BatchEntry entry : valid) {
|
||||
results.set(entry.index(), EnvelopeIngestResult.stored(
|
||||
entry.record().eventId(),
|
||||
entry.record().vin()));
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
for (BatchEntry entry : valid) {
|
||||
results.set(entry.index(), EnvelopeIngestResult.failed(
|
||||
entry.envelope().getEventId(),
|
||||
entry.envelope().getVin(),
|
||||
ex.getMessage()));
|
||||
}
|
||||
}
|
||||
return List.copyOf(results);
|
||||
}
|
||||
|
||||
private record BatchEntry(int index, VehicleEnvelope envelope, EventFileRecord record) {
|
||||
}
|
||||
|
||||
private static VehicleEnvelope parse(byte[] kafkaValue) {
|
||||
@@ -84,19 +112,25 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
}
|
||||
}
|
||||
|
||||
private void writeTdengineFacts(VehicleEnvelope envelope) throws IOException {
|
||||
private void writeTdengineFacts(List<VehicleEnvelope> envelopes) throws IOException {
|
||||
if (tdengineWriter == null) {
|
||||
return;
|
||||
}
|
||||
var rawFrame = TdengineEnvelopeRows.rawFrame(envelope);
|
||||
if (rawFrame.isPresent()) {
|
||||
tdengineWriter.appendRawFrames(List.of(rawFrame.get()));
|
||||
var rawFrames = envelopes.stream()
|
||||
.flatMap(envelope -> TdengineEnvelopeRows.rawFrame(envelope).stream())
|
||||
.toList();
|
||||
if (!rawFrames.isEmpty()) {
|
||||
tdengineWriter.appendRawFrames(rawFrames);
|
||||
}
|
||||
var location = TdengineEnvelopeRows.location(envelope);
|
||||
if (location.isPresent()) {
|
||||
tdengineWriter.appendLocations(List.of(location.get()));
|
||||
var locations = envelopes.stream()
|
||||
.flatMap(envelope -> TdengineEnvelopeRows.location(envelope).stream())
|
||||
.toList();
|
||||
if (!locations.isEmpty()) {
|
||||
tdengineWriter.appendLocations(locations);
|
||||
}
|
||||
var telemetryFields = TdengineEnvelopeRows.telemetryFields(envelope);
|
||||
var telemetryFields = envelopes.stream()
|
||||
.flatMap(envelope -> TdengineEnvelopeRows.telemetryFields(envelope).stream())
|
||||
.toList();
|
||||
if (!telemetryFields.isEmpty()) {
|
||||
tdengineWriter.appendTelemetryFields(telemetryFields);
|
||||
}
|
||||
|
||||
@@ -186,6 +186,38 @@ class EventHistoryEnvelopeIngestorTest {
|
||||
assertThat(tdengineWriter.telemetryFields.getFirst().valueDouble()).isEqualTo(42.5);
|
||||
}
|
||||
|
||||
@Test
|
||||
void tryIngestAllBatchesFileStoreAndTdengineWrites() throws Exception {
|
||||
CapturingStore store = new CapturingStore();
|
||||
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
|
||||
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
|
||||
store, new TelemetryEnvelopeRecordMapper(), tdengineWriter);
|
||||
|
||||
VehicleEnvelope first = jt808LocationEnvelope("jt808-location-1", "frame-jt808-1", "013800000001");
|
||||
VehicleEnvelope second = jt808LocationEnvelope("jt808-location-2", "frame-jt808-2", "013800000002");
|
||||
|
||||
List<EnvelopeIngestResult> results = ingestor.tryIngestAll(List.of(
|
||||
first.toByteArray(),
|
||||
new byte[]{0x01, 0x02},
|
||||
second.toByteArray()));
|
||||
|
||||
assertThat(results).extracting(EnvelopeIngestResult::status)
|
||||
.containsExactly(
|
||||
EnvelopeIngestResult.Status.STORED,
|
||||
EnvelopeIngestResult.Status.INVALID_ENVELOPE,
|
||||
EnvelopeIngestResult.Status.STORED);
|
||||
assertThat(store.appendAllCalls).isEqualTo(1);
|
||||
assertThat(store.records).extracting(EventFileRecord::eventId)
|
||||
.containsExactly("jt808-location-1", "jt808-location-2");
|
||||
assertThat(tdengineWriter.rawFrames).extracting(TdengineRawFrameRow::frameId)
|
||||
.containsExactly("frame-jt808-1", "frame-jt808-2");
|
||||
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
|
||||
.containsExactly("jt808-location-1", "jt808-location-2");
|
||||
assertThat(tdengineWriter.telemetryFields)
|
||||
.extracting(TdengineTelemetryFieldRow::fieldKey)
|
||||
.containsExactly("location.speedKmh", "location.speedKmh");
|
||||
}
|
||||
|
||||
private static VehicleEnvelope envelope(String eventId) {
|
||||
return VehicleEnvelope.newBuilder()
|
||||
.setEventId(eventId)
|
||||
@@ -204,11 +236,52 @@ class EventHistoryEnvelopeIngestorTest {
|
||||
.build();
|
||||
}
|
||||
|
||||
private static VehicleEnvelope jt808LocationEnvelope(String eventId, String frameId, String phone) {
|
||||
return VehicleEnvelope.newBuilder()
|
||||
.setSchemaVersion("1.0")
|
||||
.setEventId(eventId)
|
||||
.setVin("VINJT808001")
|
||||
.setSource("JT808")
|
||||
.setEventTimeMs(1_782_112_400_000L)
|
||||
.setIngestTimeMs(1_782_112_401_000L)
|
||||
.putMetadata("vehicle_key", "jt808:" + phone)
|
||||
.putMetadata("frame_id", frameId)
|
||||
.setRawArchive(RawArchiveRef.newBuilder()
|
||||
.setUri("archive://jt808/2026/06/29/" + frameId + ".bin")
|
||||
.setSizeBytes(68))
|
||||
.setRawFrameFact(RawFrameFactPayload.newBuilder()
|
||||
.setFrameId(frameId)
|
||||
.setVehicleKey("jt808:" + phone)
|
||||
.setVin("VINJT808001")
|
||||
.setPhone(phone)
|
||||
.setMessageId(0x0200)
|
||||
.setRawUri("archive://jt808/2026/06/29/" + frameId + ".bin")
|
||||
.setRawSizeBytes(68)
|
||||
.setParseStatus(ParseStatusProto.PARSE_STATUS_SUCCEEDED))
|
||||
.setLocation(LocationPayload.newBuilder()
|
||||
.setLongitude(113.12)
|
||||
.setLatitude(23.45)
|
||||
.setSpeedKmh(42.5))
|
||||
.setTelemetrySnapshot(TelemetrySnapshot.newBuilder()
|
||||
.setEventType("LOCATION")
|
||||
.setRawArchiveUri("archive://jt808/2026/06/29/" + frameId + ".bin")
|
||||
.addFields(TelemetryField.newBuilder()
|
||||
.setKey("location.speedKmh")
|
||||
.setValueType("DOUBLE")
|
||||
.setValue("42.5")
|
||||
.setUnit("km/h")
|
||||
.setQuality("GOOD")
|
||||
.setSourcePath("JT808.0x0200.speed")))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static final class CapturingStore implements EventFileStore {
|
||||
private final List<EventFileRecord> records = new ArrayList<>();
|
||||
private int appendAllCalls;
|
||||
|
||||
@Override
|
||||
public void appendAll(List<EventFileRecord> records) {
|
||||
appendAllCalls++;
|
||||
this.records.addAll(records);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user