fix: harden jt808 tdengine ingestion
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
This commit is contained in:
@@ -25,16 +25,45 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
private final EventFileStore store;
|
||||
private final TelemetryEnvelopeRecordMapper mapper;
|
||||
private final TdengineHistoryWriter tdengineWriter;
|
||||
private final boolean telemetryFieldsEnabled;
|
||||
|
||||
public EventHistoryEnvelopeIngestor(EventFileStore store, TelemetryEnvelopeRecordMapper mapper) {
|
||||
this(store, mapper, null);
|
||||
}
|
||||
|
||||
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
|
||||
TdengineHistoryWriter tdengineWriter) {
|
||||
this(mapper, tdengineWriter, true);
|
||||
}
|
||||
|
||||
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
|
||||
TdengineHistoryWriter tdengineWriter,
|
||||
boolean telemetryFieldsEnabled) {
|
||||
this(null, mapper, tdengineWriter, false, telemetryFieldsEnabled);
|
||||
}
|
||||
|
||||
public EventHistoryEnvelopeIngestor(EventFileStore store,
|
||||
TelemetryEnvelopeRecordMapper mapper,
|
||||
TdengineHistoryWriter tdengineWriter) {
|
||||
this(store, mapper, tdengineWriter, true, true);
|
||||
}
|
||||
|
||||
public EventHistoryEnvelopeIngestor(EventFileStore store,
|
||||
TelemetryEnvelopeRecordMapper mapper,
|
||||
TdengineHistoryWriter tdengineWriter,
|
||||
boolean telemetryFieldsEnabled) {
|
||||
this(store, mapper, tdengineWriter, true, telemetryFieldsEnabled);
|
||||
}
|
||||
|
||||
private EventHistoryEnvelopeIngestor(EventFileStore store,
|
||||
TelemetryEnvelopeRecordMapper mapper,
|
||||
TdengineHistoryWriter tdengineWriter,
|
||||
boolean requireStore,
|
||||
boolean telemetryFieldsEnabled) {
|
||||
if (store == null) {
|
||||
throw new IllegalArgumentException("store must not be null");
|
||||
if (requireStore) {
|
||||
throw new IllegalArgumentException("store must not be null");
|
||||
}
|
||||
}
|
||||
if (mapper == null) {
|
||||
throw new IllegalArgumentException("mapper must not be null");
|
||||
@@ -42,12 +71,15 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
this.store = store;
|
||||
this.mapper = mapper;
|
||||
this.tdengineWriter = tdengineWriter;
|
||||
this.telemetryFieldsEnabled = telemetryFieldsEnabled;
|
||||
}
|
||||
|
||||
public void ingest(byte[] kafkaValue) throws IOException {
|
||||
VehicleEnvelope envelope = parse(kafkaValue);
|
||||
EventFileRecord record = mapper.toRecord(envelope);
|
||||
store.append(record);
|
||||
if (store != null) {
|
||||
EventFileRecord record = mapper.toRecord(envelope);
|
||||
store.append(record);
|
||||
}
|
||||
writeTdengineFacts(List.of(envelope));
|
||||
}
|
||||
|
||||
@@ -67,7 +99,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
VehicleEnvelope envelope = null;
|
||||
try {
|
||||
envelope = parse(kafkaValue);
|
||||
EventFileRecord record = mapper.toRecord(envelope);
|
||||
EventFileRecord record = store == null ? null : mapper.toRecord(envelope);
|
||||
results.add(null);
|
||||
valid.add(new BatchEntry(results.size() - 1, envelope, record));
|
||||
} catch (IllegalArgumentException ex) {
|
||||
@@ -80,12 +112,14 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
return List.copyOf(results);
|
||||
}
|
||||
try {
|
||||
store.appendAll(valid.stream().map(BatchEntry::record).toList());
|
||||
if (store != null) {
|
||||
store.appendAll(valid.stream().map(BatchEntry::record).toList());
|
||||
}
|
||||
writeTdengineFacts(valid.stream().map(BatchEntry::envelope).toList());
|
||||
for (BatchEntry entry : valid) {
|
||||
results.set(entry.index(), EnvelopeIngestResult.stored(
|
||||
entry.record().eventId(),
|
||||
entry.record().vin()));
|
||||
entry.eventId(),
|
||||
entry.vin()));
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
for (BatchEntry entry : valid) {
|
||||
@@ -99,6 +133,13 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
}
|
||||
|
||||
private record BatchEntry(int index, VehicleEnvelope envelope, EventFileRecord record) {
|
||||
private String eventId() {
|
||||
return record == null ? envelope.getEventId() : record.eventId();
|
||||
}
|
||||
|
||||
private String vin() {
|
||||
return record == null ? envelope.getVin() : record.vin();
|
||||
}
|
||||
}
|
||||
|
||||
private static VehicleEnvelope parse(byte[] kafkaValue) {
|
||||
@@ -128,11 +169,13 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
if (!locations.isEmpty()) {
|
||||
tdengineWriter.appendLocations(locations);
|
||||
}
|
||||
var telemetryFields = envelopes.stream()
|
||||
.flatMap(envelope -> TdengineEnvelopeRows.telemetryFields(envelope).stream())
|
||||
.toList();
|
||||
if (!telemetryFields.isEmpty()) {
|
||||
tdengineWriter.appendTelemetryFields(telemetryFields);
|
||||
if (telemetryFieldsEnabled) {
|
||||
var telemetryFields = envelopes.stream()
|
||||
.flatMap(envelope -> TdengineEnvelopeRows.telemetryFields(envelope).stream())
|
||||
.toList();
|
||||
if (!telemetryFields.isEmpty()) {
|
||||
tdengineWriter.appendTelemetryFields(telemetryFields);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user