shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: stabilize raw frame history queries

Browse Source
This commit is contained in:
lingniu
2026-07-01 00:44:52 +08:00
parent 235cc46bf8
commit 519f5591e9
4 changed files with 68 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
deploy/portainer/docker-compose.yml
View File

@@ -162,7 +162,7 @@ services:
KAFKA_GROUP_HISTORY_XINDA: ${KAFKA_GROUP_HISTORY_XINDA:-vehicle-history}
SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true}
SINK_ARCHIVE_PATH: /archive/
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true}
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-false}
EVENT_FILE_STORE_PATH: /event-store/
EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai}
EVENT_FILE_STORE_BATCH_SIZE: ${EVENT_FILE_STORE_BATCH_SIZE:-500}
@@ -174,6 +174,9 @@ services:
TDENGINE_JT808_LOCATION_STABLE: ${TDENGINE_JT808_LOCATION_STABLE:-jt808_locations}
TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root}
TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata}
TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-30000}
TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-1}
TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32}
ports:
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
volumes:

10
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java
View File

@@ -8,6 +8,8 @@ import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
@@ -22,6 +24,8 @@ import java.util.List;
*/
public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor {
private static final Logger log = LoggerFactory.getLogger(EventHistoryEnvelopeIngestor.class);
private final EventFileStore store;
private final TelemetryEnvelopeRecordMapper mapper;
private final TdengineHistoryWriter tdengineWriter;
@@ -122,6 +126,12 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor
entry.vin()));
}
} catch (IOException ex) {
log.error("event history TDengine/file-store ingest failed batchSize={} validSize={} firstEventId={} firstVin={}",
kafkaValues.size(),
valid.size(),
valid.getFirst().envelope().getEventId(),
valid.getFirst().envelope().getVin(),
ex);
for (BatchEntry entry : valid) {
results.set(entry.index(), EnvelopeIngestResult.failed(
entry.envelope().getEventId(),

9
modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java
View File

@@ -24,6 +24,7 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.time.Instant;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -252,7 +253,7 @@ public final class RawFrameHistoryController {
out.putAll(parsed);
}
if (!includeParsedFields) {
return Map.copyOf(out);
return immutableMap(out);
}
for (TdengineLocationRow location : reader.queryLocationsByRawUri(
row.protocol(), row.rawUri(), LOCATION_ENRICH_LIMIT)) {
@@ -266,7 +267,7 @@ public final class RawFrameHistoryController {
out.put(field.fieldKey() + "._unit", field.unit());
}
}
return Map.copyOf(out);
return immutableMap(out);
}
private static Map<String, Object> parsed(String parsedJson) {
@@ -302,6 +303,10 @@ public final class RawFrameHistoryController {
}
return field.valueText();
}
private static Map<String, Object> immutableMap(Map<String, Object> values) {
return Collections.unmodifiableMap(new LinkedHashMap<>(values));
}
}
}

47
modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java
View File

@@ -18,6 +18,7 @@ import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
@@ -142,6 +143,52 @@ class RawFrameHistoryControllerTest {
.containsEntry("VEHICLE.speedKmh", 25.0);
}
@Test
void preservesNullValuesFromParsedJsonWithoutFailingResponseMapping() throws Exception {
Instant ts = Instant.parse("2026-06-29T12:00:01Z");
TdengineRawFrameRow raw = new TdengineRawFrameRow(
ts,
"frame-with-null-json",
ts.plusMillis(500),
0x0002,
0,
ts,
"archive://2026/06/29/GB32960/LB9A32A24P0LS1270/frame-with-null-json.bin",
"sha256:null-json",
128,
"NOT_PARSED",
"",
"8.134.95.166:49706",
"{\"rawArchiveUri\":\"archive://2026/06/29/GB32960/LB9A32A24P0LS1270/frame-with-null-json.bin\"}",
"{\"header\":{\"command\":\"REALTIME_REPORT\"},\"commandBody\":null,\"signature\":null}",
"GB32960",
"LB9A32A24P0LS1270",
"LB9A32A24P0LS1270",
"");
RawFrameHistoryController controller = new RawFrameHistoryController(
new CapturingReader(new TdenginePage<>(List.of(raw), Optional.empty())));
RawFrameHistoryController.RawFramePageResponse response = controller.rawFrames(
"gb32960",
null,
"LB9A32A24P0LS1270",
null,
null,
"not_parsed",
"2026-06-29T00:00:00+08:00",
"2026-06-29T23:59:59+08:00",
TdengineQueryOrder.DESC,
20,
false,
null,
null);
Map<String, Object> parsedFields = response.items().getFirst().parsedFields();
assertThat(parsedFields).containsKeys("header", "commandBody", "signature");
assertThat(parsedFields.get("commandBody")).isNull();
assertThat(parsedFields.get("signature")).isNull();
}
@Test
void rejectsRawFrameQueryWithoutAnyNarrowingFilter() {
RawFrameHistoryController controller = new RawFrameHistoryController(