diff --git a/deploy/portainer/docker-compose.yml b/deploy/portainer/docker-compose.yml index 9bece17a..4ec591e8 100644 --- a/deploy/portainer/docker-compose.yml +++ b/deploy/portainer/docker-compose.yml @@ -162,7 +162,7 @@ services: KAFKA_GROUP_HISTORY_XINDA: ${KAFKA_GROUP_HISTORY_XINDA:-vehicle-history} SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true} SINK_ARCHIVE_PATH: /archive/ - EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true} + EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-false} EVENT_FILE_STORE_PATH: /event-store/ EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai} EVENT_FILE_STORE_BATCH_SIZE: ${EVENT_FILE_STORE_BATCH_SIZE:-500} @@ -174,6 +174,9 @@ services: TDENGINE_JT808_LOCATION_STABLE: ${TDENGINE_JT808_LOCATION_STABLE:-jt808_locations} TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root} TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata} + TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-30000} + TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-1} + TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32} ports: - "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200" volumes: diff --git a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java index 7d89ea89..7d53379e 100644 --- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java +++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java @@ -8,6 +8,8 @@ import com.lingniu.ingest.eventfilestore.EventFileStore; import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows; import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -22,6 +24,8 @@ import java.util.List; */ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor { + private static final Logger log = LoggerFactory.getLogger(EventHistoryEnvelopeIngestor.class); + private final EventFileStore store; private final TelemetryEnvelopeRecordMapper mapper; private final TdengineHistoryWriter tdengineWriter; @@ -122,6 +126,12 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor entry.vin())); } } catch (IOException ex) { + log.error("event history TDengine/file-store ingest failed batchSize={} validSize={} firstEventId={} firstVin={}", + kafkaValues.size(), + valid.size(), + valid.getFirst().envelope().getEventId(), + valid.getFirst().envelope().getVin(), + ex); for (BatchEntry entry : valid) { results.set(entry.index(), EnvelopeIngestResult.failed( entry.envelope().getEventId(), diff --git a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java index 1bf68c50..bd117cac 100644 --- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java +++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java @@ -24,6 +24,7 @@ import org.springframework.web.server.ResponseStatusException; import java.io.IOException; import java.time.Instant; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -252,7 +253,7 @@ public final class RawFrameHistoryController { out.putAll(parsed); } if (!includeParsedFields) { - return Map.copyOf(out); + return immutableMap(out); } for (TdengineLocationRow location : reader.queryLocationsByRawUri( row.protocol(), row.rawUri(), LOCATION_ENRICH_LIMIT)) { @@ -266,7 +267,7 @@ public final class RawFrameHistoryController { out.put(field.fieldKey() + "._unit", field.unit()); } } - return Map.copyOf(out); + return immutableMap(out); } private static Map parsed(String parsedJson) { @@ -302,6 +303,10 @@ public final class RawFrameHistoryController { } return field.valueText(); } + + private static Map immutableMap(Map values) { + return Collections.unmodifiableMap(new LinkedHashMap<>(values)); + } } } diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java index efb2b5fe..bf1e386a 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; @@ -142,6 +143,52 @@ class RawFrameHistoryControllerTest { .containsEntry("VEHICLE.speedKmh", 25.0); } + @Test + void preservesNullValuesFromParsedJsonWithoutFailingResponseMapping() throws Exception { + Instant ts = Instant.parse("2026-06-29T12:00:01Z"); + TdengineRawFrameRow raw = new TdengineRawFrameRow( + ts, + "frame-with-null-json", + ts.plusMillis(500), + 0x0002, + 0, + ts, + "archive://2026/06/29/GB32960/LB9A32A24P0LS1270/frame-with-null-json.bin", + "sha256:null-json", + 128, + "NOT_PARSED", + "", + "8.134.95.166:49706", + "{\"rawArchiveUri\":\"archive://2026/06/29/GB32960/LB9A32A24P0LS1270/frame-with-null-json.bin\"}", + "{\"header\":{\"command\":\"REALTIME_REPORT\"},\"commandBody\":null,\"signature\":null}", + "GB32960", + "LB9A32A24P0LS1270", + "LB9A32A24P0LS1270", + ""); + RawFrameHistoryController controller = new RawFrameHistoryController( + new CapturingReader(new TdenginePage<>(List.of(raw), Optional.empty()))); + + RawFrameHistoryController.RawFramePageResponse response = controller.rawFrames( + "gb32960", + null, + "LB9A32A24P0LS1270", + null, + null, + "not_parsed", + "2026-06-29T00:00:00+08:00", + "2026-06-29T23:59:59+08:00", + TdengineQueryOrder.DESC, + 20, + false, + null, + null); + + Map parsedFields = response.items().getFirst().parsedFields(); + assertThat(parsedFields).containsKeys("header", "commandBody", "signature"); + assertThat(parsedFields.get("commandBody")).isNull(); + assertThat(parsedFields.get("signature")).isNull(); + } + @Test void rejectsRawFrameQueryWithoutAnyNarrowingFilter() { RawFrameHistoryController controller = new RawFrameHistoryController(