fix: harden jt808 tdengine ingestion

2026-06-29 19:53:08 +08:00
parent d1748fcc2f
commit 12de83e37f
17 changed files with 588 additions and 189 deletions
--- a/modules/apps/vehicle-history-app/src/main/java/com/lingniu/ingest/historyapp/VehicleHistoryKafkaConsumerConfiguration.java
+++ b/modules/apps/vehicle-history-app/src/main/java/com/lingniu/ingest/historyapp/VehicleHistoryKafkaConsumerConfiguration.java
@@ -11,6 +11,7 @@ import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerWorker;
 import com.lingniu.ingest.sink.mq.SinkMqProperties;
 import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -32,10 +33,17 @@ public class VehicleHistoryKafkaConsumerConfiguration {

    @Bean
    @ConditionalOnMissingBean
-    public EventHistoryEnvelopeIngestor eventHistoryEnvelopeIngestor(EventFileStore store,
+    public EventHistoryEnvelopeIngestor eventHistoryEnvelopeIngestor(ObjectProvider<EventFileStore> store,
                                                                     TelemetryEnvelopeRecordMapper mapper,
-                                                                     ObjectProvider<TdengineHistoryWriter> writer) {
-        return new EventHistoryEnvelopeIngestor(store, mapper, writer.getIfAvailable());
+                                                                     ObjectProvider<TdengineHistoryWriter> writer,
+                                                                     @Value("${lingniu.ingest.tdengine-history.telemetry-fields-enabled:false}")
+                                                                     boolean telemetryFieldsEnabled) {
+        TdengineHistoryWriter tdengineWriter = writer.getIfAvailable();
+        EventFileStore eventFileStore = store.getIfAvailable();
+        if (eventFileStore == null) {
+            return new EventHistoryEnvelopeIngestor(mapper, tdengineWriter, telemetryFieldsEnabled);
+        }
+        return new EventHistoryEnvelopeIngestor(eventFileStore, mapper, tdengineWriter, telemetryFieldsEnabled);
    }

    @Bean
@@ -62,7 +70,11 @@ public class VehicleHistoryKafkaConsumerConfiguration {
        List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(
                Map.of(
                        "eventHistoryEnvelopeConsumerProcessor", processor,
-                        "eventHistoryRawEnvelopeConsumerProcessor", rawProcessor),
+                        "eventHistoryGb32960EnvelopeConsumerProcessor", processor,
+                        "eventHistoryJt808EnvelopeConsumerProcessor", processor,
+                        "eventHistoryRawEnvelopeConsumerProcessor", rawProcessor,
+                        "eventHistoryGb32960RawEnvelopeConsumerProcessor", rawProcessor,
+                        "eventHistoryJt808RawEnvelopeConsumerProcessor", rawProcessor),
                props);
        if (workers.isEmpty()) {
            throw new IllegalStateException("no vehicle history kafka consumer workers created; check consumer bindings");
--- a/modules/apps/vehicle-history-app/src/main/resources/application.yml
+++ b/modules/apps/vehicle-history-app/src/main/resources/application.yml
@@ -48,25 +48,34 @@ lingniu:
          auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
          max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}
          max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}
+          concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3}
          bindings:
-            eventHistoryEnvelopeConsumerProcessor:
+            eventHistoryGb32960EnvelopeConsumerProcessor:
              enabled: true
-              group-id: ${KAFKA_GROUP_HISTORY_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-event}
+              group-id: ${KAFKA_GROUP_HISTORY_GB32960_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-event}
              topics:
                - ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
-                - ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
-            eventHistoryRawEnvelopeConsumerProcessor:
+            eventHistoryJt808EnvelopeConsumerProcessor:
              enabled: true
-              group-id: ${KAFKA_GROUP_HISTORY_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}}
+              group-id: ${KAFKA_GROUP_HISTORY_JT808_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-event}
+              topics:
+                - ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
+            eventHistoryGb32960RawEnvelopeConsumerProcessor:
+              enabled: true
+              group-id: ${KAFKA_GROUP_HISTORY_GB32960_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-raw}
              topics:
                - ${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}
+            eventHistoryJt808RawEnvelopeConsumerProcessor:
+              enabled: true
+              group-id: ${KAFKA_GROUP_HISTORY_JT808_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-raw}
+              topics:
                - ${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}
      archive:
        enabled: ${SINK_ARCHIVE_ENABLED:true}
        type: local
        path: ${SINK_ARCHIVE_PATH:./archive/}
    event-file-store:
-      enabled: ${EVENT_FILE_STORE_ENABLED:true}
+      enabled: ${EVENT_FILE_STORE_ENABLED:false}
      path: ${EVENT_FILE_STORE_PATH:./target/event-store/}
      zone-id: ${EVENT_FILE_STORE_ZONE_ID:Asia/Shanghai}
      batch-size: ${EVENT_FILE_STORE_BATCH_SIZE:500}
@@ -82,6 +91,7 @@ lingniu:
      minimum-idle: ${TDENGINE_MIN_IDLE:0}
      connection-timeout-millis: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:5000}
      initialization-fail-timeout-millis: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:-1}
+      telemetry-fields-enabled: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:false}
    event-history:
      enabled: true
    vehicle-state:
--- a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppCompositionTest.java
+++ b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppCompositionTest.java
@@ -3,6 +3,7 @@ package com.lingniu.ingest.historyapp;
 import com.lingniu.ingest.eventfilestore.EventFileStore;
 import com.lingniu.ingest.eventfilestore.EventFileStoreSink;
 import com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration;
+import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
 import com.lingniu.ingest.eventhistory.EventHistoryController;
 import com.lingniu.ingest.eventhistory.EventHistoryEnvelopeIngestor;
 import com.lingniu.ingest.eventhistory.Gb32960DecodedFrameService;
@@ -20,6 +21,7 @@ import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
 import com.lingniu.ingest.sink.mq.KafkaEventSink;
 import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
 import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema;
+import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
 import com.lingniu.ingest.tdenginehistory.config.TdengineHistoryAutoConfiguration;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.junit.jupiter.api.Test;
@@ -86,6 +88,21 @@ class VehicleHistoryAppCompositionTest {
                });
    }

+    @Test
+    void createsHistoryIngestorWithoutEventFileStoreForTdengineOnlyRuntime() {
+        new ApplicationContextRunner()
+                .withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
+                .withBean(TdengineHistoryWriter.class, () -> mock(TdengineHistoryWriter.class))
+                .withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
+                .withPropertyValues(
+                        "lingniu.ingest.event-file-store.enabled=false",
+                        "lingniu.ingest.sink.mq.consumer.enabled=false")
+                .run(context -> {
+                    assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
+                    assertThat(context).doesNotHaveBean(EventFileStore.class);
+                });
+    }
+
    @SuppressWarnings("unchecked")
    private static KafkaProducer<String, byte[]> kafkaProducer() {
        return mock(KafkaProducer.class);
--- a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java
+++ b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java
@@ -23,7 +23,7 @@ class VehicleHistoryAppDefaultsTest {
                .containsEntry("lingniu.ingest.gb32960.enabled", true)
                .containsEntry("lingniu.ingest.gb32960.server.enabled", false)
                .containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
-                .containsEntry("lingniu.ingest.event-file-store.enabled", "${EVENT_FILE_STORE_ENABLED:true}")
+                .containsEntry("lingniu.ingest.event-file-store.enabled", "${EVENT_FILE_STORE_ENABLED:false}")
                .containsEntry("lingniu.ingest.tdengine-history.enabled", "${TDENGINE_HISTORY_ENABLED:false}")
                .containsEntry("lingniu.ingest.tdengine-history.database", "${TDENGINE_HISTORY_DATABASE:vehicle_history}")
                .containsEntry(
@@ -36,37 +36,53 @@ class VehicleHistoryAppDefaultsTest {
                        "${TDENGINE_DRIVER_CLASS_NAME:com.taosdata.jdbc.rs.RestfulDriver}")
                .containsEntry("lingniu.ingest.tdengine-history.maximum-pool-size", "${TDENGINE_MAX_POOL_SIZE:16}")
                .containsEntry("lingniu.ingest.tdengine-history.minimum-idle", "${TDENGINE_MIN_IDLE:0}")
+                .containsEntry(
+                        "lingniu.ingest.tdengine-history.telemetry-fields-enabled",
+                        "${TDENGINE_TELEMETRY_FIELDS_ENABLED:false}")
                .containsEntry("lingniu.ingest.event-history.enabled", true)
                .containsEntry("lingniu.ingest.vehicle-state.enabled", false)
                .containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
                .containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
                .containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}")
+                .containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
                .containsEntry(
                        "lingniu.ingest.sink.mq.consumer.max-poll-interval-millis",
                        "${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}")
                .containsEntry(
-                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.enabled",
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled",
                        true)
                .containsEntry(
-                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id",
-                        "${KAFKA_GROUP_HISTORY_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-event}")
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id",
+                        "${KAFKA_GROUP_HISTORY_GB32960_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-event}")
                .containsEntry(
-                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.topics[0]",
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]",
                        "${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}")
                .containsEntry(
-                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.topics[1]",
-                        "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
-                .containsEntry(
-                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryRawEnvelopeConsumerProcessor.enabled",
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.enabled",
                        true)
                .containsEntry(
-                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryRawEnvelopeConsumerProcessor.group-id",
-                        "${KAFKA_GROUP_HISTORY_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}}")
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.group-id",
+                        "${KAFKA_GROUP_HISTORY_JT808_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-event}")
                .containsEntry(
-                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryRawEnvelopeConsumerProcessor.topics[0]",
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.topics[0]",
+                        "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
+                .containsEntry(
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.enabled",
+                        true)
+                .containsEntry(
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.group-id",
+                        "${KAFKA_GROUP_HISTORY_GB32960_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-raw}")
+                .containsEntry(
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.topics[0]",
                        "${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}")
                .containsEntry(
-                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryRawEnvelopeConsumerProcessor.topics[1]",
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.enabled",
+                        true)
+                .containsEntry(
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.group-id",
+                        "${KAFKA_GROUP_HISTORY_JT808_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-raw}")
+                .containsEntry(
+                        "lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.topics[0]",
                        "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}");
    }