shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: separate kafka consumer dlq topic

Browse Source
This commit is contained in:
lingniu
2026-07-01 16:30:51 +08:00
parent 5deadd06af
commit 6ac77b2f7d
10 changed files with 49 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
deploy/portainer/docker-compose.yml
View File

@@ -160,6 +160,7 @@ services:
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_HISTORY_DLQ: ${KAFKA_TOPIC_HISTORY_DLQ:-vehicle.dlq.history.v1}
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts}

4
docs/operations/gb32960-service-split-runbook.md
View File

@@ -25,6 +25,7 @@ This runbook covers the default production runtimes: GB32960 ingest, JT808 inges
| `vehicle.raw.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT raw payload records for TDengine `raw_frames` and troubleshooting APIs. |
| `vehicle.event.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT parsed telemetry events keyed by mapped VIN or device identity. |
| `vehicle.dlq.mqtt-yutong.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed Yutong MQTT production or consumer processing. |
| `vehicle.dlq.history.v1` | History app consumer error paths | Operators/replay tooling | Dead-letter records for failed history storage processing. |
Default local consumer groups:
@@ -84,12 +85,13 @@ kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.mqtt-yutong.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.mqtt-yutong.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.mqtt-yutong.v1 --partitions 3 --replication-factor 1
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.history.v1 --partitions 3 --replication-factor 1
```
Optional sanity check:
```bash
kafka-topics --bootstrap-server 127.0.0.1:9092 --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808|mqtt-yutong)\.v1'
kafka-topics --bootstrap-server 127.0.0.1:9092 --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808|mqtt-yutong|history)\.v1'
```
## Start Split Services Locally

4
modules/apps/vehicle-analytics-app/src/main/resources/application.yml
View File

@@ -38,10 +38,8 @@ lingniu:
kafka:
enabled: true
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
consumer:
dlq-topic: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
enabled: ${KAFKA_CONSUMER_ENABLED:true}
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}

3
modules/apps/vehicle-analytics-app/src/test/java/com/lingniu/ingest/analyticsapp/VehicleAnalyticsAppDefaultsTest.java
View File

@@ -28,6 +28,8 @@ class VehicleAnalyticsAppDefaultsTest {
"${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.dlq-topic",
"${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry(
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
true)
@@ -40,6 +42,7 @@ class VehicleAnalyticsAppDefaultsTest {
.noneMatch(name -> name.startsWith("lingniu.ingest.gb32960."))
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
.noneMatch(name -> name.startsWith("lingniu.ingest.sink.archive."))
.noneMatch(name -> name.startsWith("lingniu.ingest.sink.kafka.topics."))
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
.noneMatch(name -> name.contains("vehicleStateEnvelopeConsumerProcessor"))

1
modules/apps/vehicle-history-app/src/main/resources/application.yml
View File

@@ -34,6 +34,7 @@ lingniu:
enabled: true
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
consumer:
dlq-topic: ${KAFKA_TOPIC_HISTORY_DLQ:vehicle.dlq.history.v1}
enabled: ${KAFKA_CONSUMER_ENABLED:true}
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}

2
modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java
View File

@@ -369,6 +369,7 @@ class PortainerComposeResourceLimitsTest {
.contains("KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}")
.contains("KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}")
.contains("KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}")
.contains("KAFKA_TOPIC_HISTORY_DLQ: ${KAFKA_TOPIC_HISTORY_DLQ:-vehicle.dlq.history.v1}")
.doesNotContain("KAFKA_TOPIC_GB32960_DLQ")
.doesNotContain("KAFKA_TOPIC_JT808_DLQ")
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT_DLQ");
@@ -493,6 +494,7 @@ class PortainerComposeResourceLimitsTest {
.contains("`vehicle.raw.mqtt-yutong.v1`")
.contains("`vehicle.event.mqtt-yutong.v1`")
.contains("`vehicle.dlq.mqtt-yutong.v1`")
.contains("`vehicle.dlq.history.v1`")
.contains(activeBuildCommand)
.contains("modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar")
.doesNotContain("20482")

2
modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java
View File

@@ -40,6 +40,8 @@ class VehicleHistoryAppDefaultsTest {
.containsEntry("lingniu.ingest.event-history.enabled", true)
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.dlq-topic",
"${KAFKA_TOPIC_HISTORY_DLQ:vehicle.dlq.history.v1}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
.containsEntry(

2
modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkAutoConfiguration.java
View File

@@ -80,7 +80,7 @@ public class KafkaSinkAutoConfiguration {
@ConditionalOnMissingBean
public KafkaEnvelopeDeadLetterSink kafkaEnvelopeDeadLetterSink(KafkaProducer<String, byte[]> producer,
KafkaSinkProperties props) {
return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq());
return new KafkaEnvelopeDeadLetterSink(producer, props.deadLetterTopic());
}
}

12
modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java
View File

@@ -51,6 +51,14 @@ public class KafkaSinkProperties {
public Consumer getConsumer() { return consumer; }
public void setConsumer(Consumer consumer) { this.consumer = consumer; }
public String deadLetterTopic() {
String consumerDlq = consumer == null ? "" : consumer.getDlqTopic();
if (consumerDlq != null && !consumerDlq.isBlank()) {
return consumerDlq;
}
return topics.getDlq();
}
public static class Topics {
/** 实时遥测事件 topic；GB32960 RAW-only 架构下可逐步弱化该 topic。 */
private String realtime = "vehicle.event.gb32960.v1";
@@ -91,6 +99,8 @@ public class KafkaSinkProperties {
private int maxPollRecords = 500;
private int maxPollIntervalMillis = 1800000;
private int concurrency = 1;
/** 消费处理失败时的死信 topic；未配置时兼容使用 producer topics.dlq。 */
private String dlqTopic = "";
/**
* Processor bean name -> Kafka binding。
*
@@ -117,6 +127,8 @@ public class KafkaSinkProperties {
public void setMaxPollIntervalMillis(int maxPollIntervalMillis) { this.maxPollIntervalMillis = maxPollIntervalMillis; }
public int getConcurrency() { return concurrency; }
public void setConcurrency(int concurrency) { this.concurrency = concurrency; }
public String getDlqTopic() { return dlqTopic; }
public void setDlqTopic(String dlqTopic) { this.dlqTopic = dlqTopic; }
public Map<String, Binding> getBindings() { return bindings; }
public void setBindings(Map<String, Binding> bindings) { this.bindings = bindings; }
}

23
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaSinkConsumerAutoConfigurationTest.java
View File

@@ -11,6 +11,8 @@ import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.boot.test.system.CapturedOutput;
import org.springframework.boot.test.system.OutputCaptureExtension;
import java.lang.reflect.Field;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
@@ -44,8 +46,29 @@ class KafkaSinkConsumerAutoConfigurationTest {
assertThat(output).doesNotContain("114.55.58.251");
}
@Test
void consumerDlqTopicOverridesProducerTopicDefaultsForDeadLetterSink() {
contextRunner
.withPropertyValues("lingniu.ingest.sink.kafka.consumer.dlq-topic=vehicle.dlq.consumer.v1")
.run(context -> {
assertThat(context).hasSingleBean(KafkaEnvelopeDeadLetterSink.class);
assertThat(deadLetterTopic(context.getBean(KafkaEnvelopeDeadLetterSink.class)))
.isEqualTo("vehicle.dlq.consumer.v1");
});
}
@SuppressWarnings("unchecked")
private static KafkaProducer<String, byte[]> kafkaProducer() {
return mock(KafkaProducer.class);
}
private static String deadLetterTopic(KafkaEnvelopeDeadLetterSink sink) {
try {
Field field = KafkaEnvelopeDeadLetterSink.class.getDeclaredField("topic");
field.setAccessible(true);
return (String) field.get(sink);
} catch (ReflectiveOperationException e) {
throw new AssertionError(e);
}
}
}