shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: expose kafka sink configuration

Browse Source
This commit is contained in:
lingniu
2026-07-01 10:50:50 +08:00
parent 303b2eb47e
commit 849e8f8a8b
67 changed files with 350 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
modules/apps/gb32960-ingest-app/src/main/resources/application.yml
View File

@@ -99,9 +99,8 @@ lingniu:
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd
linger-ms: 20

15
modules/apps/gb32960-ingest-app/src/test/java/com/lingniu/ingest/gb32960app/Gb32960IngestAppCompositionTest.java
View File

@@ -11,9 +11,9 @@ import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -31,7 +31,7 @@ class Gb32960IngestAppCompositionTest {
IngestCoreAutoConfiguration.class,
SessionCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class,
Gb32960AutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true)
@@ -43,10 +43,9 @@ class Gb32960IngestAppCompositionTest {
"lingniu.ingest.gb32960.port=0",
"lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-gb32960",
"lingniu.ingest.event-history.enabled=false");

10
modules/apps/gb32960-ingest-app/src/test/java/com/lingniu/ingest/gb32960app/Gb32960IngestAppDefaultsTest.java
View File

@@ -15,6 +15,8 @@ class Gb32960IngestAppDefaultsTest {
@Test
void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() throws IOException {
Properties properties = applicationProperties();
String legacyKafkaPrefix = "lingniu.ingest.sink." + "mq.";
String kafkaTypeProperty = "lingniu.ingest.sink.kafka." + "type";
assertThat(properties)
.containsEntry("spring.application.name", "gb32960-ingest-app")
@@ -40,8 +42,8 @@ class Gb32960IngestAppDefaultsTest {
"Hyundai")
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[1]",
"YueJin")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
@@ -55,7 +57,9 @@ class Gb32960IngestAppDefaultsTest {
assertThat(properties.stringPropertyNames())
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."));
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
.noneMatch(name -> name.startsWith(legacyKafkaPrefix))
.noneMatch(name -> name.equals(kafkaTypeProperty));
assertThat(applicationYaml())
.doesNotContain("event-file-store:")
.doesNotContain("vehicle-state:")

3
modules/apps/jt808-ingest-app/src/main/resources/application.yml
View File

@@ -63,9 +63,8 @@ lingniu:
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd
linger-ms: 20

15
modules/apps/jt808-ingest-app/src/test/java/com/lingniu/ingest/jt808app/Jt808IngestAppCompositionTest.java
View File

@@ -11,9 +11,9 @@ import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -31,7 +31,7 @@ class Jt808IngestAppCompositionTest {
IngestCoreAutoConfiguration.class,
SessionCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class,
Jt808AutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true)
@@ -42,10 +42,9 @@ class Jt808IngestAppCompositionTest {
"lingniu.ingest.jt808.port=0",
"lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-jt808",
"lingniu.ingest.event-history.enabled=false");

10
modules/apps/jt808-ingest-app/src/test/java/com/lingniu/ingest/jt808app/Jt808IngestAppDefaultsTest.java
View File

@@ -28,11 +28,11 @@ class Jt808IngestAppDefaultsTest {
.containsEntry("server.port", "${HTTP_PORT:20400}")
.containsEntry("lingniu.ingest.jt808.enabled", true)
.containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:808}")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
.containsEntry("lingniu.ingest.identity.mysql.table-name",

3
modules/apps/vehicle-analytics-app/src/main/resources/application.yml
View File

@@ -37,9 +37,8 @@ lingniu:
gb32960:
enabled: false
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}

15
modules/apps/vehicle-analytics-app/src/test/java/com/lingniu/ingest/analyticsapp/VehicleAnalyticsAppCompositionTest.java
View File

@@ -2,9 +2,9 @@ package com.lingniu.ingest.analyticsapp;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import com.lingniu.ingest.vehiclestat.JdbcVehicleStatMetricRepository;
import com.lingniu.ingest.vehiclestat.VehicleStatController;
import com.lingniu.ingest.vehiclestat.VehicleStatEnvelopeIngestor;
@@ -28,17 +28,16 @@ class VehicleAnalyticsAppCompositionTest {
void createsStatsBeansWithoutProtocolListenerOrEventFileStore() {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
VehicleStatAutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true)
.withBean("kafkaProducer", KafkaProducer.class, VehicleAnalyticsAppCompositionTest::kafkaProducer)
.withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class))
.withBean(ObjectMapper.class, ObjectMapper::new)
.withPropertyValues(
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=true",
"lingniu.ingest.vehicle-stat.jt808.enabled=true",
"lingniu.ingest.event-history.enabled=false",

6
modules/apps/vehicle-analytics-app/src/test/java/com/lingniu/ingest/analyticsapp/VehicleAnalyticsAppDefaultsTest.java
View File

@@ -29,12 +29,12 @@ class VehicleAnalyticsAppDefaultsTest {
.containsEntry(
"lingniu.ingest.vehicle-stat.jt808.enabled",
"${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
"${VEHICLE_STAT_ENABLED:true}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_STAT:vehicle-stat}")
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");

12
modules/apps/vehicle-history-app/src/main/java/com/lingniu/ingest/historyapp/VehicleHistoryKafkaConsumerConfiguration.java
View File

@@ -1,10 +1,10 @@
package com.lingniu.ingest.historyapp;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerFactory;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerWorker;
import com.lingniu.ingest.sink.mq.SinkMqProperties;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerFactory;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerWorker;
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -20,11 +20,11 @@ public class VehicleHistoryKafkaConsumerConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner vehicleHistoryKafkaEnvelopeConsumerRunner(
@Qualifier("eventHistoryEnvelopeConsumerProcessor") EnvelopeConsumerProcessor processor,
@Qualifier("eventHistoryRawEnvelopeConsumerProcessor") EnvelopeConsumerProcessor rawProcessor,
SinkMqProperties props) {
KafkaSinkProperties props) {
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(
Map.of(
"eventHistoryGb32960EnvelopeConsumerProcessor", processor,

3
modules/apps/vehicle-history-app/src/main/resources/application.yml
View File

@@ -34,9 +34,8 @@ lingniu:
server:
enabled: false
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}

58
modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppCompositionTest.java
View File

@@ -15,15 +15,15 @@ import com.lingniu.ingest.eventhistory.TelemetryFieldHistoryController;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.mq.SinkMqProperties;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;
@@ -75,8 +75,8 @@ class VehicleHistoryAppCompositionTest {
() -> processor("event-history"))
.withBean("eventHistoryRawEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
() -> processor("event-history-raw"))
.withBean(SinkMqProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
.withPropertyValues("lingniu.ingest.sink.mq.consumer.enabled=true")
.withBean(KafkaSinkProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
.withPropertyValues("lingniu.ingest.sink.kafka.consumer.enabled=true")
.run(context -> {
assertThat(context).hasFailed();
assertThat(context.getStartupFailure())
@@ -89,7 +89,7 @@ class VehicleHistoryAppCompositionTest {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(
TdengineHistoryAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
Gb32960AutoConfiguration.class,
EventHistoryAutoConfiguration.class))
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
@@ -101,10 +101,9 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.gb32960.enabled=true",
"lingniu.ingest.gb32960.server.enabled=false",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=false")
.run(context -> {
@@ -137,7 +136,7 @@ class VehicleHistoryAppCompositionTest {
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
.withPropertyValues(
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=false")
"lingniu.ingest.sink.kafka.consumer.enabled=false")
.run(context -> {
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context).doesNotHaveBean("eventFileStore");
@@ -149,7 +148,7 @@ class VehicleHistoryAppCompositionTest {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(
TdengineHistoryAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
EventHistoryAutoConfiguration.class))
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
.withAllowBeanDefinitionOverriding(true)
@@ -158,13 +157,12 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.tdengine-history.enabled=true",
"lingniu.ingest.tdengine-history.database=vehicle_history_test",
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=true",
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=true",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
.run(context -> {
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context.getBeanNamesForType(EnvelopeConsumerProcessor.class))
@@ -186,7 +184,7 @@ class VehicleHistoryAppCompositionTest {
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
.withPropertyValues(
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=false")
"lingniu.ingest.sink.kafka.consumer.enabled=false")
.run(context -> {
EventHistoryEnvelopeIngestor ingestor = context.getBean(EventHistoryEnvelopeIngestor.class);
@@ -211,8 +209,8 @@ class VehicleHistoryAppCompositionTest {
ignored -> {});
}
private static SinkMqProperties genericOnlyHistoryConsumerProps() {
SinkMqProperties props = new SinkMqProperties();
private static KafkaSinkProperties genericOnlyHistoryConsumerProps() {
KafkaSinkProperties props = new KafkaSinkProperties();
props.setBootstrapServers("localhost:9092");
props.getConsumer().setEnabled(true);
props.getConsumer().setBindings(Map.of(
@@ -221,8 +219,8 @@ class VehicleHistoryAppCompositionTest {
return props;
}
private static SinkMqProperties.Binding binding(String groupId, String topic) {
SinkMqProperties.Binding binding = new SinkMqProperties.Binding();
private static KafkaSinkProperties.Binding binding(String groupId, String topic) {
KafkaSinkProperties.Binding binding = new KafkaSinkProperties.Binding();
binding.setGroupId(groupId);
binding.setTopics(List.of(topic));
return binding;

44
modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java
View File

@@ -44,65 +44,65 @@ class VehicleHistoryAppDefaultsTest {
.containsEntry("lingniu.ingest.event-history.enabled", true)
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
.containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.max-poll-interval-millis",
"lingniu.ingest.sink.kafka.consumer.max-poll-interval-millis",
"${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled",
true)
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_GB32960_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-event}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.enabled",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.enabled",
true)
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.group-id",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_JT808_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-event}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.topics[0]",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.enabled",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.enabled",
true)
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.group-id",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-event}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.topics[0]",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.enabled",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.enabled",
true)
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.group-id",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_GB32960_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-raw}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.topics[0]",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.enabled",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.enabled",
true)
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.group-id",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_JT808_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-raw}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.topics[0]",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.enabled",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.enabled",
true)
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.group-id",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-raw}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.topics[0]",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");

16
modules/apps/xinda-push-app/src/main/resources/application.yml
View File

@@ -53,20 +53,16 @@ lingniu:
rate-limit:
per-vin-qps: 50
identity:
store: ${VEHICLE_IDENTITY_STORE:file}
file:
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
store: ${VEHICLE_IDENTITY_STORE:mysql}
mysql:
table: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:}
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
driver-class-name: ${VEHICLE_IDENTITY_MYSQL_DRIVER_CLASS_NAME:com.mysql.cj.jdbc.Driver}
refresh-interval: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd
linger-ms: 20

17
modules/apps/xinda-push-app/src/test/java/com/lingniu/ingest/xindapushapp/XindaPushAppCompositionTest.java
View File

@@ -9,9 +9,9 @@ import com.lingniu.ingest.inbound.xinda.config.XindaPushAutoConfiguration;
import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -26,7 +26,7 @@ class XindaPushAppCompositionTest {
.withConfiguration(AutoConfigurations.of(
IngestCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class,
XindaPushAutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true)
@@ -37,10 +37,11 @@ class XindaPushAppCompositionTest {
"lingniu.ingest.xinda-push.port=10100",
"lingniu.ingest.xinda-push.username=test",
"lingniu.ingest.xinda-push.password=test",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-xinda",
"lingniu.ingest.event-file-store.enabled=false",

15
modules/apps/xinda-push-app/src/test/java/com/lingniu/ingest/xindapushapp/XindaPushAppDefaultsTest.java
View File

@@ -26,13 +26,14 @@ class XindaPushAppDefaultsTest {
.containsEntry("lingniu.ingest.xinda-push.port", "${XINDA_PUSH_PORT:10100}")
.containsEntry("lingniu.ingest.xinda-push.username", "${XINDA_PUSH_USERNAME:}")
.containsEntry("lingniu.ingest.xinda-push.subscribe-msg-ids[0]", "${XINDA_PUSH_MSG_ID_LOCATION:0200}")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_XINDA_PUSH_EVENT:vehicle.event.xinda-push.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_XINDA_PUSH_RAW:vehicle.raw.xinda-push.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_XINDA_PUSH_DLQ:vehicle.dlq.xinda-push.v1}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}")
.containsEntry("lingniu.ingest.identity.mysql.table", "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}")
.containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_XINDA_PUSH_EVENT:vehicle.event.xinda-push.v1}")
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_XINDA_PUSH_RAW:vehicle.raw.xinda-push.v1}")
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_XINDA_PUSH_DLQ:vehicle.dlq.xinda-push.v1}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
.containsEntry("lingniu.ingest.identity.mysql.table-name",
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
.containsEntry("lingniu.ingest.event-history.enabled", false)

3
modules/apps/yutong-mqtt-app/src/main/resources/application.yml
View File

@@ -69,9 +69,8 @@ lingniu:
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd
linger-ms: 20

15
modules/apps/yutong-mqtt-app/src/test/java/com/lingniu/ingest/yutongmqttapp/YutongMqttAppCompositionTest.java
View File

@@ -10,9 +10,9 @@ import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -31,7 +31,7 @@ class YutongMqttAppCompositionTest {
.withConfiguration(AutoConfigurations.of(
IngestCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class,
MqttInboundAutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true)
@@ -45,10 +45,9 @@ class YutongMqttAppCompositionTest {
"lingniu.ingest.mqtt.endpoints[0].profile=yutong",
"lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-yutong",
"lingniu.ingest.event-history.enabled=false");

10
modules/apps/yutong-mqtt-app/src/test/java/com/lingniu/ingest/yutongmqttapp/YutongMqttAppDefaultsTest.java
View File

@@ -28,11 +28,11 @@ class YutongMqttAppDefaultsTest {
.containsEntry("lingniu.ingest.mqtt.endpoints[0].uri", "${YUTONG_MQTT_URI:}")
.containsEntry("lingniu.ingest.mqtt.endpoints[0].topic", "${YUTONG_MQTT_TOPIC:#}")
.containsEntry("lingniu.ingest.mqtt.endpoints[0].profile", "yutong")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_YUTONG_MQTT_DLQ:vehicle.dlq.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_YUTONG_MQTT_DLQ:vehicle.dlq.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
.containsEntry("lingniu.ingest.identity.mysql.table-name",
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")