refactor: expose kafka sink configuration

This commit is contained in:
lingniu
2026-07-01 10:50:50 +08:00
parent 303b2eb47e
commit 849e8f8a8b
67 changed files with 350 additions and 385 deletions

View File

@@ -18,7 +18,7 @@
- Existing handler currently writes report ACK before `dispatcher.dispatch(rf)`. - Existing handler currently writes report ACK before `dispatcher.dispatch(rf)`.
- Existing `DisruptorEventBus` publishes to sinks asynchronously and does not wait for sink completion. - Existing `DisruptorEventBus` publishes to sinks asynchronously and does not wait for sink completion.
- Existing `KafkaEventSink.accepts()` rejects `VehicleEvent.RawArchive`. - Existing `KafkaEventSink.accepts()` rejects `VehicleEvent.RawArchive`.
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.mq.topics`. - Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.kafka.topics`.
- Existing history consumer logic: `modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java`. - Existing history consumer logic: `modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java`.
- Existing analytics consumers: `VehicleStateEnvelopeIngestor` and `VehicleStatEnvelopeIngestor`. - Existing analytics consumers: `VehicleStateEnvelopeIngestor` and `VehicleStatEnvelopeIngestor`.
- There is an unrelated working-tree change in `modules/apps/bootstrap-all/src/main/resources/application.yml` for GB32960 diagnostics capacity. Do not revert it. Do not include it in service-split commits unless the user explicitly asks. - There is an unrelated working-tree change in `modules/apps/bootstrap-all/src/main/resources/application.yml` for GB32960 diagnostics capacity. Do not revert it. Do not include it in service-split commits unless the user explicitly asks.
@@ -48,7 +48,7 @@ Create app modules:
Modify shared build and Kafka contract files: Modify shared build and Kafka contract files:
- `pom.xml` - `pom.xml`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java` - `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java` - `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java` - `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java` - `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
@@ -466,16 +466,19 @@ lingniu:
rate-limit: rate-limit:
per-vin-qps: 50 per-vin-qps: 50
session: session:
store: ${SESSION_STORE:memory} store: ${SESSION_STORE:redis}
ttl: ${SESSION_TTL:30m} ttl: ${SESSION_TTL:30m}
identity: identity:
store: ${VEHICLE_IDENTITY_STORE:file} store: ${VEHICLE_IDENTITY_STORE:mysql}
file: mysql:
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl} jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd compression-type: zstd
linger-ms: 20 linger-ms: 20
@@ -568,9 +571,8 @@ lingniu:
gb32960: gb32960:
enabled: false enabled: false
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics: topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1} realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
@@ -666,9 +668,8 @@ lingniu:
gb32960: gb32960:
enabled: false enabled: false
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics: topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1} realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
@@ -754,7 +755,7 @@ import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration; import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer; import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
import com.lingniu.ingest.sink.archive.ArchiveStore; import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.mq.KafkaEventSink; import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.boot.test.context.runner.ApplicationContextRunner;
@@ -770,7 +771,7 @@ class Gb32960IngestAppCompositionTest {
.withPropertyValues( .withPropertyValues(
"lingniu.ingest.gb32960.enabled=true", "lingniu.ingest.gb32960.enabled=true",
"lingniu.ingest.gb32960.port=0", "lingniu.ingest.gb32960.port=0",
"lingniu.ingest.sink.mq.enabled=false", "lingniu.ingest.sink.kafka.enabled=false",
"lingniu.ingest.sink.archive.enabled=false", "lingniu.ingest.sink.archive.enabled=false",
"lingniu.ingest.event-file-store.enabled=false", "lingniu.ingest.event-file-store.enabled=false",
"lingniu.ingest.event-history.enabled=false", "lingniu.ingest.event-history.enabled=false",
@@ -835,7 +836,7 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.event-history.enabled=true", "lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.vehicle-state.enabled=false", "lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=false", "lingniu.ingest.vehicle-stat.enabled=false",
"lingniu.ingest.sink.mq.enabled=false") "lingniu.ingest.sink.kafka.enabled=false")
.withConfiguration(AutoConfigurations.of( .withConfiguration(AutoConfigurations.of(
com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration.class, com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration.class,
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration.class, com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration.class,
@@ -896,7 +897,7 @@ class VehicleAnalyticsAppCompositionTest {
"lingniu.ingest.vehicle-state.enabled=false", "lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=true", "lingniu.ingest.vehicle-stat.enabled=true",
"lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"), "lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"),
"lingniu.ingest.sink.mq.enabled=false") "lingniu.ingest.sink.kafka.enabled=false")
.withConfiguration(AutoConfigurations.of( .withConfiguration(AutoConfigurations.of(
com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration.class, com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration.class,
com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration.class)) com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration.class))
@@ -932,7 +933,7 @@ git commit -m "test: cover split service composition"
### Task 4: Version Kafka Topics for GB32960 Raw/Event/DLQ ### Task 4: Version Kafka Topics for GB32960 Raw/Event/DLQ
**Files:** **Files:**
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java` - Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java`
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java` - Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java` - Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java` - Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
@@ -947,7 +948,7 @@ Expected test names:
```java ```java
@Test @Test
void rawArchiveRoutesToVersionedGb32960RawTopic() { void rawArchiveRoutesToVersionedGb32960RawTopic() {
SinkMqProperties.Topics topics = new SinkMqProperties.Topics(); KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
topics.setRawArchive("vehicle.raw.gb32960.v1"); topics.setRawArchive("vehicle.raw.gb32960.v1");
TopicRouter router = new TopicRouter(topics); TopicRouter router = new TopicRouter(topics);
@@ -970,7 +971,7 @@ Also add a normalized realtime event test:
```java ```java
@Test @Test
void realtimeRoutesToVersionedGb32960EventTopic() { void realtimeRoutesToVersionedGb32960EventTopic() {
SinkMqProperties.Topics topics = new SinkMqProperties.Topics(); KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
topics.setRealtime("vehicle.event.gb32960.v1"); topics.setRealtime("vehicle.event.gb32960.v1");
TopicRouter router = new TopicRouter(topics); TopicRouter router = new TopicRouter(topics);
@@ -1000,9 +1001,9 @@ mvn -pl :sink-mq -Dtest=TopicRouterTest test
Expected before implementation: compile failure if constructors are wrong or assertion failure if defaults are old. Fix constructors first; keep routing assertions. Expected before implementation: compile failure if constructors are wrong or assertion failure if defaults are old. Fix constructors first; keep routing assertions.
- [ ] **Step 3: Update `SinkMqProperties.Topics` defaults** - [ ] **Step 3: Update `KafkaSinkProperties.Topics` defaults**
In `SinkMqProperties.Topics`, change defaults: In `KafkaSinkProperties.Topics`, change defaults:
```java ```java
private String realtime = "vehicle.event.gb32960.v1"; private String realtime = "vehicle.event.gb32960.v1";
@@ -1040,7 +1041,7 @@ Expected: PASS.
- [ ] **Step 6: Commit** - [ ] **Step 6: Commit**
```bash ```bash
git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java \ git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java \
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java \ modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java \
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java \ modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java \
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java \ modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java \
@@ -1353,8 +1354,8 @@ void gb32960IngestDefaultsOnlyEnableProtocolAndKafkaProducer() {
assertThat(props.getProperty("spring.application.name")).isEqualTo("gb32960-ingest-app"); assertThat(props.getProperty("spring.application.name")).isEqualTo("gb32960-ingest-app");
assertThat(props.getProperty("lingniu.ingest.gb32960.enabled")).isEqualTo("true"); assertThat(props.getProperty("lingniu.ingest.gb32960.enabled")).isEqualTo("true");
assertThat(props.getProperty("lingniu.ingest.gb32960.port")).isEqualTo("${GB32960_PORT:32960}"); assertThat(props.getProperty("lingniu.ingest.gb32960.port")).isEqualTo("${GB32960_PORT:32960}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.enabled")).isEqualTo("${KAFKA_ENABLED:true}"); assertThat(props.getProperty("lingniu.ingest.sink.kafka.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.enabled")).isEqualTo("false"); assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false"); assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false"); assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false"); assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
@@ -1378,7 +1379,7 @@ void historyDefaultsEnableStorageAndHistoryConsumerOnly() {
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("true"); assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("true");
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false"); assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("false"); assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id")) assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
.isEqualTo("${KAFKA_GROUP_HISTORY:vehicle-history}"); .isEqualTo("${KAFKA_GROUP_HISTORY:vehicle-history}");
} }
``` ```
@@ -1398,7 +1399,7 @@ void analyticsDefaultsEnableStatConsumerAndDisableProtocolAndHistoryStorage() {
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false"); assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false"); assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("${VEHICLE_STAT_ENABLED:true}"); assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("${VEHICLE_STAT_ENABLED:true}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id")) assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
.isEqualTo("${KAFKA_GROUP_STAT:vehicle-stat}"); .isEqualTo("${KAFKA_GROUP_STAT:vehicle-stat}");
} }
``` ```

View File

@@ -1269,12 +1269,12 @@ git commit -m "feat: add raw archive store contract"
Create `VehicleEnvelopeProtoCompatibilityTest.java`: Create `VehicleEnvelopeProtoCompatibilityTest.java`:
```java ```java
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.sink.mq.proto.DecodedFactPayload; import com.lingniu.ingest.sink.kafka.proto.DecodedFactPayload;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto; import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload; import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;

View File

@@ -133,7 +133,7 @@ Use versioned topics:
- `vehicle.event.gb32960.v1` - `vehicle.event.gb32960.v1`
- `vehicle.dlq.gb32960.v1` - `vehicle.dlq.gb32960.v1`
The current topic names under `lingniu.ingest.sink.mq.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names. The current topic names under `lingniu.ingest.sink.kafka.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
### Keys ### Keys

View File

@@ -71,7 +71,7 @@ repositories, and statistic calculators.
### Event Contract ### Event Contract
`sink-mq` owns the Kafka protobuf envelope and consumer plumbing. The Kafka sink owns the protobuf envelope and consumer plumbing.
Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped
envelopes, and downstream failures are converted to structured results and DLQ envelopes, and downstream failures are converted to structured results and DLQ
records instead of blocking a partition. records instead of blocking a partition.

View File

@@ -99,9 +99,8 @@ lingniu:
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings} table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true} initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd compression-type: zstd
linger-ms: 20 linger-ms: 20

View File

@@ -11,9 +11,9 @@ import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
import com.lingniu.ingest.sink.archive.ArchiveStore; import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink; import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration; import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEventSink; import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration; import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -31,7 +31,7 @@ class Gb32960IngestAppCompositionTest {
IngestCoreAutoConfiguration.class, IngestCoreAutoConfiguration.class,
SessionCoreAutoConfiguration.class, SessionCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class, VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class, KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class, SinkArchiveAutoConfiguration.class,
Gb32960AutoConfiguration.class)) Gb32960AutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true) .withAllowBeanDefinitionOverriding(true)
@@ -43,10 +43,9 @@ class Gb32960IngestAppCompositionTest {
"lingniu.ingest.gb32960.port=0", "lingniu.ingest.gb32960.port=0",
"lingniu.ingest.identity.store=mysql", "lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false", "lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.sink.mq.enabled=true", "lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.mq.type=kafka", "lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092", "lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true", "lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-gb32960", "lingniu.ingest.sink.archive.path=target/test-archive-gb32960",
"lingniu.ingest.event-history.enabled=false"); "lingniu.ingest.event-history.enabled=false");

View File

@@ -15,6 +15,8 @@ class Gb32960IngestAppDefaultsTest {
@Test @Test
void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() throws IOException { void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() throws IOException {
Properties properties = applicationProperties(); Properties properties = applicationProperties();
String legacyKafkaPrefix = "lingniu.ingest.sink." + "mq.";
String kafkaTypeProperty = "lingniu.ingest.sink.kafka." + "type";
assertThat(properties) assertThat(properties)
.containsEntry("spring.application.name", "gb32960-ingest-app") .containsEntry("spring.application.name", "gb32960-ingest-app")
@@ -40,8 +42,8 @@ class Gb32960IngestAppDefaultsTest {
"Hyundai") "Hyundai")
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[1]", .containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[1]",
"YueJin") "YueJin")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}") .containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false) .containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}") .containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}") .containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}") .containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
@@ -55,7 +57,9 @@ class Gb32960IngestAppDefaultsTest {
assertThat(properties.stringPropertyNames()) assertThat(properties.stringPropertyNames())
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store.")) .noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state.")) .noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat.")); .noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
.noneMatch(name -> name.startsWith(legacyKafkaPrefix))
.noneMatch(name -> name.equals(kafkaTypeProperty));
assertThat(applicationYaml()) assertThat(applicationYaml())
.doesNotContain("event-file-store:") .doesNotContain("event-file-store:")
.doesNotContain("vehicle-state:") .doesNotContain("vehicle-state:")

View File

@@ -63,9 +63,8 @@ lingniu:
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings} table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true} initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd compression-type: zstd
linger-ms: 20 linger-ms: 20

View File

@@ -11,9 +11,9 @@ import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
import com.lingniu.ingest.sink.archive.ArchiveStore; import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink; import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration; import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink; import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration; import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -31,7 +31,7 @@ class Jt808IngestAppCompositionTest {
IngestCoreAutoConfiguration.class, IngestCoreAutoConfiguration.class,
SessionCoreAutoConfiguration.class, SessionCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class, VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class, KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class, SinkArchiveAutoConfiguration.class,
Jt808AutoConfiguration.class)) Jt808AutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true) .withAllowBeanDefinitionOverriding(true)
@@ -42,10 +42,9 @@ class Jt808IngestAppCompositionTest {
"lingniu.ingest.jt808.port=0", "lingniu.ingest.jt808.port=0",
"lingniu.ingest.identity.store=mysql", "lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false", "lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.sink.mq.enabled=true", "lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.mq.type=kafka", "lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092", "lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true", "lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-jt808", "lingniu.ingest.sink.archive.path=target/test-archive-jt808",
"lingniu.ingest.event-history.enabled=false"); "lingniu.ingest.event-history.enabled=false");

View File

@@ -28,11 +28,11 @@ class Jt808IngestAppDefaultsTest {
.containsEntry("server.port", "${HTTP_PORT:20400}") .containsEntry("server.port", "${HTTP_PORT:20400}")
.containsEntry("lingniu.ingest.jt808.enabled", true) .containsEntry("lingniu.ingest.jt808.enabled", true)
.containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:808}") .containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:808}")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}") .containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false) .containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}") .containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}") .containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
.containsEntry("lingniu.ingest.identity.mysql.table-name", .containsEntry("lingniu.ingest.identity.mysql.table-name",

View File

@@ -37,9 +37,8 @@ lingniu:
gb32960: gb32960:
enabled: false enabled: false
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics: topics:
realtime: ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1} realtime: ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}

View File

@@ -2,9 +2,9 @@ package com.lingniu.ingest.analyticsapp;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink; import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration; import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import com.lingniu.ingest.vehiclestat.JdbcVehicleStatMetricRepository; import com.lingniu.ingest.vehiclestat.JdbcVehicleStatMetricRepository;
import com.lingniu.ingest.vehiclestat.VehicleStatController; import com.lingniu.ingest.vehiclestat.VehicleStatController;
import com.lingniu.ingest.vehiclestat.VehicleStatEnvelopeIngestor; import com.lingniu.ingest.vehiclestat.VehicleStatEnvelopeIngestor;
@@ -28,17 +28,16 @@ class VehicleAnalyticsAppCompositionTest {
void createsStatsBeansWithoutProtocolListenerOrEventFileStore() { void createsStatsBeansWithoutProtocolListenerOrEventFileStore() {
new ApplicationContextRunner() new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of( .withConfiguration(AutoConfigurations.of(
SinkMqAutoConfiguration.class, KafkaSinkAutoConfiguration.class,
VehicleStatAutoConfiguration.class)) VehicleStatAutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true) .withAllowBeanDefinitionOverriding(true)
.withBean("kafkaProducer", KafkaProducer.class, VehicleAnalyticsAppCompositionTest::kafkaProducer) .withBean("kafkaProducer", KafkaProducer.class, VehicleAnalyticsAppCompositionTest::kafkaProducer)
.withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class)) .withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class))
.withBean(ObjectMapper.class, ObjectMapper::new) .withBean(ObjectMapper.class, ObjectMapper::new)
.withPropertyValues( .withPropertyValues(
"lingniu.ingest.sink.mq.enabled=true", "lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.mq.type=kafka", "lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092", "lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=true", "lingniu.ingest.vehicle-stat.enabled=true",
"lingniu.ingest.vehicle-stat.jt808.enabled=true", "lingniu.ingest.vehicle-stat.jt808.enabled=true",
"lingniu.ingest.event-history.enabled=false", "lingniu.ingest.event-history.enabled=false",

View File

@@ -29,12 +29,12 @@ class VehicleAnalyticsAppDefaultsTest {
.containsEntry( .containsEntry(
"lingniu.ingest.vehicle-stat.jt808.enabled", "lingniu.ingest.vehicle-stat.jt808.enabled",
"${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}") "${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}") .containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled", "lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
"${VEHICLE_STAT_ENABLED:true}") "${VEHICLE_STAT_ENABLED:true}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id", "lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_STAT:vehicle-stat}") "${KAFKA_GROUP_STAT:vehicle-stat}")
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus"); .containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");

View File

@@ -1,10 +1,10 @@
package com.lingniu.ingest.historyapp; package com.lingniu.ingest.historyapp;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerFactory; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerFactory;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerWorker; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerWorker;
import com.lingniu.ingest.sink.mq.SinkMqProperties; import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -20,11 +20,11 @@ public class VehicleHistoryKafkaConsumerConfiguration {
@Bean @Bean
@ConditionalOnMissingBean @ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner vehicleHistoryKafkaEnvelopeConsumerRunner( public KafkaEnvelopeConsumerRunner vehicleHistoryKafkaEnvelopeConsumerRunner(
@Qualifier("eventHistoryEnvelopeConsumerProcessor") EnvelopeConsumerProcessor processor, @Qualifier("eventHistoryEnvelopeConsumerProcessor") EnvelopeConsumerProcessor processor,
@Qualifier("eventHistoryRawEnvelopeConsumerProcessor") EnvelopeConsumerProcessor rawProcessor, @Qualifier("eventHistoryRawEnvelopeConsumerProcessor") EnvelopeConsumerProcessor rawProcessor,
SinkMqProperties props) { KafkaSinkProperties props) {
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers( List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(
Map.of( Map.of(
"eventHistoryGb32960EnvelopeConsumerProcessor", processor, "eventHistoryGb32960EnvelopeConsumerProcessor", processor,

View File

@@ -34,9 +34,8 @@ lingniu:
server: server:
enabled: false enabled: false
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics: topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1} realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}

View File

@@ -15,15 +15,15 @@ import com.lingniu.ingest.eventhistory.TelemetryFieldHistoryController;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder; import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration; import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer; import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink; import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.mq.SinkMqProperties; import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto; import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef; import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload; import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration; import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema; import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter; import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow; import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;
@@ -75,8 +75,8 @@ class VehicleHistoryAppCompositionTest {
() -> processor("event-history")) () -> processor("event-history"))
.withBean("eventHistoryRawEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class, .withBean("eventHistoryRawEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
() -> processor("event-history-raw")) () -> processor("event-history-raw"))
.withBean(SinkMqProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps) .withBean(KafkaSinkProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
.withPropertyValues("lingniu.ingest.sink.mq.consumer.enabled=true") .withPropertyValues("lingniu.ingest.sink.kafka.consumer.enabled=true")
.run(context -> { .run(context -> {
assertThat(context).hasFailed(); assertThat(context).hasFailed();
assertThat(context.getStartupFailure()) assertThat(context.getStartupFailure())
@@ -89,7 +89,7 @@ class VehicleHistoryAppCompositionTest {
new ApplicationContextRunner() new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of( .withConfiguration(AutoConfigurations.of(
TdengineHistoryAutoConfiguration.class, TdengineHistoryAutoConfiguration.class,
SinkMqAutoConfiguration.class, KafkaSinkAutoConfiguration.class,
Gb32960AutoConfiguration.class, Gb32960AutoConfiguration.class,
EventHistoryAutoConfiguration.class)) EventHistoryAutoConfiguration.class))
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class) .withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
@@ -101,10 +101,9 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.event-history.enabled=true", "lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.gb32960.enabled=true", "lingniu.ingest.gb32960.enabled=true",
"lingniu.ingest.gb32960.server.enabled=false", "lingniu.ingest.gb32960.server.enabled=false",
"lingniu.ingest.sink.mq.enabled=true", "lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.mq.type=kafka", "lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092", "lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.vehicle-state.enabled=false", "lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=false") "lingniu.ingest.vehicle-stat.enabled=false")
.run(context -> { .run(context -> {
@@ -137,7 +136,7 @@ class VehicleHistoryAppCompositionTest {
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class)) .withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
.withPropertyValues( .withPropertyValues(
"lingniu.ingest.event-history.enabled=true", "lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=false") "lingniu.ingest.sink.kafka.consumer.enabled=false")
.run(context -> { .run(context -> {
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class); assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context).doesNotHaveBean("eventFileStore"); assertThat(context).doesNotHaveBean("eventFileStore");
@@ -149,7 +148,7 @@ class VehicleHistoryAppCompositionTest {
new ApplicationContextRunner() new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of( .withConfiguration(AutoConfigurations.of(
TdengineHistoryAutoConfiguration.class, TdengineHistoryAutoConfiguration.class,
SinkMqAutoConfiguration.class, KafkaSinkAutoConfiguration.class,
EventHistoryAutoConfiguration.class)) EventHistoryAutoConfiguration.class))
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class) .withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
.withAllowBeanDefinitionOverriding(true) .withAllowBeanDefinitionOverriding(true)
@@ -158,13 +157,12 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.tdengine-history.enabled=true", "lingniu.ingest.tdengine-history.enabled=true",
"lingniu.ingest.tdengine-history.database=vehicle_history_test", "lingniu.ingest.tdengine-history.database=vehicle_history_test",
"lingniu.ingest.event-history.enabled=true", "lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.enabled=true", "lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.mq.type=kafka", "lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092", "lingniu.ingest.sink.kafka.consumer.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=true", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
.run(context -> { .run(context -> {
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class); assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context.getBeanNamesForType(EnvelopeConsumerProcessor.class)) assertThat(context.getBeanNamesForType(EnvelopeConsumerProcessor.class))
@@ -186,7 +184,7 @@ class VehicleHistoryAppCompositionTest {
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class)) .withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
.withPropertyValues( .withPropertyValues(
"lingniu.ingest.event-history.enabled=true", "lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=false") "lingniu.ingest.sink.kafka.consumer.enabled=false")
.run(context -> { .run(context -> {
EventHistoryEnvelopeIngestor ingestor = context.getBean(EventHistoryEnvelopeIngestor.class); EventHistoryEnvelopeIngestor ingestor = context.getBean(EventHistoryEnvelopeIngestor.class);
@@ -211,8 +209,8 @@ class VehicleHistoryAppCompositionTest {
ignored -> {}); ignored -> {});
} }
private static SinkMqProperties genericOnlyHistoryConsumerProps() { private static KafkaSinkProperties genericOnlyHistoryConsumerProps() {
SinkMqProperties props = new SinkMqProperties(); KafkaSinkProperties props = new KafkaSinkProperties();
props.setBootstrapServers("localhost:9092"); props.setBootstrapServers("localhost:9092");
props.getConsumer().setEnabled(true); props.getConsumer().setEnabled(true);
props.getConsumer().setBindings(Map.of( props.getConsumer().setBindings(Map.of(
@@ -221,8 +219,8 @@ class VehicleHistoryAppCompositionTest {
return props; return props;
} }
private static SinkMqProperties.Binding binding(String groupId, String topic) { private static KafkaSinkProperties.Binding binding(String groupId, String topic) {
SinkMqProperties.Binding binding = new SinkMqProperties.Binding(); KafkaSinkProperties.Binding binding = new KafkaSinkProperties.Binding();
binding.setGroupId(groupId); binding.setGroupId(groupId);
binding.setTopics(List.of(topic)); binding.setTopics(List.of(topic));
return binding; return binding;

View File

@@ -44,65 +44,65 @@ class VehicleHistoryAppDefaultsTest {
.containsEntry("lingniu.ingest.event-history.enabled", true) .containsEntry("lingniu.ingest.event-history.enabled", true)
.containsEntry("lingniu.ingest.vehicle-state.enabled", false) .containsEntry("lingniu.ingest.vehicle-state.enabled", false)
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false) .containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}") .containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}") .containsEntry("lingniu.ingest.sink.kafka.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
.containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}") .containsEntry("lingniu.ingest.sink.kafka.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.max-poll-interval-millis", "lingniu.ingest.sink.kafka.consumer.max-poll-interval-millis",
"${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}") "${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled",
true) true)
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_GB32960_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-event}") "${KAFKA_GROUP_HISTORY_GB32960_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-event}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}") "${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.enabled", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.enabled",
true) true)
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.group-id", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_JT808_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-event}") "${KAFKA_GROUP_HISTORY_JT808_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-event}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.topics[0]", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}") "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.enabled", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.enabled",
true) true)
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.group-id", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-event}") "${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-event}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.topics[0]", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}") "${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.enabled", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.enabled",
true) true)
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.group-id", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_GB32960_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-raw}") "${KAFKA_GROUP_HISTORY_GB32960_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-raw}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.topics[0]", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}") "${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.enabled", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.enabled",
true) true)
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.group-id", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_JT808_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-raw}") "${KAFKA_GROUP_HISTORY_JT808_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-raw}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.topics[0]", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}") "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.enabled", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.enabled",
true) true)
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.group-id", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-raw}") "${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-raw}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.topics[0]", "lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.topics[0]",
"${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}") "${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus"); .containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");

View File

@@ -53,20 +53,16 @@ lingniu:
rate-limit: rate-limit:
per-vin-qps: 50 per-vin-qps: 50
identity: identity:
store: ${VEHICLE_IDENTITY_STORE:file} store: ${VEHICLE_IDENTITY_STORE:mysql}
file:
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
mysql: mysql:
table: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding} jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:} username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:}
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:} password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
driver-class-name: ${VEHICLE_IDENTITY_MYSQL_DRIVER_CLASS_NAME:com.mysql.cj.jdbc.Driver} table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
refresh-interval: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s} initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd compression-type: zstd
linger-ms: 20 linger-ms: 20

View File

@@ -9,9 +9,9 @@ import com.lingniu.ingest.inbound.xinda.config.XindaPushAutoConfiguration;
import com.lingniu.ingest.sink.archive.ArchiveStore; import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink; import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration; import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink; import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration; import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -26,7 +26,7 @@ class XindaPushAppCompositionTest {
.withConfiguration(AutoConfigurations.of( .withConfiguration(AutoConfigurations.of(
IngestCoreAutoConfiguration.class, IngestCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class, VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class, KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class, SinkArchiveAutoConfiguration.class,
XindaPushAutoConfiguration.class)) XindaPushAutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true) .withAllowBeanDefinitionOverriding(true)
@@ -37,10 +37,11 @@ class XindaPushAppCompositionTest {
"lingniu.ingest.xinda-push.port=10100", "lingniu.ingest.xinda-push.port=10100",
"lingniu.ingest.xinda-push.username=test", "lingniu.ingest.xinda-push.username=test",
"lingniu.ingest.xinda-push.password=test", "lingniu.ingest.xinda-push.password=test",
"lingniu.ingest.sink.mq.enabled=true", "lingniu.ingest.identity.store=mysql",
"lingniu.ingest.sink.mq.type=kafka", "lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092", "lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=false", "lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true", "lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-xinda", "lingniu.ingest.sink.archive.path=target/test-archive-xinda",
"lingniu.ingest.event-file-store.enabled=false", "lingniu.ingest.event-file-store.enabled=false",

View File

@@ -26,13 +26,14 @@ class XindaPushAppDefaultsTest {
.containsEntry("lingniu.ingest.xinda-push.port", "${XINDA_PUSH_PORT:10100}") .containsEntry("lingniu.ingest.xinda-push.port", "${XINDA_PUSH_PORT:10100}")
.containsEntry("lingniu.ingest.xinda-push.username", "${XINDA_PUSH_USERNAME:}") .containsEntry("lingniu.ingest.xinda-push.username", "${XINDA_PUSH_USERNAME:}")
.containsEntry("lingniu.ingest.xinda-push.subscribe-msg-ids[0]", "${XINDA_PUSH_MSG_ID_LOCATION:0200}") .containsEntry("lingniu.ingest.xinda-push.subscribe-msg-ids[0]", "${XINDA_PUSH_MSG_ID_LOCATION:0200}")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}") .containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false) .containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_XINDA_PUSH_EVENT:vehicle.event.xinda-push.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_XINDA_PUSH_EVENT:vehicle.event.xinda-push.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_XINDA_PUSH_RAW:vehicle.raw.xinda-push.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_XINDA_PUSH_RAW:vehicle.raw.xinda-push.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_XINDA_PUSH_DLQ:vehicle.dlq.xinda-push.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_XINDA_PUSH_DLQ:vehicle.dlq.xinda-push.v1}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}") .containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
.containsEntry("lingniu.ingest.identity.mysql.table", "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}") .containsEntry("lingniu.ingest.identity.mysql.table-name",
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}") .containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
.containsEntry("lingniu.ingest.event-file-store.enabled", false) .containsEntry("lingniu.ingest.event-file-store.enabled", false)
.containsEntry("lingniu.ingest.event-history.enabled", false) .containsEntry("lingniu.ingest.event-history.enabled", false)

View File

@@ -69,9 +69,8 @@ lingniu:
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings} table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true} initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink: sink:
mq: kafka:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092} bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd compression-type: zstd
linger-ms: 20 linger-ms: 20

View File

@@ -10,9 +10,9 @@ import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import com.lingniu.ingest.sink.archive.ArchiveStore; import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink; import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration; import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink; import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink; import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration; import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
@@ -31,7 +31,7 @@ class YutongMqttAppCompositionTest {
.withConfiguration(AutoConfigurations.of( .withConfiguration(AutoConfigurations.of(
IngestCoreAutoConfiguration.class, IngestCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class, VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class, KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class, SinkArchiveAutoConfiguration.class,
MqttInboundAutoConfiguration.class)) MqttInboundAutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true) .withAllowBeanDefinitionOverriding(true)
@@ -45,10 +45,9 @@ class YutongMqttAppCompositionTest {
"lingniu.ingest.mqtt.endpoints[0].profile=yutong", "lingniu.ingest.mqtt.endpoints[0].profile=yutong",
"lingniu.ingest.identity.store=mysql", "lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false", "lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.sink.mq.enabled=true", "lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.mq.type=kafka", "lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092", "lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true", "lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-yutong", "lingniu.ingest.sink.archive.path=target/test-archive-yutong",
"lingniu.ingest.event-history.enabled=false"); "lingniu.ingest.event-history.enabled=false");

View File

@@ -28,11 +28,11 @@ class YutongMqttAppDefaultsTest {
.containsEntry("lingniu.ingest.mqtt.endpoints[0].uri", "${YUTONG_MQTT_URI:}") .containsEntry("lingniu.ingest.mqtt.endpoints[0].uri", "${YUTONG_MQTT_URI:}")
.containsEntry("lingniu.ingest.mqtt.endpoints[0].topic", "${YUTONG_MQTT_TOPIC:#}") .containsEntry("lingniu.ingest.mqtt.endpoints[0].topic", "${YUTONG_MQTT_TOPIC:#}")
.containsEntry("lingniu.ingest.mqtt.endpoints[0].profile", "yutong") .containsEntry("lingniu.ingest.mqtt.endpoints[0].profile", "yutong")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}") .containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false) .containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_YUTONG_MQTT_DLQ:vehicle.dlq.mqtt-yutong.v1}") .containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_YUTONG_MQTT_DLQ:vehicle.dlq.mqtt-yutong.v1}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}") .containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
.containsEntry("lingniu.ingest.identity.mysql.table-name", .containsEntry("lingniu.ingest.identity.mysql.table-name",
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}") "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")

View File

@@ -5,7 +5,7 @@ import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.history.EventFileRecord; import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore; import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows; import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter; import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
import org.slf4j.Logger; import org.slf4j.Logger;

View File

@@ -5,8 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.RawArchiveKeys; import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.history.EventFileRecord; import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef; import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat;

View File

@@ -15,7 +15,7 @@ import com.lingniu.ingest.eventhistory.TelemetryFieldHistoryController;
import com.lingniu.ingest.eventhistory.TelemetryEnvelopeRecordMapper; import com.lingniu.ingest.eventhistory.TelemetryEnvelopeRecordMapper;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder; import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration; import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration; import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import com.lingniu.ingest.tdenginehistory.config.TdengineHistoryAutoConfiguration; import com.lingniu.ingest.tdenginehistory.config.TdengineHistoryAutoConfiguration;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader; import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter; import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
@@ -41,12 +41,12 @@ import java.nio.file.Path;
* </ul> * </ul>
* *
* <p>当前生产 history app 以 TDengine 为准Kafka consumer 是否启动还取决于 * <p>当前生产 history app 以 TDengine 为准Kafka consumer 是否启动还取决于
* {@code lingniu.ingest.sink.mq.consumer.enabled=true}。 * {@code lingniu.ingest.sink.kafka.consumer.enabled=true}。
*/ */
@AutoConfiguration @AutoConfiguration
@AutoConfigureAfter({ @AutoConfigureAfter({
Gb32960AutoConfiguration.class, Gb32960AutoConfiguration.class,
SinkMqAutoConfiguration.class, KafkaSinkAutoConfiguration.class,
TdengineHistoryAutoConfiguration.class TdengineHistoryAutoConfiguration.class
}) })
@ConditionalOnProperty(prefix = "lingniu.ingest.event-history", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "lingniu.ingest.event-history", name = "enabled", havingValue = "true")
@@ -82,7 +82,7 @@ public class EventHistoryAutoConfiguration {
@ConditionalOnMissingBean(name = "eventHistoryEnvelopeConsumerProcessor") @ConditionalOnMissingBean(name = "eventHistoryEnvelopeConsumerProcessor")
public EnvelopeConsumerProcessor eventHistoryEnvelopeConsumerProcessor(EventHistoryEnvelopeIngestor ingestor, public EnvelopeConsumerProcessor eventHistoryEnvelopeConsumerProcessor(EventHistoryEnvelopeIngestor ingestor,
EnvelopeDeadLetterSink deadLetterSink) { EnvelopeDeadLetterSink deadLetterSink) {
// 只注册 processor真正拉 Kafka 的 runner 在 sink-mq 模块按 consumer.enabled 决定是否创建。 // 只注册 processor真正拉 Kafka 的 runner 按 consumer.enabled 决定是否创建。
return new EnvelopeConsumerProcessor("event-history", ingestor, deadLetterSink); return new EnvelopeConsumerProcessor("event-history", ingestor, deadLetterSink);
} }

View File

@@ -9,13 +9,13 @@ import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.history.EventFileQuery; import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord; import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore; import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef; import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload; import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.sink.mq.proto.LocationPayload; import com.lingniu.ingest.sink.kafka.proto.LocationPayload;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto; import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter; import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow; import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow; import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;

View File

@@ -2,10 +2,10 @@ package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.history.EventFileRecord; import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef; import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;

View File

@@ -37,7 +37,7 @@ class EventHistoryAutoConfigurationTest {
contextRunner contextRunner
.withPropertyValues( .withPropertyValues(
"lingniu.ingest.event-history.enabled=true", "lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=true") "lingniu.ingest.sink.kafka.consumer.enabled=true")
.run(context -> { .run(context -> {
assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class); assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class);
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class); assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);

View File

@@ -22,7 +22,7 @@ class EventHistoryPomBoundaryTest {
} }
@Test @Test
void kafkaClientsIsOwnedBySinkMqModule() throws Exception { void kafkaClientsIsOwnedByKafkaSinkModule() throws Exception {
for (Path pom : servicePoms()) { for (Path pom : servicePoms()) {
assertThat(directDependencies(readPom(pom))) assertThat(directDependencies(readPom(pom)))
.as(pom.toString()) .as(pom.toString())

View File

@@ -3,7 +3,7 @@ package com.lingniu.ingest.vehiclestat;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.consumer.EnvelopeIngestor; import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor; import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
public final class VehicleStatEnvelopeIngestor implements EnvelopeIngestor { public final class VehicleStatEnvelopeIngestor implements EnvelopeIngestor {

View File

@@ -78,7 +78,7 @@ public class VehicleStatAutoConfiguration {
@ConditionalOnMissingBean(name = "vehicleStatEnvelopeConsumerProcessor") @ConditionalOnMissingBean(name = "vehicleStatEnvelopeConsumerProcessor")
public EnvelopeConsumerProcessor vehicleStatEnvelopeConsumerProcessor(VehicleStatEnvelopeIngestor ingestor, public EnvelopeConsumerProcessor vehicleStatEnvelopeConsumerProcessor(VehicleStatEnvelopeIngestor ingestor,
EnvelopeDeadLetterSink deadLetterSink) { EnvelopeDeadLetterSink deadLetterSink) {
// Bean 名必须和 sink-mq 默认 binding 对齐KafkaEnvelopeConsumerFactory 才能自动创建 worker。 // Bean 名必须和 Kafka 默认 binding 对齐KafkaEnvelopeConsumerFactory 才能自动创建 worker。
return new EnvelopeConsumerProcessor("vehicle-stat", ingestor, deadLetterSink); return new EnvelopeConsumerProcessor("vehicle-stat", ingestor, deadLetterSink);
} }
} }

View File

@@ -1,7 +1,7 @@
package com.lingniu.ingest.vehiclestat.jt808; package com.lingniu.ingest.vehiclestat.jt808;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import java.time.Instant; import java.time.Instant;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;

View File

@@ -1,6 +1,6 @@
package com.lingniu.ingest.vehiclestat.jt808; package com.lingniu.ingest.vehiclestat.jt808;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.vehiclestat.VehicleStatRepository; import com.lingniu.ingest.vehiclestat.VehicleStatRepository;
import java.time.LocalDate; import java.time.LocalDate;

View File

@@ -2,10 +2,10 @@ package com.lingniu.ingest.vehiclestat;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.consumer.EnvelopeIngestor; import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef; import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor; import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;

View File

@@ -1,8 +1,8 @@
package com.lingniu.ingest.vehiclestat.jt808; package com.lingniu.ingest.vehiclestat.jt808;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.time.Instant; import java.time.Instant;

View File

@@ -1,8 +1,8 @@
package com.lingniu.ingest.vehiclestat.jt808; package com.lingniu.ingest.vehiclestat.jt808;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import com.lingniu.ingest.vehiclestat.DailyMileageStrategy; import com.lingniu.ingest.vehiclestat.DailyMileageStrategy;
import com.lingniu.ingest.vehiclestat.VehicleDailyStatResult; import com.lingniu.ingest.vehiclestat.VehicleDailyStatResult;
import com.lingniu.ingest.vehiclestat.VehicleStatRepository; import com.lingniu.ingest.vehiclestat.VehicleStatRepository;

View File

@@ -3,7 +3,7 @@ package com.lingniu.ingest.vehiclestate;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.consumer.EnvelopeIngestor; import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
public final class VehicleStateEnvelopeIngestor implements EnvelopeIngestor { public final class VehicleStateEnvelopeIngestor implements EnvelopeIngestor {

View File

@@ -1,7 +1,7 @@
package com.lingniu.ingest.vehiclestate; package com.lingniu.ingest.vehiclestate;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;

View File

@@ -45,7 +45,7 @@ public class VehicleStateAutoConfiguration {
@ConditionalOnMissingBean(name = "vehicleStateEnvelopeConsumerProcessor") @ConditionalOnMissingBean(name = "vehicleStateEnvelopeConsumerProcessor")
public EnvelopeConsumerProcessor vehicleStateEnvelopeConsumerProcessor(VehicleStateEnvelopeIngestor ingestor, public EnvelopeConsumerProcessor vehicleStateEnvelopeConsumerProcessor(VehicleStateEnvelopeIngestor ingestor,
EnvelopeDeadLetterSink deadLetterSink) { EnvelopeDeadLetterSink deadLetterSink) {
// Bean 名必须和 sink-mq 默认 binding 对齐KafkaEnvelopeConsumerFactory 才能自动创建 worker。 // Bean 名必须和 Kafka 默认 binding 对齐KafkaEnvelopeConsumerFactory 才能自动创建 worker。
return new EnvelopeConsumerProcessor("vehicle-state", ingestor, deadLetterSink); return new EnvelopeConsumerProcessor("vehicle-state", ingestor, deadLetterSink);
} }

View File

@@ -2,10 +2,10 @@ package com.lingniu.ingest.vehiclestate;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.consumer.EnvelopeIngestor; import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef; import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.util.Optional; import java.util.Optional;

View File

@@ -1,8 +1,8 @@
package com.lingniu.ingest.vehiclestate; package com.lingniu.ingest.vehiclestate;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.event.AlarmPayload; import com.lingniu.ingest.api.event.AlarmPayload;
import com.lingniu.ingest.api.event.LocationPayload; import com.lingniu.ingest.api.event.LocationPayload;
@@ -10,10 +10,10 @@ import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.api.event.VehicleEventTelemetrySnapshotMapper; import com.lingniu.ingest.api.event.VehicleEventTelemetrySnapshotMapper;
import com.lingniu.ingest.facts.FactIds; import com.lingniu.ingest.facts.FactIds;
import com.lingniu.ingest.facts.VehicleKey; import com.lingniu.ingest.facts.VehicleKey;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto; import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot.Builder; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot.Builder;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
@@ -53,23 +53,23 @@ public final class EnvelopeMapper {
case VehicleEvent.Location l -> b.setLocation(buildLocation(l.payload())); case VehicleEvent.Location l -> b.setLocation(buildLocation(l.payload()));
case VehicleEvent.Alarm a -> b.setAlarm(buildAlarm(a.payload())); case VehicleEvent.Alarm a -> b.setAlarm(buildAlarm(a.payload()));
case VehicleEvent.Login lg -> b.setLogin( case VehicleEvent.Login lg -> b.setLogin(
com.lingniu.ingest.sink.mq.proto.LoginPayload.newBuilder() com.lingniu.ingest.sink.kafka.proto.LoginPayload.newBuilder()
.setIccid(nullToEmpty(lg.iccid())) .setIccid(nullToEmpty(lg.iccid()))
.setProtocolVersion(nullToEmpty(lg.protocolVersion())) .setProtocolVersion(nullToEmpty(lg.protocolVersion()))
.build()); .build());
case VehicleEvent.Logout ignored -> b.setLogout( case VehicleEvent.Logout ignored -> b.setLogout(
com.lingniu.ingest.sink.mq.proto.LogoutPayload.getDefaultInstance()); com.lingniu.ingest.sink.kafka.proto.LogoutPayload.getDefaultInstance());
case VehicleEvent.Heartbeat ignored -> b.setHeartbeat( case VehicleEvent.Heartbeat ignored -> b.setHeartbeat(
com.lingniu.ingest.sink.mq.proto.HeartbeatPayload.getDefaultInstance()); com.lingniu.ingest.sink.kafka.proto.HeartbeatPayload.getDefaultInstance());
case VehicleEvent.MediaMeta m -> b.setMediaMeta( case VehicleEvent.MediaMeta m -> b.setMediaMeta(
com.lingniu.ingest.sink.mq.proto.MediaMetaPayload.newBuilder() com.lingniu.ingest.sink.kafka.proto.MediaMetaPayload.newBuilder()
.setMediaId(nullToEmpty(m.mediaId())) .setMediaId(nullToEmpty(m.mediaId()))
.setMediaType(nullToEmpty(m.mediaType())) .setMediaType(nullToEmpty(m.mediaType()))
.setSizeBytes(m.sizeBytes()) .setSizeBytes(m.sizeBytes())
.setArchiveRef(nullToEmpty(m.archiveRef())) .setArchiveRef(nullToEmpty(m.archiveRef()))
.build()); .build());
case VehicleEvent.Passthrough p -> b.setPassthrough( case VehicleEvent.Passthrough p -> b.setPassthrough(
com.lingniu.ingest.sink.mq.proto.PassthroughPayload.newBuilder() com.lingniu.ingest.sink.kafka.proto.PassthroughPayload.newBuilder()
.setPassthroughType(p.passthroughType()) .setPassthroughType(p.passthroughType())
.setData(com.google.protobuf.ByteString.copyFrom( .setData(com.google.protobuf.ByteString.copyFrom(
p.data() == null ? new byte[0] : p.data())) p.data() == null ? new byte[0] : p.data()))
@@ -88,13 +88,13 @@ public final class EnvelopeMapper {
b.putMetadata(RawArchiveKeys.META_KEY, key); b.putMetadata(RawArchiveKeys.META_KEY, key);
b.putMetadata(RawArchiveKeys.META_URI, uri); b.putMetadata(RawArchiveKeys.META_URI, uri);
b.putMetadata(RawArchiveKeys.META_EVENT_ID, ra.eventId()); b.putMetadata(RawArchiveKeys.META_EVENT_ID, ra.eventId());
b.setRawArchive(com.lingniu.ingest.sink.mq.proto.RawArchiveRef.newBuilder() b.setRawArchive(com.lingniu.ingest.sink.kafka.proto.RawArchiveRef.newBuilder()
.setUri(uri) .setUri(uri)
.setChecksum(checksum) .setChecksum(checksum)
.setSizeBytes(size) .setSizeBytes(size)
.setParsedJson(nullToEmpty(ra.parsedJson())) .setParsedJson(nullToEmpty(ra.parsedJson()))
.build()); .build());
b.setRawFrameFact(com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload.newBuilder() b.setRawFrameFact(com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload.newBuilder()
.setFrameId(frameId) .setFrameId(frameId)
.setVehicleKey(vehicleKey) .setVehicleKey(vehicleKey)
.setVin(nullToEmpty(ra.vin())) .setVin(nullToEmpty(ra.vin()))
@@ -114,8 +114,8 @@ public final class EnvelopeMapper {
return b.build(); return b.build();
} }
private static com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot buildTelemetrySnapshot(TelemetrySnapshot snapshot) { private static com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot buildTelemetrySnapshot(TelemetrySnapshot snapshot) {
Builder b = com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot.newBuilder() Builder b = com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot.newBuilder()
.setEventType(snapshot.eventType()) .setEventType(snapshot.eventType())
.setRawArchiveUri(snapshot.rawArchiveUri()); .setRawArchiveUri(snapshot.rawArchiveUri());
for (TelemetryFieldValue field : snapshot.fields()) { for (TelemetryFieldValue field : snapshot.fields()) {
@@ -131,8 +131,8 @@ public final class EnvelopeMapper {
return b.build(); return b.build();
} }
private static com.lingniu.ingest.sink.mq.proto.RealtimePayload buildRealtime(RealtimePayload p) { private static com.lingniu.ingest.sink.kafka.proto.RealtimePayload buildRealtime(RealtimePayload p) {
var b = com.lingniu.ingest.sink.mq.proto.RealtimePayload.newBuilder(); var b = com.lingniu.ingest.sink.kafka.proto.RealtimePayload.newBuilder();
if (p.speedKmh() != null) b.setSpeedKmh(p.speedKmh()); if (p.speedKmh() != null) b.setSpeedKmh(p.speedKmh());
if (p.totalMileageKm() != null) b.setTotalMileageKm(p.totalMileageKm()); if (p.totalMileageKm() != null) b.setTotalMileageKm(p.totalMileageKm());
if (p.batterySoc() != null) b.setBatterySoc(p.batterySoc()); if (p.batterySoc() != null) b.setBatterySoc(p.batterySoc());
@@ -159,8 +159,8 @@ public final class EnvelopeMapper {
return b.build(); return b.build();
} }
private static com.lingniu.ingest.sink.mq.proto.LocationPayload buildLocation(LocationPayload p) { private static com.lingniu.ingest.sink.kafka.proto.LocationPayload buildLocation(LocationPayload p) {
return com.lingniu.ingest.sink.mq.proto.LocationPayload.newBuilder() return com.lingniu.ingest.sink.kafka.proto.LocationPayload.newBuilder()
.setLongitude(p.longitude()) .setLongitude(p.longitude())
.setLatitude(p.latitude()) .setLatitude(p.latitude())
.setAltitudeM(p.altitudeM()) .setAltitudeM(p.altitudeM())
@@ -171,8 +171,8 @@ public final class EnvelopeMapper {
.build(); .build();
} }
private static com.lingniu.ingest.sink.mq.proto.AlarmPayload buildAlarm(AlarmPayload p) { private static com.lingniu.ingest.sink.kafka.proto.AlarmPayload buildAlarm(AlarmPayload p) {
var b = com.lingniu.ingest.sink.mq.proto.AlarmPayload.newBuilder() var b = com.lingniu.ingest.sink.kafka.proto.AlarmPayload.newBuilder()
.setLevel(p.level().name()) .setLevel(p.level().name())
.setAlarmTypeCode(p.alarmTypeCode()) .setAlarmTypeCode(p.alarmTypeCode())
.setAlarmTypeName(nullToEmpty(p.alarmTypeName())); .setAlarmTypeName(nullToEmpty(p.alarmTypeName()));

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -28,15 +28,15 @@ public final class KafkaEnvelopeConsumerFactory {
} }
public List<KafkaEnvelopeConsumerWorker> createWorkers(Map<String, EnvelopeConsumerProcessor> processors, public List<KafkaEnvelopeConsumerWorker> createWorkers(Map<String, EnvelopeConsumerProcessor> processors,
SinkMqProperties props) { KafkaSinkProperties props) {
Map<String, SinkMqProperties.Binding> bindings = effectiveBindings(props); Map<String, KafkaSinkProperties.Binding> bindings = effectiveBindings(props);
List<KafkaEnvelopeConsumerWorker> workers = new ArrayList<>(); List<KafkaEnvelopeConsumerWorker> workers = new ArrayList<>();
for (Map.Entry<String, SinkMqProperties.Binding> entry : bindings.entrySet()) { for (Map.Entry<String, KafkaSinkProperties.Binding> entry : bindings.entrySet()) {
String processorBeanName = entry.getKey(); String processorBeanName = entry.getKey();
// binding key 必须和 Spring Bean 名一致这样配置只声明 topic/group // binding key 必须和 Spring Bean 名一致这样配置只声明 topic/group
// 实际处理逻辑仍由各业务模块自己的 EnvelopeConsumerProcessor 承接 // 实际处理逻辑仍由各业务模块自己的 EnvelopeConsumerProcessor 承接
EnvelopeConsumerProcessor processor = processors.get(processorBeanName); EnvelopeConsumerProcessor processor = processors.get(processorBeanName);
SinkMqProperties.Binding binding = entry.getValue(); KafkaSinkProperties.Binding binding = entry.getValue();
if (processor == null || binding == null || !binding.isEnabled()) { if (processor == null || binding == null || !binding.isEnabled()) {
continue; continue;
} }
@@ -57,15 +57,15 @@ public final class KafkaEnvelopeConsumerFactory {
return workers; return workers;
} }
private Map<String, SinkMqProperties.Binding> effectiveBindings(SinkMqProperties props) { private Map<String, KafkaSinkProperties.Binding> effectiveBindings(KafkaSinkProperties props) {
Map<String, SinkMqProperties.Binding> configured = props.getConsumer().getBindings(); Map<String, KafkaSinkProperties.Binding> configured = props.getConsumer().getBindings();
if (configured != null && !configured.isEmpty()) { if (configured != null && !configured.isEmpty()) {
return configured; return configured;
} }
// 默认绑定仅给未显式配置 bindings 的轻量运行时兜底 // 默认绑定仅给未显式配置 bindings 的轻量运行时兜底
// 生产 history app 会显式绑定各协议 event/raw topic并写入 TDengine raw_frames/locations // 生产 history app 会显式绑定各协议 event/raw topic并写入 TDengine raw_frames/locations
SinkMqProperties.Topics topics = props.getTopics(); KafkaSinkProperties.Topics topics = props.getTopics();
Map<String, SinkMqProperties.Binding> defaults = new LinkedHashMap<>(); Map<String, KafkaSinkProperties.Binding> defaults = new LinkedHashMap<>();
defaults.put("eventHistoryEnvelopeConsumerProcessor", binding( defaults.put("eventHistoryEnvelopeConsumerProcessor", binding(
"vehicle-event-history", "vehicle-event-history",
topics.getRealtime(), topics.getLocation(), topics.getAlarm(), topics.getSession(), topics.getMediaMeta())); topics.getRealtime(), topics.getLocation(), topics.getAlarm(), topics.getSession(), topics.getMediaMeta()));
@@ -78,8 +78,8 @@ public final class KafkaEnvelopeConsumerFactory {
return defaults; return defaults;
} }
private SinkMqProperties.Binding binding(String groupId, String... topics) { private KafkaSinkProperties.Binding binding(String groupId, String... topics) {
SinkMqProperties.Binding binding = new SinkMqProperties.Binding(); KafkaSinkProperties.Binding binding = new KafkaSinkProperties.Binding();
binding.setGroupId(groupId); binding.setGroupId(groupId);
binding.setTopics(List.of(topics)); binding.setTopics(List.of(topics));
return binding; return binding;
@@ -106,12 +106,12 @@ public final class KafkaEnvelopeConsumerFactory {
return List.copyOf(clean); return List.copyOf(clean);
} }
private Properties consumerProperties(SinkMqProperties props, private Properties consumerProperties(KafkaSinkProperties props,
SinkMqProperties.Binding binding, KafkaSinkProperties.Binding binding,
String processorBeanName, String processorBeanName,
int workerIndex, int workerIndex,
int concurrency) { int concurrency) {
SinkMqProperties.Consumer consumer = props.getConsumer(); KafkaSinkProperties.Consumer consumer = props.getConsumer();
Properties p = new Properties(); Properties p = new Properties();
p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers()); p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers());
p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
@@ -133,7 +133,7 @@ public final class KafkaEnvelopeConsumerFactory {
return concurrency <= 1 ? base : base + "-" + workerIndex; return concurrency <= 1 ? base : base + "-" + workerIndex;
} }
private String groupId(SinkMqProperties.Binding binding, String processorBeanName) { private String groupId(KafkaSinkProperties.Binding binding, String processorBeanName) {
if (binding.getGroupId() != null && !binding.getGroupId().isBlank()) { if (binding.getGroupId() != null && !binding.getGroupId().isBlank()) {
return binding.getGroupId(); return binding.getGroupId();
} }

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord; import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord; import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink; import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.event.VehicleEvent; import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.api.sink.EventSink; import com.lingniu.ingest.api.sink.EventSink;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import io.github.resilience4j.circuitbreaker.CircuitBreaker; import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig; import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
@@ -16,40 +16,31 @@ import java.time.Duration;
import java.util.Properties; import java.util.Properties;
/** /**
* MQ Sink 自动装配 * Kafka Sink 自动装配
* *
* <p>装配前提两个条件都要满足 * <p>{@code lingniu.ingest.sink.kafka.enabled=false} 时本模块完全不装配Producer Consumer
* <ol> * 是两个独立开关消费端还需要开启 {@code lingniu.ingest.sink.kafka.consumer.enabled=true}
* <li>{@code lingniu.ingest.sink.mq.enabled=true}默认 true <b>总开关</b> * 并配置 bindings
* 设为 false 时本模块完全不装配ingest-core DisruptorEventBus 仍然运行但
* 没有 Kafka sink
* <li>{@code lingniu.ingest.sink.mq.type=kafka}默认 kafka 唯一生产 MQ 后端
* </ol>
*
* <p>Producer Consumer 是两个独立开关{@code sink.mq.enabled=true} 只表示可以创建
* Kafka producer/sink是否从 Kafka envelope 还要单独开启
* {@code lingniu.ingest.sink.mq.consumer.enabled=true} 并配置 bindings
*/ */
@AutoConfiguration @AutoConfiguration
@EnableConfigurationProperties(SinkMqProperties.class) @EnableConfigurationProperties(KafkaSinkProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true) @ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka", name = "enabled", havingValue = "true", matchIfMissing = true)
public class SinkMqAutoConfiguration { public class KafkaSinkAutoConfiguration {
@Bean @Bean
@ConditionalOnMissingBean @ConditionalOnMissingBean
public EnvelopeMapper envelopeMapper(SinkMqProperties props) { public EnvelopeMapper envelopeMapper(KafkaSinkProperties props) {
return new EnvelopeMapper(props.getNodeId()); return new EnvelopeMapper(props.getNodeId());
} }
@Bean @Bean
@ConditionalOnMissingBean @ConditionalOnMissingBean
public TopicRouter topicRouter(SinkMqProperties props) { public TopicRouter topicRouter(KafkaSinkProperties props) {
return new TopicRouter(props.getTopics()); return new TopicRouter(props.getTopics());
} }
@Bean @Bean
@ConditionalOnMissingBean @ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "type", havingValue = "kafka", matchIfMissing = true)
public CircuitBreaker kafkaSinkCircuitBreaker() { public CircuitBreaker kafkaSinkCircuitBreaker() {
return CircuitBreaker.of("kafka-sink", CircuitBreakerConfig.custom() return CircuitBreaker.of("kafka-sink", CircuitBreakerConfig.custom()
.slidingWindowSize(100) .slidingWindowSize(100)
@@ -61,8 +52,7 @@ public class SinkMqAutoConfiguration {
@Bean(destroyMethod = "close") @Bean(destroyMethod = "close")
@ConditionalOnMissingBean @ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "type", havingValue = "kafka", matchIfMissing = true) public KafkaProducer<String, byte[]> kafkaProducer(KafkaSinkProperties props) {
public KafkaProducer<String, byte[]> kafkaProducer(SinkMqProperties props) {
Properties p = new Properties(); Properties p = new Properties();
p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers()); p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers());
p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
@@ -77,11 +67,10 @@ public class SinkMqAutoConfiguration {
@Bean(destroyMethod = "close") @Bean(destroyMethod = "close")
@ConditionalOnMissingBean @ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "type", havingValue = "kafka", matchIfMissing = true)
public KafkaEventSink kafkaEventSink(KafkaProducer<String, byte[]> producer, public KafkaEventSink kafkaEventSink(KafkaProducer<String, byte[]> producer,
EnvelopeMapper mapper, EnvelopeMapper mapper,
TopicRouter router, TopicRouter router,
SinkMqProperties props, KafkaSinkProperties props,
CircuitBreaker breaker) { CircuitBreaker breaker) {
// KafkaEventSink EventBus 的生产端出口它不会启动任何 Kafka 消费线程 // KafkaEventSink EventBus 的生产端出口它不会启动任何 Kafka 消费线程
return new KafkaEventSink(producer, mapper, router, props.getTopics().getDlq(), breaker); return new KafkaEventSink(producer, mapper, router, props.getTopics().getDlq(), breaker);
@@ -89,9 +78,8 @@ public class SinkMqAutoConfiguration {
@Bean @Bean
@ConditionalOnMissingBean @ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "type", havingValue = "kafka", matchIfMissing = true)
public KafkaEnvelopeDeadLetterSink kafkaEnvelopeDeadLetterSink(KafkaProducer<String, byte[]> producer, public KafkaEnvelopeDeadLetterSink kafkaEnvelopeDeadLetterSink(KafkaProducer<String, byte[]> producer,
SinkMqProperties props) { KafkaSinkProperties props) {
return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq()); return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq());
} }

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import org.springframework.beans.factory.ListableBeanFactory; import org.springframework.beans.factory.ListableBeanFactory;
@@ -13,15 +13,15 @@ import java.time.Duration;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@AutoConfiguration(after = SinkMqAutoConfiguration.class) @AutoConfiguration(after = KafkaSinkAutoConfiguration.class)
@AutoConfigureAfter(name = { @AutoConfigureAfter(name = {
"com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration", "com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration",
"com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration", "com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration",
"com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration" "com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration"
}) })
@EnableConfigurationProperties(SinkMqProperties.class) @EnableConfigurationProperties(KafkaSinkProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true) @ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka", name = "enabled", havingValue = "true", matchIfMissing = true)
public class SinkMqConsumerAutoConfiguration { public class KafkaSinkConsumerAutoConfiguration {
@Bean @Bean
@ConditionalOnMissingBean @ConditionalOnMissingBean
@@ -31,10 +31,10 @@ public class SinkMqConsumerAutoConfiguration {
@Bean @Bean
@ConditionalOnMissingBean @ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory, public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory,
KafkaEnvelopeConsumerFactory consumerFactory, KafkaEnvelopeConsumerFactory consumerFactory,
SinkMqProperties props) { KafkaSinkProperties props) {
return new KafkaEnvelopeConsumerRunner( return new KafkaEnvelopeConsumerRunner(
() -> createWorkers(beanFactory, consumerFactory, props), () -> createWorkers(beanFactory, consumerFactory, props),
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()), Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
@@ -44,7 +44,7 @@ public class SinkMqConsumerAutoConfiguration {
private List<KafkaEnvelopeConsumerWorker> createWorkers(ListableBeanFactory beanFactory, private List<KafkaEnvelopeConsumerWorker> createWorkers(ListableBeanFactory beanFactory,
KafkaEnvelopeConsumerFactory consumerFactory, KafkaEnvelopeConsumerFactory consumerFactory,
SinkMqProperties props) { KafkaSinkProperties props) {
Map<String, EnvelopeConsumerProcessor> processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class); Map<String, EnvelopeConsumerProcessor> processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class);
return consumerFactory.createWorkers(processors, props); return consumerFactory.createWorkers(processors, props);
} }

View File

@@ -1,26 +1,23 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Map; import java.util.Map;
@ConfigurationProperties(prefix = "lingniu.ingest.sink.mq") @ConfigurationProperties(prefix = "lingniu.ingest.sink.kafka")
public class SinkMqProperties { public class KafkaSinkProperties {
/** /**
* MQ Sink 总开关默认 {@code true} * Kafka Sink 总开关默认 {@code true}
* 设为 {@code false} {@link SinkMqAutoConfiguration} 完全不装配任何 BeanKafka Producer * 设为 {@code false} {@link KafkaSinkAutoConfiguration} 完全不装配任何 BeanKafka Producer
* EnvelopeMapperTopicRouterKafkaEventSink 都不会创建ingest-core DisruptorEventBus * EnvelopeMapperTopicRouterKafkaEventSink 都不会创建ingest-core DisruptorEventBus
* 仍然正常运行但没有外部 sink事件落地到 sink-archive Noop 吞掉 * 仍然正常运行但没有外部 sink事件落地到 sink-archive Noop 吞掉
*/ */
private boolean enabled = true; private boolean enabled = true;
/** MQ 后端类型。生产链路只允许 kafka。 */
private String type = "kafka";
/** Kafka bootstrap servers生产环境应通过环境变量覆盖不建议使用默认开发地址。 */ /** Kafka bootstrap servers生产环境应通过环境变量覆盖不建议使用默认开发地址。 */
private String bootstrapServers = "114.55.58.251:9092"; private String bootstrapServers = "114.55.58.251:9092";
private String compressionType = "zstd"; private String compressionType = "zstd";
@@ -35,14 +32,6 @@ public class SinkMqProperties {
public boolean isEnabled() { return enabled; } public boolean isEnabled() { return enabled; }
public void setEnabled(boolean enabled) { this.enabled = enabled; } public void setEnabled(boolean enabled) { this.enabled = enabled; }
public String getType() { return type; }
public void setType(String type) {
String value = type == null || type.isBlank() ? "kafka" : type.trim().toLowerCase(Locale.ROOT);
if (!"kafka".equals(value)) {
throw new IllegalStateException("sink.mq.type supports only kafka; configured value: " + type);
}
this.type = value;
}
public String getBootstrapServers() { return bootstrapServers; } public String getBootstrapServers() { return bootstrapServers; }
public void setBootstrapServers(String bootstrapServers) { this.bootstrapServers = bootstrapServers; } public void setBootstrapServers(String bootstrapServers) { this.bootstrapServers = bootstrapServers; }
public String getCompressionType() { return compressionType; } public String getCompressionType() { return compressionType; }

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.event.VehicleEvent; import com.lingniu.ingest.api.event.VehicleEvent;
@@ -7,9 +7,9 @@ import com.lingniu.ingest.api.event.VehicleEvent;
*/ */
public final class TopicRouter { public final class TopicRouter {
private final SinkMqProperties.Topics topics; private final KafkaSinkProperties.Topics topics;
public TopicRouter(SinkMqProperties.Topics topics) { public TopicRouter(KafkaSinkProperties.Topics topics) {
this.topics = topics; this.topics = topics;
} }

View File

@@ -1,9 +1,9 @@
syntax = "proto3"; syntax = "proto3";
package com.lingniu.ingest.sink.mq.proto; package com.lingniu.ingest.sink.kafka.proto;
option java_multiple_files = true; option java_multiple_files = true;
option java_package = "com.lingniu.ingest.sink.mq.proto"; option java_package = "com.lingniu.ingest.sink.kafka.proto";
option java_outer_classname = "VehicleEnvelopeProto"; option java_outer_classname = "VehicleEnvelopeProto";
// 统一消息外壳:所有 Topic 共用此 Envelopepayload 通过 oneof 区分具体事件类型。 // 统一消息外壳:所有 Topic 共用此 Envelopepayload 通过 oneof 区分具体事件类型。

View File

@@ -1,2 +1,2 @@
com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration
com.lingniu.ingest.sink.mq.SinkMqConsumerAutoConfiguration com.lingniu.ingest.sink.kafka.KafkaSinkConsumerAutoConfiguration

View File

@@ -1,10 +1,10 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.RawArchiveKeys; import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.event.RealtimePayload; import com.lingniu.ingest.api.event.RealtimePayload;
import com.lingniu.ingest.api.event.VehicleEvent; import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.time.Instant; import java.time.Instant;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
@@ -23,7 +23,7 @@ class KafkaEnvelopeConsumerFactoryTest {
created.add(props); created.add(props);
return new MockConsumer<>(OffsetResetStrategy.EARLIEST); return new MockConsumer<>(OffsetResetStrategy.EARLIEST);
}); });
SinkMqProperties props = new SinkMqProperties(); KafkaSinkProperties props = new KafkaSinkProperties();
props.setBootstrapServers("kafka-1:9092"); props.setBootstrapServers("kafka-1:9092");
EnvelopeConsumerProcessor stateProcessor = processor(); EnvelopeConsumerProcessor stateProcessor = processor();
EnvelopeConsumerProcessor statProcessor = processor(); EnvelopeConsumerProcessor statProcessor = processor();
@@ -51,7 +51,7 @@ class KafkaEnvelopeConsumerFactoryTest {
created.add(props); created.add(props);
return new MockConsumer<>(OffsetResetStrategy.EARLIEST); return new MockConsumer<>(OffsetResetStrategy.EARLIEST);
}); });
SinkMqProperties props = new SinkMqProperties(); KafkaSinkProperties props = new KafkaSinkProperties();
props.getConsumer().setConcurrency(3); props.getConsumer().setConcurrency(3);
EnvelopeConsumerProcessor stateProcessor = processor(); EnvelopeConsumerProcessor stateProcessor = processor();

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord; import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor; import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord; import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.LocationPayload; import com.lingniu.ingest.api.event.LocationPayload;
@@ -26,7 +26,7 @@ class KafkaEventSinkTest {
KafkaEventSink sink = new KafkaEventSink( KafkaEventSink sink = new KafkaEventSink(
null, null,
new EnvelopeMapper("node-1"), new EnvelopeMapper("node-1"),
new TopicRouter(new SinkMqProperties.Topics()), new TopicRouter(new KafkaSinkProperties.Topics()),
"vehicle.dlq.gb32960.v1", "vehicle.dlq.gb32960.v1",
CircuitBreaker.ofDefaults("kafka-test")); CircuitBreaker.ofDefaults("kafka-test"));
@@ -46,7 +46,7 @@ class KafkaEventSinkTest {
KafkaEventSink sink = new KafkaEventSink( KafkaEventSink sink = new KafkaEventSink(
producer, producer,
new EnvelopeMapper("node-1"), new EnvelopeMapper("node-1"),
new TopicRouter(new SinkMqProperties.Topics()), new TopicRouter(new KafkaSinkProperties.Topics()),
"vehicle.dlq.gb32960.v1", "vehicle.dlq.gb32960.v1",
CircuitBreaker.ofDefaults("kafka-test")); CircuitBreaker.ofDefaults("kafka-test"));

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
@@ -15,26 +15,25 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
@ExtendWith(OutputCaptureExtension.class) @ExtendWith(OutputCaptureExtension.class)
class SinkMqConsumerAutoConfigurationTest { class KafkaSinkConsumerAutoConfigurationTest {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(SinkMqAutoConfiguration.class, SinkMqConsumerAutoConfiguration.class) .withUserConfiguration(KafkaSinkAutoConfiguration.class, KafkaSinkConsumerAutoConfiguration.class)
.withAllowBeanDefinitionOverriding(true) .withAllowBeanDefinitionOverriding(true)
.withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class, .withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
() -> new EnvelopeConsumerProcessor( () -> new EnvelopeConsumerProcessor(
"vehicle-state", "vehicle-state",
bytes -> EnvelopeIngestResult.processed("evt-1", "VIN001"), bytes -> EnvelopeIngestResult.processed("evt-1", "VIN001"),
record -> {})) record -> {}))
.withBean("kafkaProducer", KafkaProducer.class, SinkMqConsumerAutoConfigurationTest::kafkaProducer) .withBean("kafkaProducer", KafkaProducer.class, KafkaSinkConsumerAutoConfigurationTest::kafkaProducer)
.withBean(KafkaEnvelopeConsumerFactory.class, .withBean(KafkaEnvelopeConsumerFactory.class,
() -> new KafkaEnvelopeConsumerFactory(props -> new MockConsumer<>(OffsetResetStrategy.EARLIEST))) () -> new KafkaEnvelopeConsumerFactory(props -> new MockConsumer<>(OffsetResetStrategy.EARLIEST)))
.withPropertyValues( .withPropertyValues(
"lingniu.ingest.sink.mq.enabled=true", "lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.mq.type=kafka", "lingniu.ingest.sink.kafka.consumer.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=true", "lingniu.ingest.sink.kafka.consumer.auto-startup=false",
"lingniu.ingest.sink.mq.consumer.auto-startup=false", "lingniu.ingest.sink.kafka.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.group-id=vehicle-state",
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.group-id=vehicle-state", "lingniu.ingest.sink.kafka.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.topics[0]=vehicle.realtime");
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.topics[0]=vehicle.realtime");
@Test @Test
void createsKafkaEnvelopeConsumerRunnerWhenConsumerBindingIsConfigured(CapturedOutput output) { void createsKafkaEnvelopeConsumerRunnerWhenConsumerBindingIsConfigured(CapturedOutput output) {

View File

@@ -0,0 +1,26 @@
package com.lingniu.ingest.sink.kafka;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import static org.assertj.core.api.Assertions.assertThat;
class KafkaSinkPropertiesTest {
@Test
void defaultKafkaBrokerUsesProductionAddressButCanStillBeOverriddenByConfigBinding() {
KafkaSinkProperties props = new KafkaSinkProperties();
assertThat(props.getBootstrapServers()).isEqualTo("114.55.58.251:9092");
props.setBootstrapServers("kafka.internal:9092");
assertThat(props.getBootstrapServers()).isEqualTo("kafka.internal:9092");
}
@Test
void exposesNoBackendTypeBecauseKafkaIsTheOnlySink() {
assertThat(Arrays.stream(KafkaSinkProperties.class.getMethods()).map(method -> method.getName()))
.doesNotContain("getType", "setType");
}
}

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.AlarmPayload; import com.lingniu.ingest.api.event.AlarmPayload;
@@ -18,14 +18,14 @@ class TopicRouterTest {
@Test @Test
void rawArchiveRoutesToVersionedGb32960RawTopicByDefault() { void rawArchiveRoutesToVersionedGb32960RawTopicByDefault() {
TopicRouter router = new TopicRouter(new SinkMqProperties.Topics()); TopicRouter router = new TopicRouter(new KafkaSinkProperties.Topics());
assertThat(router.route(rawArchive())).isEqualTo("vehicle.raw.gb32960.v1"); assertThat(router.route(rawArchive())).isEqualTo("vehicle.raw.gb32960.v1");
} }
@Test @Test
void normalizedGb32960EventsRouteToVersionedEventTopicByDefault() { void normalizedGb32960EventsRouteToVersionedEventTopicByDefault() {
TopicRouter router = new TopicRouter(new SinkMqProperties.Topics()); TopicRouter router = new TopicRouter(new KafkaSinkProperties.Topics());
assertThat(router.route(realtime())).isEqualTo("vehicle.event.gb32960.v1"); assertThat(router.route(realtime())).isEqualTo("vehicle.event.gb32960.v1");
assertThat(router.route(location())).isEqualTo("vehicle.event.gb32960.v1"); assertThat(router.route(location())).isEqualTo("vehicle.event.gb32960.v1");
@@ -37,7 +37,7 @@ class TopicRouterTest {
@Test @Test
void dlqDefaultsToVersionedGb32960DlqTopic() { void dlqDefaultsToVersionedGb32960DlqTopic() {
SinkMqProperties.Topics topics = new SinkMqProperties.Topics(); KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
assertThat(topics.getDlq()).isEqualTo("vehicle.dlq.gb32960.v1"); assertThat(topics.getDlq()).isEqualTo("vehicle.dlq.gb32960.v1");
} }

View File

@@ -1,9 +1,9 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.sink.mq.proto.DecodedFactPayload; import com.lingniu.ingest.sink.kafka.proto.DecodedFactPayload;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto; import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload; import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;

View File

@@ -1,29 +0,0 @@
package com.lingniu.ingest.sink.mq;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
class SinkMqPropertiesTest {
@Test
void defaultKafkaBrokerUsesProductionAddressButCanStillBeOverriddenByConfigBinding() {
SinkMqProperties props = new SinkMqProperties();
assertThat(props.getBootstrapServers()).isEqualTo("114.55.58.251:9092");
props.setBootstrapServers("kafka.internal:9092");
assertThat(props.getBootstrapServers()).isEqualTo("kafka.internal:9092");
}
@Test
void rejectsUnsupportedMqTypeInsteadOfSilentlyDisablingKafkaBeans() {
SinkMqProperties props = new SinkMqProperties();
assertThatThrownBy(() -> props.setType("rocketmq"))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("sink.mq.type supports only kafka")
.hasMessageContaining("rocketmq");
}
}

View File

@@ -2,10 +2,10 @@ package com.lingniu.ingest.tdenginehistory;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.sink.mq.proto.LocationPayload; import com.lingniu.ingest.sink.kafka.proto.LocationPayload;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload; import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import java.time.Instant; import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;

View File

@@ -1,12 +1,12 @@
package com.lingniu.ingest.tdenginehistory; package com.lingniu.ingest.tdenginehistory;
import com.lingniu.ingest.sink.mq.proto.LocationPayload; import com.lingniu.ingest.sink.kafka.proto.LocationPayload;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto; import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef; import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload; import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.time.Instant; import java.time.Instant;