refactor: expose kafka sink configuration
This commit is contained in:
@@ -18,7 +18,7 @@
|
|||||||
- Existing handler currently writes report ACK before `dispatcher.dispatch(rf)`.
|
- Existing handler currently writes report ACK before `dispatcher.dispatch(rf)`.
|
||||||
- Existing `DisruptorEventBus` publishes to sinks asynchronously and does not wait for sink completion.
|
- Existing `DisruptorEventBus` publishes to sinks asynchronously and does not wait for sink completion.
|
||||||
- Existing `KafkaEventSink.accepts()` rejects `VehicleEvent.RawArchive`.
|
- Existing `KafkaEventSink.accepts()` rejects `VehicleEvent.RawArchive`.
|
||||||
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.mq.topics`.
|
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.kafka.topics`.
|
||||||
- Existing history consumer logic: `modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java`.
|
- Existing history consumer logic: `modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java`.
|
||||||
- Existing analytics consumers: `VehicleStateEnvelopeIngestor` and `VehicleStatEnvelopeIngestor`.
|
- Existing analytics consumers: `VehicleStateEnvelopeIngestor` and `VehicleStatEnvelopeIngestor`.
|
||||||
- There is an unrelated working-tree change in `modules/apps/bootstrap-all/src/main/resources/application.yml` for GB32960 diagnostics capacity. Do not revert it. Do not include it in service-split commits unless the user explicitly asks.
|
- There is an unrelated working-tree change in `modules/apps/bootstrap-all/src/main/resources/application.yml` for GB32960 diagnostics capacity. Do not revert it. Do not include it in service-split commits unless the user explicitly asks.
|
||||||
@@ -48,7 +48,7 @@ Create app modules:
|
|||||||
Modify shared build and Kafka contract files:
|
Modify shared build and Kafka contract files:
|
||||||
|
|
||||||
- `pom.xml`
|
- `pom.xml`
|
||||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
|
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java`
|
||||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
|
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
|
||||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
|
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
|
||||||
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
|
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
|
||||||
@@ -466,16 +466,19 @@ lingniu:
|
|||||||
rate-limit:
|
rate-limit:
|
||||||
per-vin-qps: 50
|
per-vin-qps: 50
|
||||||
session:
|
session:
|
||||||
store: ${SESSION_STORE:memory}
|
store: ${SESSION_STORE:redis}
|
||||||
ttl: ${SESSION_TTL:30m}
|
ttl: ${SESSION_TTL:30m}
|
||||||
identity:
|
identity:
|
||||||
store: ${VEHICLE_IDENTITY_STORE:file}
|
store: ${VEHICLE_IDENTITY_STORE:mysql}
|
||||||
file:
|
mysql:
|
||||||
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
|
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||||
|
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||||
|
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
|
||||||
|
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||||
|
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
compression-type: zstd
|
compression-type: zstd
|
||||||
linger-ms: 20
|
linger-ms: 20
|
||||||
@@ -568,9 +571,8 @@ lingniu:
|
|||||||
gb32960:
|
gb32960:
|
||||||
enabled: false
|
enabled: false
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
topics:
|
topics:
|
||||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||||
@@ -666,9 +668,8 @@ lingniu:
|
|||||||
gb32960:
|
gb32960:
|
||||||
enabled: false
|
enabled: false
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
topics:
|
topics:
|
||||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||||
@@ -754,7 +755,7 @@ import com.lingniu.ingest.eventfilestore.EventFileStore;
|
|||||||
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
||||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
||||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||||
@@ -770,7 +771,7 @@ class Gb32960IngestAppCompositionTest {
|
|||||||
.withPropertyValues(
|
.withPropertyValues(
|
||||||
"lingniu.ingest.gb32960.enabled=true",
|
"lingniu.ingest.gb32960.enabled=true",
|
||||||
"lingniu.ingest.gb32960.port=0",
|
"lingniu.ingest.gb32960.port=0",
|
||||||
"lingniu.ingest.sink.mq.enabled=false",
|
"lingniu.ingest.sink.kafka.enabled=false",
|
||||||
"lingniu.ingest.sink.archive.enabled=false",
|
"lingniu.ingest.sink.archive.enabled=false",
|
||||||
"lingniu.ingest.event-file-store.enabled=false",
|
"lingniu.ingest.event-file-store.enabled=false",
|
||||||
"lingniu.ingest.event-history.enabled=false",
|
"lingniu.ingest.event-history.enabled=false",
|
||||||
@@ -835,7 +836,7 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
"lingniu.ingest.event-history.enabled=true",
|
"lingniu.ingest.event-history.enabled=true",
|
||||||
"lingniu.ingest.vehicle-state.enabled=false",
|
"lingniu.ingest.vehicle-state.enabled=false",
|
||||||
"lingniu.ingest.vehicle-stat.enabled=false",
|
"lingniu.ingest.vehicle-stat.enabled=false",
|
||||||
"lingniu.ingest.sink.mq.enabled=false")
|
"lingniu.ingest.sink.kafka.enabled=false")
|
||||||
.withConfiguration(AutoConfigurations.of(
|
.withConfiguration(AutoConfigurations.of(
|
||||||
com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration.class,
|
com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration.class,
|
||||||
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration.class,
|
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration.class,
|
||||||
@@ -896,7 +897,7 @@ class VehicleAnalyticsAppCompositionTest {
|
|||||||
"lingniu.ingest.vehicle-state.enabled=false",
|
"lingniu.ingest.vehicle-state.enabled=false",
|
||||||
"lingniu.ingest.vehicle-stat.enabled=true",
|
"lingniu.ingest.vehicle-stat.enabled=true",
|
||||||
"lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"),
|
"lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"),
|
||||||
"lingniu.ingest.sink.mq.enabled=false")
|
"lingniu.ingest.sink.kafka.enabled=false")
|
||||||
.withConfiguration(AutoConfigurations.of(
|
.withConfiguration(AutoConfigurations.of(
|
||||||
com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration.class,
|
com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration.class,
|
||||||
com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration.class))
|
com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration.class))
|
||||||
@@ -932,7 +933,7 @@ git commit -m "test: cover split service composition"
|
|||||||
### Task 4: Version Kafka Topics for GB32960 Raw/Event/DLQ
|
### Task 4: Version Kafka Topics for GB32960 Raw/Event/DLQ
|
||||||
|
|
||||||
**Files:**
|
**Files:**
|
||||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
|
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java`
|
||||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
|
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
|
||||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
|
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
|
||||||
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
|
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
|
||||||
@@ -947,7 +948,7 @@ Expected test names:
|
|||||||
```java
|
```java
|
||||||
@Test
|
@Test
|
||||||
void rawArchiveRoutesToVersionedGb32960RawTopic() {
|
void rawArchiveRoutesToVersionedGb32960RawTopic() {
|
||||||
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
|
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
|
||||||
topics.setRawArchive("vehicle.raw.gb32960.v1");
|
topics.setRawArchive("vehicle.raw.gb32960.v1");
|
||||||
TopicRouter router = new TopicRouter(topics);
|
TopicRouter router = new TopicRouter(topics);
|
||||||
|
|
||||||
@@ -970,7 +971,7 @@ Also add a normalized realtime event test:
|
|||||||
```java
|
```java
|
||||||
@Test
|
@Test
|
||||||
void realtimeRoutesToVersionedGb32960EventTopic() {
|
void realtimeRoutesToVersionedGb32960EventTopic() {
|
||||||
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
|
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
|
||||||
topics.setRealtime("vehicle.event.gb32960.v1");
|
topics.setRealtime("vehicle.event.gb32960.v1");
|
||||||
TopicRouter router = new TopicRouter(topics);
|
TopicRouter router = new TopicRouter(topics);
|
||||||
|
|
||||||
@@ -1000,9 +1001,9 @@ mvn -pl :sink-mq -Dtest=TopicRouterTest test
|
|||||||
|
|
||||||
Expected before implementation: compile failure if constructors are wrong or assertion failure if defaults are old. Fix constructors first; keep routing assertions.
|
Expected before implementation: compile failure if constructors are wrong or assertion failure if defaults are old. Fix constructors first; keep routing assertions.
|
||||||
|
|
||||||
- [ ] **Step 3: Update `SinkMqProperties.Topics` defaults**
|
- [ ] **Step 3: Update `KafkaSinkProperties.Topics` defaults**
|
||||||
|
|
||||||
In `SinkMqProperties.Topics`, change defaults:
|
In `KafkaSinkProperties.Topics`, change defaults:
|
||||||
|
|
||||||
```java
|
```java
|
||||||
private String realtime = "vehicle.event.gb32960.v1";
|
private String realtime = "vehicle.event.gb32960.v1";
|
||||||
@@ -1040,7 +1041,7 @@ Expected: PASS.
|
|||||||
- [ ] **Step 6: Commit**
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java \
|
git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java \
|
||||||
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java \
|
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java \
|
||||||
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java \
|
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java \
|
||||||
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java \
|
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java \
|
||||||
@@ -1353,8 +1354,8 @@ void gb32960IngestDefaultsOnlyEnableProtocolAndKafkaProducer() {
|
|||||||
assertThat(props.getProperty("spring.application.name")).isEqualTo("gb32960-ingest-app");
|
assertThat(props.getProperty("spring.application.name")).isEqualTo("gb32960-ingest-app");
|
||||||
assertThat(props.getProperty("lingniu.ingest.gb32960.enabled")).isEqualTo("true");
|
assertThat(props.getProperty("lingniu.ingest.gb32960.enabled")).isEqualTo("true");
|
||||||
assertThat(props.getProperty("lingniu.ingest.gb32960.port")).isEqualTo("${GB32960_PORT:32960}");
|
assertThat(props.getProperty("lingniu.ingest.gb32960.port")).isEqualTo("${GB32960_PORT:32960}");
|
||||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
|
assertThat(props.getProperty("lingniu.ingest.sink.kafka.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
|
||||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.enabled")).isEqualTo("false");
|
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.enabled")).isEqualTo("false");
|
||||||
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
|
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
|
||||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
|
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
|
||||||
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
|
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
|
||||||
@@ -1378,7 +1379,7 @@ void historyDefaultsEnableStorageAndHistoryConsumerOnly() {
|
|||||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("true");
|
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("true");
|
||||||
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
|
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
|
||||||
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("false");
|
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("false");
|
||||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
|
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
|
||||||
.isEqualTo("${KAFKA_GROUP_HISTORY:vehicle-history}");
|
.isEqualTo("${KAFKA_GROUP_HISTORY:vehicle-history}");
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
@@ -1398,7 +1399,7 @@ void analyticsDefaultsEnableStatConsumerAndDisableProtocolAndHistoryStorage() {
|
|||||||
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
|
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
|
||||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
|
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
|
||||||
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("${VEHICLE_STAT_ENABLED:true}");
|
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("${VEHICLE_STAT_ENABLED:true}");
|
||||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
|
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
|
||||||
.isEqualTo("${KAFKA_GROUP_STAT:vehicle-stat}");
|
.isEqualTo("${KAFKA_GROUP_STAT:vehicle-stat}");
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -1269,12 +1269,12 @@ git commit -m "feat: add raw archive store contract"
|
|||||||
Create `VehicleEnvelopeProtoCompatibilityTest.java`:
|
Create `VehicleEnvelopeProtoCompatibilityTest.java`:
|
||||||
|
|
||||||
```java
|
```java
|
||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.DecodedFactPayload;
|
import com.lingniu.ingest.sink.kafka.proto.DecodedFactPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
|
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
|
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import static org.assertj.core.api.Assertions.assertThat;
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ Use versioned topics:
|
|||||||
- `vehicle.event.gb32960.v1`
|
- `vehicle.event.gb32960.v1`
|
||||||
- `vehicle.dlq.gb32960.v1`
|
- `vehicle.dlq.gb32960.v1`
|
||||||
|
|
||||||
The current topic names under `lingniu.ingest.sink.mq.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
|
The current topic names under `lingniu.ingest.sink.kafka.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
|
||||||
|
|
||||||
### Keys
|
### Keys
|
||||||
|
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ repositories, and statistic calculators.
|
|||||||
|
|
||||||
### Event Contract
|
### Event Contract
|
||||||
|
|
||||||
`sink-mq` owns the Kafka protobuf envelope and consumer plumbing.
|
The Kafka sink owns the protobuf envelope and consumer plumbing.
|
||||||
Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped
|
Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped
|
||||||
envelopes, and downstream failures are converted to structured results and DLQ
|
envelopes, and downstream failures are converted to structured results and DLQ
|
||||||
records instead of blocking a partition.
|
records instead of blocking a partition.
|
||||||
|
|||||||
@@ -99,9 +99,8 @@ lingniu:
|
|||||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
compression-type: zstd
|
compression-type: zstd
|
||||||
linger-ms: 20
|
linger-ms: 20
|
||||||
|
|||||||
@@ -11,9 +11,9 @@ import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
|
|||||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||||
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
||||||
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||||
@@ -31,7 +31,7 @@ class Gb32960IngestAppCompositionTest {
|
|||||||
IngestCoreAutoConfiguration.class,
|
IngestCoreAutoConfiguration.class,
|
||||||
SessionCoreAutoConfiguration.class,
|
SessionCoreAutoConfiguration.class,
|
||||||
VehicleIdentityAutoConfiguration.class,
|
VehicleIdentityAutoConfiguration.class,
|
||||||
SinkMqAutoConfiguration.class,
|
KafkaSinkAutoConfiguration.class,
|
||||||
SinkArchiveAutoConfiguration.class,
|
SinkArchiveAutoConfiguration.class,
|
||||||
Gb32960AutoConfiguration.class))
|
Gb32960AutoConfiguration.class))
|
||||||
.withAllowBeanDefinitionOverriding(true)
|
.withAllowBeanDefinitionOverriding(true)
|
||||||
@@ -43,10 +43,9 @@ class Gb32960IngestAppCompositionTest {
|
|||||||
"lingniu.ingest.gb32960.port=0",
|
"lingniu.ingest.gb32960.port=0",
|
||||||
"lingniu.ingest.identity.store=mysql",
|
"lingniu.ingest.identity.store=mysql",
|
||||||
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
||||||
"lingniu.ingest.sink.mq.enabled=true",
|
"lingniu.ingest.sink.kafka.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.type=kafka",
|
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
|
||||||
"lingniu.ingest.sink.archive.enabled=true",
|
"lingniu.ingest.sink.archive.enabled=true",
|
||||||
"lingniu.ingest.sink.archive.path=target/test-archive-gb32960",
|
"lingniu.ingest.sink.archive.path=target/test-archive-gb32960",
|
||||||
"lingniu.ingest.event-history.enabled=false");
|
"lingniu.ingest.event-history.enabled=false");
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ class Gb32960IngestAppDefaultsTest {
|
|||||||
@Test
|
@Test
|
||||||
void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() throws IOException {
|
void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() throws IOException {
|
||||||
Properties properties = applicationProperties();
|
Properties properties = applicationProperties();
|
||||||
|
String legacyKafkaPrefix = "lingniu.ingest.sink." + "mq.";
|
||||||
|
String kafkaTypeProperty = "lingniu.ingest.sink.kafka." + "type";
|
||||||
|
|
||||||
assertThat(properties)
|
assertThat(properties)
|
||||||
.containsEntry("spring.application.name", "gb32960-ingest-app")
|
.containsEntry("spring.application.name", "gb32960-ingest-app")
|
||||||
@@ -40,8 +42,8 @@ class Gb32960IngestAppDefaultsTest {
|
|||||||
"Hyundai")
|
"Hyundai")
|
||||||
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[1]",
|
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[1]",
|
||||||
"YueJin")
|
"YueJin")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
|
.containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
|
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
|
||||||
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
|
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
|
||||||
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
|
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
|
||||||
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
|
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
|
||||||
@@ -55,7 +57,9 @@ class Gb32960IngestAppDefaultsTest {
|
|||||||
assertThat(properties.stringPropertyNames())
|
assertThat(properties.stringPropertyNames())
|
||||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
|
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
|
||||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
|
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
|
||||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."));
|
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
|
||||||
|
.noneMatch(name -> name.startsWith(legacyKafkaPrefix))
|
||||||
|
.noneMatch(name -> name.equals(kafkaTypeProperty));
|
||||||
assertThat(applicationYaml())
|
assertThat(applicationYaml())
|
||||||
.doesNotContain("event-file-store:")
|
.doesNotContain("event-file-store:")
|
||||||
.doesNotContain("vehicle-state:")
|
.doesNotContain("vehicle-state:")
|
||||||
|
|||||||
@@ -63,9 +63,8 @@ lingniu:
|
|||||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
compression-type: zstd
|
compression-type: zstd
|
||||||
linger-ms: 20
|
linger-ms: 20
|
||||||
|
|||||||
@@ -11,9 +11,9 @@ import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
|
|||||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||||
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
||||||
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||||
@@ -31,7 +31,7 @@ class Jt808IngestAppCompositionTest {
|
|||||||
IngestCoreAutoConfiguration.class,
|
IngestCoreAutoConfiguration.class,
|
||||||
SessionCoreAutoConfiguration.class,
|
SessionCoreAutoConfiguration.class,
|
||||||
VehicleIdentityAutoConfiguration.class,
|
VehicleIdentityAutoConfiguration.class,
|
||||||
SinkMqAutoConfiguration.class,
|
KafkaSinkAutoConfiguration.class,
|
||||||
SinkArchiveAutoConfiguration.class,
|
SinkArchiveAutoConfiguration.class,
|
||||||
Jt808AutoConfiguration.class))
|
Jt808AutoConfiguration.class))
|
||||||
.withAllowBeanDefinitionOverriding(true)
|
.withAllowBeanDefinitionOverriding(true)
|
||||||
@@ -42,10 +42,9 @@ class Jt808IngestAppCompositionTest {
|
|||||||
"lingniu.ingest.jt808.port=0",
|
"lingniu.ingest.jt808.port=0",
|
||||||
"lingniu.ingest.identity.store=mysql",
|
"lingniu.ingest.identity.store=mysql",
|
||||||
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
||||||
"lingniu.ingest.sink.mq.enabled=true",
|
"lingniu.ingest.sink.kafka.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.type=kafka",
|
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
|
||||||
"lingniu.ingest.sink.archive.enabled=true",
|
"lingniu.ingest.sink.archive.enabled=true",
|
||||||
"lingniu.ingest.sink.archive.path=target/test-archive-jt808",
|
"lingniu.ingest.sink.archive.path=target/test-archive-jt808",
|
||||||
"lingniu.ingest.event-history.enabled=false");
|
"lingniu.ingest.event-history.enabled=false");
|
||||||
|
|||||||
@@ -28,11 +28,11 @@ class Jt808IngestAppDefaultsTest {
|
|||||||
.containsEntry("server.port", "${HTTP_PORT:20400}")
|
.containsEntry("server.port", "${HTTP_PORT:20400}")
|
||||||
.containsEntry("lingniu.ingest.jt808.enabled", true)
|
.containsEntry("lingniu.ingest.jt808.enabled", true)
|
||||||
.containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:808}")
|
.containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:808}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
|
.containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
|
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
||||||
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
|
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
|
||||||
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
|
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
|
||||||
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
||||||
|
|||||||
@@ -37,9 +37,8 @@ lingniu:
|
|||||||
gb32960:
|
gb32960:
|
||||||
enabled: false
|
enabled: false
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
topics:
|
topics:
|
||||||
realtime: ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
|
realtime: ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
|
||||||
|
|||||||
@@ -2,9 +2,9 @@ package com.lingniu.ingest.analyticsapp;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||||
import com.lingniu.ingest.vehiclestat.JdbcVehicleStatMetricRepository;
|
import com.lingniu.ingest.vehiclestat.JdbcVehicleStatMetricRepository;
|
||||||
import com.lingniu.ingest.vehiclestat.VehicleStatController;
|
import com.lingniu.ingest.vehiclestat.VehicleStatController;
|
||||||
import com.lingniu.ingest.vehiclestat.VehicleStatEnvelopeIngestor;
|
import com.lingniu.ingest.vehiclestat.VehicleStatEnvelopeIngestor;
|
||||||
@@ -28,17 +28,16 @@ class VehicleAnalyticsAppCompositionTest {
|
|||||||
void createsStatsBeansWithoutProtocolListenerOrEventFileStore() {
|
void createsStatsBeansWithoutProtocolListenerOrEventFileStore() {
|
||||||
new ApplicationContextRunner()
|
new ApplicationContextRunner()
|
||||||
.withConfiguration(AutoConfigurations.of(
|
.withConfiguration(AutoConfigurations.of(
|
||||||
SinkMqAutoConfiguration.class,
|
KafkaSinkAutoConfiguration.class,
|
||||||
VehicleStatAutoConfiguration.class))
|
VehicleStatAutoConfiguration.class))
|
||||||
.withAllowBeanDefinitionOverriding(true)
|
.withAllowBeanDefinitionOverriding(true)
|
||||||
.withBean("kafkaProducer", KafkaProducer.class, VehicleAnalyticsAppCompositionTest::kafkaProducer)
|
.withBean("kafkaProducer", KafkaProducer.class, VehicleAnalyticsAppCompositionTest::kafkaProducer)
|
||||||
.withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class))
|
.withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class))
|
||||||
.withBean(ObjectMapper.class, ObjectMapper::new)
|
.withBean(ObjectMapper.class, ObjectMapper::new)
|
||||||
.withPropertyValues(
|
.withPropertyValues(
|
||||||
"lingniu.ingest.sink.mq.enabled=true",
|
"lingniu.ingest.sink.kafka.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.type=kafka",
|
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
|
||||||
"lingniu.ingest.vehicle-stat.enabled=true",
|
"lingniu.ingest.vehicle-stat.enabled=true",
|
||||||
"lingniu.ingest.vehicle-stat.jt808.enabled=true",
|
"lingniu.ingest.vehicle-stat.jt808.enabled=true",
|
||||||
"lingniu.ingest.event-history.enabled=false",
|
"lingniu.ingest.event-history.enabled=false",
|
||||||
|
|||||||
@@ -29,12 +29,12 @@ class VehicleAnalyticsAppDefaultsTest {
|
|||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.vehicle-stat.jt808.enabled",
|
"lingniu.ingest.vehicle-stat.jt808.enabled",
|
||||||
"${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}")
|
"${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
|
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
|
||||||
"${VEHICLE_STAT_ENABLED:true}")
|
"${VEHICLE_STAT_ENABLED:true}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
|
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
|
||||||
"${KAFKA_GROUP_STAT:vehicle-stat}")
|
"${KAFKA_GROUP_STAT:vehicle-stat}")
|
||||||
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
|
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.lingniu.ingest.historyapp;
|
package com.lingniu.ingest.historyapp;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerFactory;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerFactory;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerWorker;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerWorker;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqProperties;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
|
||||||
import org.springframework.beans.factory.annotation.Qualifier;
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
@@ -20,11 +20,11 @@ public class VehicleHistoryKafkaConsumerConfiguration {
|
|||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
|
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka.consumer", name = "enabled", havingValue = "true")
|
||||||
public KafkaEnvelopeConsumerRunner vehicleHistoryKafkaEnvelopeConsumerRunner(
|
public KafkaEnvelopeConsumerRunner vehicleHistoryKafkaEnvelopeConsumerRunner(
|
||||||
@Qualifier("eventHistoryEnvelopeConsumerProcessor") EnvelopeConsumerProcessor processor,
|
@Qualifier("eventHistoryEnvelopeConsumerProcessor") EnvelopeConsumerProcessor processor,
|
||||||
@Qualifier("eventHistoryRawEnvelopeConsumerProcessor") EnvelopeConsumerProcessor rawProcessor,
|
@Qualifier("eventHistoryRawEnvelopeConsumerProcessor") EnvelopeConsumerProcessor rawProcessor,
|
||||||
SinkMqProperties props) {
|
KafkaSinkProperties props) {
|
||||||
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(
|
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(
|
||||||
Map.of(
|
Map.of(
|
||||||
"eventHistoryGb32960EnvelopeConsumerProcessor", processor,
|
"eventHistoryGb32960EnvelopeConsumerProcessor", processor,
|
||||||
|
|||||||
@@ -34,9 +34,8 @@ lingniu:
|
|||||||
server:
|
server:
|
||||||
enabled: false
|
enabled: false
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
topics:
|
topics:
|
||||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||||
|
|||||||
@@ -15,15 +15,15 @@ import com.lingniu.ingest.eventhistory.TelemetryFieldHistoryController;
|
|||||||
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
|
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
|
||||||
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
||||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqProperties;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
|
||||||
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
|
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
|
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema;
|
import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;
|
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;
|
||||||
@@ -75,8 +75,8 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
() -> processor("event-history"))
|
() -> processor("event-history"))
|
||||||
.withBean("eventHistoryRawEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
|
.withBean("eventHistoryRawEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
|
||||||
() -> processor("event-history-raw"))
|
() -> processor("event-history-raw"))
|
||||||
.withBean(SinkMqProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
|
.withBean(KafkaSinkProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
|
||||||
.withPropertyValues("lingniu.ingest.sink.mq.consumer.enabled=true")
|
.withPropertyValues("lingniu.ingest.sink.kafka.consumer.enabled=true")
|
||||||
.run(context -> {
|
.run(context -> {
|
||||||
assertThat(context).hasFailed();
|
assertThat(context).hasFailed();
|
||||||
assertThat(context.getStartupFailure())
|
assertThat(context.getStartupFailure())
|
||||||
@@ -89,7 +89,7 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
new ApplicationContextRunner()
|
new ApplicationContextRunner()
|
||||||
.withConfiguration(AutoConfigurations.of(
|
.withConfiguration(AutoConfigurations.of(
|
||||||
TdengineHistoryAutoConfiguration.class,
|
TdengineHistoryAutoConfiguration.class,
|
||||||
SinkMqAutoConfiguration.class,
|
KafkaSinkAutoConfiguration.class,
|
||||||
Gb32960AutoConfiguration.class,
|
Gb32960AutoConfiguration.class,
|
||||||
EventHistoryAutoConfiguration.class))
|
EventHistoryAutoConfiguration.class))
|
||||||
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
|
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
|
||||||
@@ -101,10 +101,9 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
"lingniu.ingest.event-history.enabled=true",
|
"lingniu.ingest.event-history.enabled=true",
|
||||||
"lingniu.ingest.gb32960.enabled=true",
|
"lingniu.ingest.gb32960.enabled=true",
|
||||||
"lingniu.ingest.gb32960.server.enabled=false",
|
"lingniu.ingest.gb32960.server.enabled=false",
|
||||||
"lingniu.ingest.sink.mq.enabled=true",
|
"lingniu.ingest.sink.kafka.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.type=kafka",
|
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
|
||||||
"lingniu.ingest.vehicle-state.enabled=false",
|
"lingniu.ingest.vehicle-state.enabled=false",
|
||||||
"lingniu.ingest.vehicle-stat.enabled=false")
|
"lingniu.ingest.vehicle-stat.enabled=false")
|
||||||
.run(context -> {
|
.run(context -> {
|
||||||
@@ -137,7 +136,7 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
|
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
|
||||||
.withPropertyValues(
|
.withPropertyValues(
|
||||||
"lingniu.ingest.event-history.enabled=true",
|
"lingniu.ingest.event-history.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=false")
|
"lingniu.ingest.sink.kafka.consumer.enabled=false")
|
||||||
.run(context -> {
|
.run(context -> {
|
||||||
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
||||||
assertThat(context).doesNotHaveBean("eventFileStore");
|
assertThat(context).doesNotHaveBean("eventFileStore");
|
||||||
@@ -149,7 +148,7 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
new ApplicationContextRunner()
|
new ApplicationContextRunner()
|
||||||
.withConfiguration(AutoConfigurations.of(
|
.withConfiguration(AutoConfigurations.of(
|
||||||
TdengineHistoryAutoConfiguration.class,
|
TdengineHistoryAutoConfiguration.class,
|
||||||
SinkMqAutoConfiguration.class,
|
KafkaSinkAutoConfiguration.class,
|
||||||
EventHistoryAutoConfiguration.class))
|
EventHistoryAutoConfiguration.class))
|
||||||
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
|
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
|
||||||
.withAllowBeanDefinitionOverriding(true)
|
.withAllowBeanDefinitionOverriding(true)
|
||||||
@@ -158,13 +157,12 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
"lingniu.ingest.tdengine-history.enabled=true",
|
"lingniu.ingest.tdengine-history.enabled=true",
|
||||||
"lingniu.ingest.tdengine-history.database=vehicle_history_test",
|
"lingniu.ingest.tdengine-history.database=vehicle_history_test",
|
||||||
"lingniu.ingest.event-history.enabled=true",
|
"lingniu.ingest.event-history.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.enabled=true",
|
"lingniu.ingest.sink.kafka.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.type=kafka",
|
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
"lingniu.ingest.sink.kafka.consumer.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=true",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
|
|
||||||
.run(context -> {
|
.run(context -> {
|
||||||
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
||||||
assertThat(context.getBeanNamesForType(EnvelopeConsumerProcessor.class))
|
assertThat(context.getBeanNamesForType(EnvelopeConsumerProcessor.class))
|
||||||
@@ -186,7 +184,7 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
|
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
|
||||||
.withPropertyValues(
|
.withPropertyValues(
|
||||||
"lingniu.ingest.event-history.enabled=true",
|
"lingniu.ingest.event-history.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=false")
|
"lingniu.ingest.sink.kafka.consumer.enabled=false")
|
||||||
.run(context -> {
|
.run(context -> {
|
||||||
EventHistoryEnvelopeIngestor ingestor = context.getBean(EventHistoryEnvelopeIngestor.class);
|
EventHistoryEnvelopeIngestor ingestor = context.getBean(EventHistoryEnvelopeIngestor.class);
|
||||||
|
|
||||||
@@ -211,8 +209,8 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
ignored -> {});
|
ignored -> {});
|
||||||
}
|
}
|
||||||
|
|
||||||
private static SinkMqProperties genericOnlyHistoryConsumerProps() {
|
private static KafkaSinkProperties genericOnlyHistoryConsumerProps() {
|
||||||
SinkMqProperties props = new SinkMqProperties();
|
KafkaSinkProperties props = new KafkaSinkProperties();
|
||||||
props.setBootstrapServers("localhost:9092");
|
props.setBootstrapServers("localhost:9092");
|
||||||
props.getConsumer().setEnabled(true);
|
props.getConsumer().setEnabled(true);
|
||||||
props.getConsumer().setBindings(Map.of(
|
props.getConsumer().setBindings(Map.of(
|
||||||
@@ -221,8 +219,8 @@ class VehicleHistoryAppCompositionTest {
|
|||||||
return props;
|
return props;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static SinkMqProperties.Binding binding(String groupId, String topic) {
|
private static KafkaSinkProperties.Binding binding(String groupId, String topic) {
|
||||||
SinkMqProperties.Binding binding = new SinkMqProperties.Binding();
|
KafkaSinkProperties.Binding binding = new KafkaSinkProperties.Binding();
|
||||||
binding.setGroupId(groupId);
|
binding.setGroupId(groupId);
|
||||||
binding.setTopics(List.of(topic));
|
binding.setTopics(List.of(topic));
|
||||||
return binding;
|
return binding;
|
||||||
|
|||||||
@@ -44,65 +44,65 @@ class VehicleHistoryAppDefaultsTest {
|
|||||||
.containsEntry("lingniu.ingest.event-history.enabled", true)
|
.containsEntry("lingniu.ingest.event-history.enabled", true)
|
||||||
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
|
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
|
||||||
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
|
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
|
||||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
|
.containsEntry("lingniu.ingest.sink.kafka.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
|
.containsEntry("lingniu.ingest.sink.kafka.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.max-poll-interval-millis",
|
"lingniu.ingest.sink.kafka.consumer.max-poll-interval-millis",
|
||||||
"${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}")
|
"${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled",
|
||||||
true)
|
true)
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id",
|
||||||
"${KAFKA_GROUP_HISTORY_GB32960_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-event}")
|
"${KAFKA_GROUP_HISTORY_GB32960_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-event}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]",
|
||||||
"${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}")
|
"${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.enabled",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.enabled",
|
||||||
true)
|
true)
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.group-id",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.group-id",
|
||||||
"${KAFKA_GROUP_HISTORY_JT808_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-event}")
|
"${KAFKA_GROUP_HISTORY_JT808_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-event}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.topics[0]",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808EnvelopeConsumerProcessor.topics[0]",
|
||||||
"${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
|
"${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.enabled",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.enabled",
|
||||||
true)
|
true)
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.group-id",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.group-id",
|
||||||
"${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-event}")
|
"${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-event}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.topics[0]",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttEnvelopeConsumerProcessor.topics[0]",
|
||||||
"${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
|
"${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.enabled",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.enabled",
|
||||||
true)
|
true)
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.group-id",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.group-id",
|
||||||
"${KAFKA_GROUP_HISTORY_GB32960_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-raw}")
|
"${KAFKA_GROUP_HISTORY_GB32960_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-raw}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.topics[0]",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960RawEnvelopeConsumerProcessor.topics[0]",
|
||||||
"${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}")
|
"${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.enabled",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.enabled",
|
||||||
true)
|
true)
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.group-id",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.group-id",
|
||||||
"${KAFKA_GROUP_HISTORY_JT808_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-raw}")
|
"${KAFKA_GROUP_HISTORY_JT808_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-raw}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.topics[0]",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryJt808RawEnvelopeConsumerProcessor.topics[0]",
|
||||||
"${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
"${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.enabled",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.enabled",
|
||||||
true)
|
true)
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.group-id",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.group-id",
|
||||||
"${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-raw}")
|
"${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-raw}")
|
||||||
.containsEntry(
|
.containsEntry(
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.topics[0]",
|
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryYutongMqttRawEnvelopeConsumerProcessor.topics[0]",
|
||||||
"${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
|
"${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
|
||||||
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
|
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
|
||||||
|
|
||||||
|
|||||||
@@ -53,20 +53,16 @@ lingniu:
|
|||||||
rate-limit:
|
rate-limit:
|
||||||
per-vin-qps: 50
|
per-vin-qps: 50
|
||||||
identity:
|
identity:
|
||||||
store: ${VEHICLE_IDENTITY_STORE:file}
|
store: ${VEHICLE_IDENTITY_STORE:mysql}
|
||||||
file:
|
|
||||||
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
|
|
||||||
mysql:
|
mysql:
|
||||||
table: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}
|
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}
|
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:}
|
|
||||||
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
|
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
|
||||||
driver-class-name: ${VEHICLE_IDENTITY_MYSQL_DRIVER_CLASS_NAME:com.mysql.cj.jdbc.Driver}
|
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||||
refresh-interval: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}
|
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
compression-type: zstd
|
compression-type: zstd
|
||||||
linger-ms: 20
|
linger-ms: 20
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ import com.lingniu.ingest.inbound.xinda.config.XindaPushAutoConfiguration;
|
|||||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||||
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
||||||
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||||
@@ -26,7 +26,7 @@ class XindaPushAppCompositionTest {
|
|||||||
.withConfiguration(AutoConfigurations.of(
|
.withConfiguration(AutoConfigurations.of(
|
||||||
IngestCoreAutoConfiguration.class,
|
IngestCoreAutoConfiguration.class,
|
||||||
VehicleIdentityAutoConfiguration.class,
|
VehicleIdentityAutoConfiguration.class,
|
||||||
SinkMqAutoConfiguration.class,
|
KafkaSinkAutoConfiguration.class,
|
||||||
SinkArchiveAutoConfiguration.class,
|
SinkArchiveAutoConfiguration.class,
|
||||||
XindaPushAutoConfiguration.class))
|
XindaPushAutoConfiguration.class))
|
||||||
.withAllowBeanDefinitionOverriding(true)
|
.withAllowBeanDefinitionOverriding(true)
|
||||||
@@ -37,10 +37,11 @@ class XindaPushAppCompositionTest {
|
|||||||
"lingniu.ingest.xinda-push.port=10100",
|
"lingniu.ingest.xinda-push.port=10100",
|
||||||
"lingniu.ingest.xinda-push.username=test",
|
"lingniu.ingest.xinda-push.username=test",
|
||||||
"lingniu.ingest.xinda-push.password=test",
|
"lingniu.ingest.xinda-push.password=test",
|
||||||
"lingniu.ingest.sink.mq.enabled=true",
|
"lingniu.ingest.identity.store=mysql",
|
||||||
"lingniu.ingest.sink.mq.type=kafka",
|
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
||||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
"lingniu.ingest.sink.kafka.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||||
|
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||||
"lingniu.ingest.sink.archive.enabled=true",
|
"lingniu.ingest.sink.archive.enabled=true",
|
||||||
"lingniu.ingest.sink.archive.path=target/test-archive-xinda",
|
"lingniu.ingest.sink.archive.path=target/test-archive-xinda",
|
||||||
"lingniu.ingest.event-file-store.enabled=false",
|
"lingniu.ingest.event-file-store.enabled=false",
|
||||||
|
|||||||
@@ -26,13 +26,14 @@ class XindaPushAppDefaultsTest {
|
|||||||
.containsEntry("lingniu.ingest.xinda-push.port", "${XINDA_PUSH_PORT:10100}")
|
.containsEntry("lingniu.ingest.xinda-push.port", "${XINDA_PUSH_PORT:10100}")
|
||||||
.containsEntry("lingniu.ingest.xinda-push.username", "${XINDA_PUSH_USERNAME:}")
|
.containsEntry("lingniu.ingest.xinda-push.username", "${XINDA_PUSH_USERNAME:}")
|
||||||
.containsEntry("lingniu.ingest.xinda-push.subscribe-msg-ids[0]", "${XINDA_PUSH_MSG_ID_LOCATION:0200}")
|
.containsEntry("lingniu.ingest.xinda-push.subscribe-msg-ids[0]", "${XINDA_PUSH_MSG_ID_LOCATION:0200}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
|
.containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
|
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_XINDA_PUSH_EVENT:vehicle.event.xinda-push.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_XINDA_PUSH_EVENT:vehicle.event.xinda-push.v1}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_XINDA_PUSH_RAW:vehicle.raw.xinda-push.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_XINDA_PUSH_RAW:vehicle.raw.xinda-push.v1}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_XINDA_PUSH_DLQ:vehicle.dlq.xinda-push.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_XINDA_PUSH_DLQ:vehicle.dlq.xinda-push.v1}")
|
||||||
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}")
|
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
|
||||||
.containsEntry("lingniu.ingest.identity.mysql.table", "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}")
|
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
||||||
|
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
|
||||||
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
|
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
|
||||||
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
|
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
|
||||||
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
||||||
|
|||||||
@@ -69,9 +69,8 @@ lingniu:
|
|||||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||||
sink:
|
sink:
|
||||||
mq:
|
kafka:
|
||||||
enabled: ${KAFKA_ENABLED:true}
|
enabled: ${KAFKA_ENABLED:true}
|
||||||
type: kafka
|
|
||||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||||
compression-type: zstd
|
compression-type: zstd
|
||||||
linger-ms: 20
|
linger-ms: 20
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
|
|||||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||||
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
||||||
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
@@ -31,7 +31,7 @@ class YutongMqttAppCompositionTest {
|
|||||||
.withConfiguration(AutoConfigurations.of(
|
.withConfiguration(AutoConfigurations.of(
|
||||||
IngestCoreAutoConfiguration.class,
|
IngestCoreAutoConfiguration.class,
|
||||||
VehicleIdentityAutoConfiguration.class,
|
VehicleIdentityAutoConfiguration.class,
|
||||||
SinkMqAutoConfiguration.class,
|
KafkaSinkAutoConfiguration.class,
|
||||||
SinkArchiveAutoConfiguration.class,
|
SinkArchiveAutoConfiguration.class,
|
||||||
MqttInboundAutoConfiguration.class))
|
MqttInboundAutoConfiguration.class))
|
||||||
.withAllowBeanDefinitionOverriding(true)
|
.withAllowBeanDefinitionOverriding(true)
|
||||||
@@ -45,10 +45,9 @@ class YutongMqttAppCompositionTest {
|
|||||||
"lingniu.ingest.mqtt.endpoints[0].profile=yutong",
|
"lingniu.ingest.mqtt.endpoints[0].profile=yutong",
|
||||||
"lingniu.ingest.identity.store=mysql",
|
"lingniu.ingest.identity.store=mysql",
|
||||||
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
||||||
"lingniu.ingest.sink.mq.enabled=true",
|
"lingniu.ingest.sink.kafka.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.type=kafka",
|
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
|
||||||
"lingniu.ingest.sink.archive.enabled=true",
|
"lingniu.ingest.sink.archive.enabled=true",
|
||||||
"lingniu.ingest.sink.archive.path=target/test-archive-yutong",
|
"lingniu.ingest.sink.archive.path=target/test-archive-yutong",
|
||||||
"lingniu.ingest.event-history.enabled=false");
|
"lingniu.ingest.event-history.enabled=false");
|
||||||
|
|||||||
@@ -28,11 +28,11 @@ class YutongMqttAppDefaultsTest {
|
|||||||
.containsEntry("lingniu.ingest.mqtt.endpoints[0].uri", "${YUTONG_MQTT_URI:}")
|
.containsEntry("lingniu.ingest.mqtt.endpoints[0].uri", "${YUTONG_MQTT_URI:}")
|
||||||
.containsEntry("lingniu.ingest.mqtt.endpoints[0].topic", "${YUTONG_MQTT_TOPIC:#}")
|
.containsEntry("lingniu.ingest.mqtt.endpoints[0].topic", "${YUTONG_MQTT_TOPIC:#}")
|
||||||
.containsEntry("lingniu.ingest.mqtt.endpoints[0].profile", "yutong")
|
.containsEntry("lingniu.ingest.mqtt.endpoints[0].profile", "yutong")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
|
.containsEntry("lingniu.ingest.sink.kafka.enabled", "${KAFKA_ENABLED:true}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
|
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}")
|
||||||
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_YUTONG_MQTT_DLQ:vehicle.dlq.mqtt-yutong.v1}")
|
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_YUTONG_MQTT_DLQ:vehicle.dlq.mqtt-yutong.v1}")
|
||||||
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
|
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
|
||||||
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
||||||
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
|
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
|
|||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||||
import com.lingniu.ingest.api.history.EventFileRecord;
|
import com.lingniu.ingest.api.history.EventFileRecord;
|
||||||
import com.lingniu.ingest.api.history.EventFileStore;
|
import com.lingniu.ingest.api.history.EventFileStore;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows;
|
import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
|
|||||||
@@ -5,8 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import com.lingniu.ingest.api.ProtocolId;
|
import com.lingniu.ingest.api.ProtocolId;
|
||||||
import com.lingniu.ingest.api.event.RawArchiveKeys;
|
import com.lingniu.ingest.api.event.RawArchiveKeys;
|
||||||
import com.lingniu.ingest.api.history.EventFileRecord;
|
import com.lingniu.ingest.api.history.EventFileRecord;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.google.protobuf.InvalidProtocolBufferException;
|
import com.google.protobuf.InvalidProtocolBufferException;
|
||||||
import com.google.protobuf.util.JsonFormat;
|
import com.google.protobuf.util.JsonFormat;
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import com.lingniu.ingest.eventhistory.TelemetryFieldHistoryController;
|
|||||||
import com.lingniu.ingest.eventhistory.TelemetryEnvelopeRecordMapper;
|
import com.lingniu.ingest.eventhistory.TelemetryEnvelopeRecordMapper;
|
||||||
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
|
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
|
||||||
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
||||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||||
import com.lingniu.ingest.tdenginehistory.config.TdengineHistoryAutoConfiguration;
|
import com.lingniu.ingest.tdenginehistory.config.TdengineHistoryAutoConfiguration;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
|
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
||||||
@@ -41,12 +41,12 @@ import java.nio.file.Path;
|
|||||||
* </ul>
|
* </ul>
|
||||||
*
|
*
|
||||||
* <p>当前生产 history app 以 TDengine 为准;Kafka consumer 是否启动还取决于
|
* <p>当前生产 history app 以 TDengine 为准;Kafka consumer 是否启动还取决于
|
||||||
* {@code lingniu.ingest.sink.mq.consumer.enabled=true}。
|
* {@code lingniu.ingest.sink.kafka.consumer.enabled=true}。
|
||||||
*/
|
*/
|
||||||
@AutoConfiguration
|
@AutoConfiguration
|
||||||
@AutoConfigureAfter({
|
@AutoConfigureAfter({
|
||||||
Gb32960AutoConfiguration.class,
|
Gb32960AutoConfiguration.class,
|
||||||
SinkMqAutoConfiguration.class,
|
KafkaSinkAutoConfiguration.class,
|
||||||
TdengineHistoryAutoConfiguration.class
|
TdengineHistoryAutoConfiguration.class
|
||||||
})
|
})
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.event-history", name = "enabled", havingValue = "true")
|
@ConditionalOnProperty(prefix = "lingniu.ingest.event-history", name = "enabled", havingValue = "true")
|
||||||
@@ -82,7 +82,7 @@ public class EventHistoryAutoConfiguration {
|
|||||||
@ConditionalOnMissingBean(name = "eventHistoryEnvelopeConsumerProcessor")
|
@ConditionalOnMissingBean(name = "eventHistoryEnvelopeConsumerProcessor")
|
||||||
public EnvelopeConsumerProcessor eventHistoryEnvelopeConsumerProcessor(EventHistoryEnvelopeIngestor ingestor,
|
public EnvelopeConsumerProcessor eventHistoryEnvelopeConsumerProcessor(EventHistoryEnvelopeIngestor ingestor,
|
||||||
EnvelopeDeadLetterSink deadLetterSink) {
|
EnvelopeDeadLetterSink deadLetterSink) {
|
||||||
// 只注册 processor;真正拉 Kafka 的 runner 在 sink-mq 模块按 consumer.enabled 决定是否创建。
|
// 只注册 processor;真正拉 Kafka 的 runner 按 consumer.enabled 决定是否创建。
|
||||||
return new EnvelopeConsumerProcessor("event-history", ingestor, deadLetterSink);
|
return new EnvelopeConsumerProcessor("event-history", ingestor, deadLetterSink);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,13 +9,13 @@ import com.lingniu.ingest.api.event.RawArchiveKeys;
|
|||||||
import com.lingniu.ingest.api.history.EventFileQuery;
|
import com.lingniu.ingest.api.history.EventFileQuery;
|
||||||
import com.lingniu.ingest.api.history.EventFileRecord;
|
import com.lingniu.ingest.api.history.EventFileRecord;
|
||||||
import com.lingniu.ingest.api.history.EventFileStore;
|
import com.lingniu.ingest.api.history.EventFileStore;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
|
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.lingniu.ingest.sink.mq.proto.LocationPayload;
|
import com.lingniu.ingest.sink.kafka.proto.LocationPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
|
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
|
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
|
||||||
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;
|
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;
|
||||||
|
|||||||
@@ -2,10 +2,10 @@ package com.lingniu.ingest.eventhistory;
|
|||||||
|
|
||||||
import com.lingniu.ingest.api.ProtocolId;
|
import com.lingniu.ingest.api.ProtocolId;
|
||||||
import com.lingniu.ingest.api.history.EventFileRecord;
|
import com.lingniu.ingest.api.history.EventFileRecord;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ class EventHistoryAutoConfigurationTest {
|
|||||||
contextRunner
|
contextRunner
|
||||||
.withPropertyValues(
|
.withPropertyValues(
|
||||||
"lingniu.ingest.event-history.enabled=true",
|
"lingniu.ingest.event-history.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=true")
|
"lingniu.ingest.sink.kafka.consumer.enabled=true")
|
||||||
.run(context -> {
|
.run(context -> {
|
||||||
assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class);
|
assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class);
|
||||||
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ class EventHistoryPomBoundaryTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void kafkaClientsIsOwnedBySinkMqModule() throws Exception {
|
void kafkaClientsIsOwnedByKafkaSinkModule() throws Exception {
|
||||||
for (Path pom : servicePoms()) {
|
for (Path pom : servicePoms()) {
|
||||||
assertThat(directDependencies(readPom(pom)))
|
assertThat(directDependencies(readPom(pom)))
|
||||||
.as(pom.toString())
|
.as(pom.toString())
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package com.lingniu.ingest.vehiclestat;
|
|||||||
import com.google.protobuf.InvalidProtocolBufferException;
|
import com.google.protobuf.InvalidProtocolBufferException;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
|
import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
|
||||||
|
|
||||||
public final class VehicleStatEnvelopeIngestor implements EnvelopeIngestor {
|
public final class VehicleStatEnvelopeIngestor implements EnvelopeIngestor {
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ public class VehicleStatAutoConfiguration {
|
|||||||
@ConditionalOnMissingBean(name = "vehicleStatEnvelopeConsumerProcessor")
|
@ConditionalOnMissingBean(name = "vehicleStatEnvelopeConsumerProcessor")
|
||||||
public EnvelopeConsumerProcessor vehicleStatEnvelopeConsumerProcessor(VehicleStatEnvelopeIngestor ingestor,
|
public EnvelopeConsumerProcessor vehicleStatEnvelopeConsumerProcessor(VehicleStatEnvelopeIngestor ingestor,
|
||||||
EnvelopeDeadLetterSink deadLetterSink) {
|
EnvelopeDeadLetterSink deadLetterSink) {
|
||||||
// Bean 名必须和 sink-mq 默认 binding 对齐,KafkaEnvelopeConsumerFactory 才能自动创建 worker。
|
// Bean 名必须和 Kafka 默认 binding 对齐,KafkaEnvelopeConsumerFactory 才能自动创建 worker。
|
||||||
return new EnvelopeConsumerProcessor("vehicle-stat", ingestor, deadLetterSink);
|
return new EnvelopeConsumerProcessor("vehicle-stat", ingestor, deadLetterSink);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.lingniu.ingest.vehiclestat.jt808;
|
package com.lingniu.ingest.vehiclestat.jt808;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
|
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.lingniu.ingest.vehiclestat.jt808;
|
package com.lingniu.ingest.vehiclestat.jt808;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.lingniu.ingest.vehiclestat.VehicleStatRepository;
|
import com.lingniu.ingest.vehiclestat.VehicleStatRepository;
|
||||||
|
|
||||||
import java.time.LocalDate;
|
import java.time.LocalDate;
|
||||||
|
|||||||
@@ -2,10 +2,10 @@ package com.lingniu.ingest.vehiclestat;
|
|||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
|
import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.lingniu.ingest.vehiclestat.jt808;
|
package com.lingniu.ingest.vehiclestat.jt808;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.lingniu.ingest.vehiclestat.jt808;
|
package com.lingniu.ingest.vehiclestat.jt808;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import com.lingniu.ingest.vehiclestat.DailyMileageStrategy;
|
import com.lingniu.ingest.vehiclestat.DailyMileageStrategy;
|
||||||
import com.lingniu.ingest.vehiclestat.VehicleDailyStatResult;
|
import com.lingniu.ingest.vehiclestat.VehicleDailyStatResult;
|
||||||
import com.lingniu.ingest.vehiclestat.VehicleStatRepository;
|
import com.lingniu.ingest.vehiclestat.VehicleStatRepository;
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package com.lingniu.ingest.vehiclestate;
|
|||||||
import com.google.protobuf.InvalidProtocolBufferException;
|
import com.google.protobuf.InvalidProtocolBufferException;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
|
|
||||||
public final class VehicleStateEnvelopeIngestor implements EnvelopeIngestor {
|
public final class VehicleStateEnvelopeIngestor implements EnvelopeIngestor {
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.lingniu.ingest.vehiclestate;
|
package com.lingniu.ingest.vehiclestate;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ public class VehicleStateAutoConfiguration {
|
|||||||
@ConditionalOnMissingBean(name = "vehicleStateEnvelopeConsumerProcessor")
|
@ConditionalOnMissingBean(name = "vehicleStateEnvelopeConsumerProcessor")
|
||||||
public EnvelopeConsumerProcessor vehicleStateEnvelopeConsumerProcessor(VehicleStateEnvelopeIngestor ingestor,
|
public EnvelopeConsumerProcessor vehicleStateEnvelopeConsumerProcessor(VehicleStateEnvelopeIngestor ingestor,
|
||||||
EnvelopeDeadLetterSink deadLetterSink) {
|
EnvelopeDeadLetterSink deadLetterSink) {
|
||||||
// Bean 名必须和 sink-mq 默认 binding 对齐,KafkaEnvelopeConsumerFactory 才能自动创建 worker。
|
// Bean 名必须和 Kafka 默认 binding 对齐,KafkaEnvelopeConsumerFactory 才能自动创建 worker。
|
||||||
return new EnvelopeConsumerProcessor("vehicle-state", ingestor, deadLetterSink);
|
return new EnvelopeConsumerProcessor("vehicle-state", ingestor, deadLetterSink);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,10 +2,10 @@ package com.lingniu.ingest.vehiclestate;
|
|||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.lingniu.ingest.vehiclestate;
|
package com.lingniu.ingest.vehiclestate;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import static org.assertj.core.api.Assertions.assertThat;
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.event.AlarmPayload;
|
import com.lingniu.ingest.api.event.AlarmPayload;
|
||||||
import com.lingniu.ingest.api.event.LocationPayload;
|
import com.lingniu.ingest.api.event.LocationPayload;
|
||||||
@@ -10,10 +10,10 @@ import com.lingniu.ingest.api.event.VehicleEvent;
|
|||||||
import com.lingniu.ingest.api.event.VehicleEventTelemetrySnapshotMapper;
|
import com.lingniu.ingest.api.event.VehicleEventTelemetrySnapshotMapper;
|
||||||
import com.lingniu.ingest.facts.FactIds;
|
import com.lingniu.ingest.facts.FactIds;
|
||||||
import com.lingniu.ingest.facts.VehicleKey;
|
import com.lingniu.ingest.facts.VehicleKey;
|
||||||
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
|
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot.Builder;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot.Builder;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
|
|
||||||
import java.security.MessageDigest;
|
import java.security.MessageDigest;
|
||||||
import java.security.NoSuchAlgorithmException;
|
import java.security.NoSuchAlgorithmException;
|
||||||
@@ -53,23 +53,23 @@ public final class EnvelopeMapper {
|
|||||||
case VehicleEvent.Location l -> b.setLocation(buildLocation(l.payload()));
|
case VehicleEvent.Location l -> b.setLocation(buildLocation(l.payload()));
|
||||||
case VehicleEvent.Alarm a -> b.setAlarm(buildAlarm(a.payload()));
|
case VehicleEvent.Alarm a -> b.setAlarm(buildAlarm(a.payload()));
|
||||||
case VehicleEvent.Login lg -> b.setLogin(
|
case VehicleEvent.Login lg -> b.setLogin(
|
||||||
com.lingniu.ingest.sink.mq.proto.LoginPayload.newBuilder()
|
com.lingniu.ingest.sink.kafka.proto.LoginPayload.newBuilder()
|
||||||
.setIccid(nullToEmpty(lg.iccid()))
|
.setIccid(nullToEmpty(lg.iccid()))
|
||||||
.setProtocolVersion(nullToEmpty(lg.protocolVersion()))
|
.setProtocolVersion(nullToEmpty(lg.protocolVersion()))
|
||||||
.build());
|
.build());
|
||||||
case VehicleEvent.Logout ignored -> b.setLogout(
|
case VehicleEvent.Logout ignored -> b.setLogout(
|
||||||
com.lingniu.ingest.sink.mq.proto.LogoutPayload.getDefaultInstance());
|
com.lingniu.ingest.sink.kafka.proto.LogoutPayload.getDefaultInstance());
|
||||||
case VehicleEvent.Heartbeat ignored -> b.setHeartbeat(
|
case VehicleEvent.Heartbeat ignored -> b.setHeartbeat(
|
||||||
com.lingniu.ingest.sink.mq.proto.HeartbeatPayload.getDefaultInstance());
|
com.lingniu.ingest.sink.kafka.proto.HeartbeatPayload.getDefaultInstance());
|
||||||
case VehicleEvent.MediaMeta m -> b.setMediaMeta(
|
case VehicleEvent.MediaMeta m -> b.setMediaMeta(
|
||||||
com.lingniu.ingest.sink.mq.proto.MediaMetaPayload.newBuilder()
|
com.lingniu.ingest.sink.kafka.proto.MediaMetaPayload.newBuilder()
|
||||||
.setMediaId(nullToEmpty(m.mediaId()))
|
.setMediaId(nullToEmpty(m.mediaId()))
|
||||||
.setMediaType(nullToEmpty(m.mediaType()))
|
.setMediaType(nullToEmpty(m.mediaType()))
|
||||||
.setSizeBytes(m.sizeBytes())
|
.setSizeBytes(m.sizeBytes())
|
||||||
.setArchiveRef(nullToEmpty(m.archiveRef()))
|
.setArchiveRef(nullToEmpty(m.archiveRef()))
|
||||||
.build());
|
.build());
|
||||||
case VehicleEvent.Passthrough p -> b.setPassthrough(
|
case VehicleEvent.Passthrough p -> b.setPassthrough(
|
||||||
com.lingniu.ingest.sink.mq.proto.PassthroughPayload.newBuilder()
|
com.lingniu.ingest.sink.kafka.proto.PassthroughPayload.newBuilder()
|
||||||
.setPassthroughType(p.passthroughType())
|
.setPassthroughType(p.passthroughType())
|
||||||
.setData(com.google.protobuf.ByteString.copyFrom(
|
.setData(com.google.protobuf.ByteString.copyFrom(
|
||||||
p.data() == null ? new byte[0] : p.data()))
|
p.data() == null ? new byte[0] : p.data()))
|
||||||
@@ -88,13 +88,13 @@ public final class EnvelopeMapper {
|
|||||||
b.putMetadata(RawArchiveKeys.META_KEY, key);
|
b.putMetadata(RawArchiveKeys.META_KEY, key);
|
||||||
b.putMetadata(RawArchiveKeys.META_URI, uri);
|
b.putMetadata(RawArchiveKeys.META_URI, uri);
|
||||||
b.putMetadata(RawArchiveKeys.META_EVENT_ID, ra.eventId());
|
b.putMetadata(RawArchiveKeys.META_EVENT_ID, ra.eventId());
|
||||||
b.setRawArchive(com.lingniu.ingest.sink.mq.proto.RawArchiveRef.newBuilder()
|
b.setRawArchive(com.lingniu.ingest.sink.kafka.proto.RawArchiveRef.newBuilder()
|
||||||
.setUri(uri)
|
.setUri(uri)
|
||||||
.setChecksum(checksum)
|
.setChecksum(checksum)
|
||||||
.setSizeBytes(size)
|
.setSizeBytes(size)
|
||||||
.setParsedJson(nullToEmpty(ra.parsedJson()))
|
.setParsedJson(nullToEmpty(ra.parsedJson()))
|
||||||
.build());
|
.build());
|
||||||
b.setRawFrameFact(com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload.newBuilder()
|
b.setRawFrameFact(com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload.newBuilder()
|
||||||
.setFrameId(frameId)
|
.setFrameId(frameId)
|
||||||
.setVehicleKey(vehicleKey)
|
.setVehicleKey(vehicleKey)
|
||||||
.setVin(nullToEmpty(ra.vin()))
|
.setVin(nullToEmpty(ra.vin()))
|
||||||
@@ -114,8 +114,8 @@ public final class EnvelopeMapper {
|
|||||||
return b.build();
|
return b.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot buildTelemetrySnapshot(TelemetrySnapshot snapshot) {
|
private static com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot buildTelemetrySnapshot(TelemetrySnapshot snapshot) {
|
||||||
Builder b = com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot.newBuilder()
|
Builder b = com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot.newBuilder()
|
||||||
.setEventType(snapshot.eventType())
|
.setEventType(snapshot.eventType())
|
||||||
.setRawArchiveUri(snapshot.rawArchiveUri());
|
.setRawArchiveUri(snapshot.rawArchiveUri());
|
||||||
for (TelemetryFieldValue field : snapshot.fields()) {
|
for (TelemetryFieldValue field : snapshot.fields()) {
|
||||||
@@ -131,8 +131,8 @@ public final class EnvelopeMapper {
|
|||||||
return b.build();
|
return b.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static com.lingniu.ingest.sink.mq.proto.RealtimePayload buildRealtime(RealtimePayload p) {
|
private static com.lingniu.ingest.sink.kafka.proto.RealtimePayload buildRealtime(RealtimePayload p) {
|
||||||
var b = com.lingniu.ingest.sink.mq.proto.RealtimePayload.newBuilder();
|
var b = com.lingniu.ingest.sink.kafka.proto.RealtimePayload.newBuilder();
|
||||||
if (p.speedKmh() != null) b.setSpeedKmh(p.speedKmh());
|
if (p.speedKmh() != null) b.setSpeedKmh(p.speedKmh());
|
||||||
if (p.totalMileageKm() != null) b.setTotalMileageKm(p.totalMileageKm());
|
if (p.totalMileageKm() != null) b.setTotalMileageKm(p.totalMileageKm());
|
||||||
if (p.batterySoc() != null) b.setBatterySoc(p.batterySoc());
|
if (p.batterySoc() != null) b.setBatterySoc(p.batterySoc());
|
||||||
@@ -159,8 +159,8 @@ public final class EnvelopeMapper {
|
|||||||
return b.build();
|
return b.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static com.lingniu.ingest.sink.mq.proto.LocationPayload buildLocation(LocationPayload p) {
|
private static com.lingniu.ingest.sink.kafka.proto.LocationPayload buildLocation(LocationPayload p) {
|
||||||
return com.lingniu.ingest.sink.mq.proto.LocationPayload.newBuilder()
|
return com.lingniu.ingest.sink.kafka.proto.LocationPayload.newBuilder()
|
||||||
.setLongitude(p.longitude())
|
.setLongitude(p.longitude())
|
||||||
.setLatitude(p.latitude())
|
.setLatitude(p.latitude())
|
||||||
.setAltitudeM(p.altitudeM())
|
.setAltitudeM(p.altitudeM())
|
||||||
@@ -171,8 +171,8 @@ public final class EnvelopeMapper {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static com.lingniu.ingest.sink.mq.proto.AlarmPayload buildAlarm(AlarmPayload p) {
|
private static com.lingniu.ingest.sink.kafka.proto.AlarmPayload buildAlarm(AlarmPayload p) {
|
||||||
var b = com.lingniu.ingest.sink.mq.proto.AlarmPayload.newBuilder()
|
var b = com.lingniu.ingest.sink.kafka.proto.AlarmPayload.newBuilder()
|
||||||
.setLevel(p.level().name())
|
.setLevel(p.level().name())
|
||||||
.setAlarmTypeCode(p.alarmTypeCode())
|
.setAlarmTypeCode(p.alarmTypeCode())
|
||||||
.setAlarmTypeName(nullToEmpty(p.alarmTypeName()));
|
.setAlarmTypeName(nullToEmpty(p.alarmTypeName()));
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||||
@@ -28,15 +28,15 @@ public final class KafkaEnvelopeConsumerFactory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public List<KafkaEnvelopeConsumerWorker> createWorkers(Map<String, EnvelopeConsumerProcessor> processors,
|
public List<KafkaEnvelopeConsumerWorker> createWorkers(Map<String, EnvelopeConsumerProcessor> processors,
|
||||||
SinkMqProperties props) {
|
KafkaSinkProperties props) {
|
||||||
Map<String, SinkMqProperties.Binding> bindings = effectiveBindings(props);
|
Map<String, KafkaSinkProperties.Binding> bindings = effectiveBindings(props);
|
||||||
List<KafkaEnvelopeConsumerWorker> workers = new ArrayList<>();
|
List<KafkaEnvelopeConsumerWorker> workers = new ArrayList<>();
|
||||||
for (Map.Entry<String, SinkMqProperties.Binding> entry : bindings.entrySet()) {
|
for (Map.Entry<String, KafkaSinkProperties.Binding> entry : bindings.entrySet()) {
|
||||||
String processorBeanName = entry.getKey();
|
String processorBeanName = entry.getKey();
|
||||||
// binding 的 key 必须和 Spring Bean 名一致;这样配置只声明 topic/group,
|
// binding 的 key 必须和 Spring Bean 名一致;这样配置只声明 topic/group,
|
||||||
// 实际处理逻辑仍由各业务模块自己的 EnvelopeConsumerProcessor 承接。
|
// 实际处理逻辑仍由各业务模块自己的 EnvelopeConsumerProcessor 承接。
|
||||||
EnvelopeConsumerProcessor processor = processors.get(processorBeanName);
|
EnvelopeConsumerProcessor processor = processors.get(processorBeanName);
|
||||||
SinkMqProperties.Binding binding = entry.getValue();
|
KafkaSinkProperties.Binding binding = entry.getValue();
|
||||||
if (processor == null || binding == null || !binding.isEnabled()) {
|
if (processor == null || binding == null || !binding.isEnabled()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -57,15 +57,15 @@ public final class KafkaEnvelopeConsumerFactory {
|
|||||||
return workers;
|
return workers;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, SinkMqProperties.Binding> effectiveBindings(SinkMqProperties props) {
|
private Map<String, KafkaSinkProperties.Binding> effectiveBindings(KafkaSinkProperties props) {
|
||||||
Map<String, SinkMqProperties.Binding> configured = props.getConsumer().getBindings();
|
Map<String, KafkaSinkProperties.Binding> configured = props.getConsumer().getBindings();
|
||||||
if (configured != null && !configured.isEmpty()) {
|
if (configured != null && !configured.isEmpty()) {
|
||||||
return configured;
|
return configured;
|
||||||
}
|
}
|
||||||
// 默认绑定仅给未显式配置 bindings 的轻量运行时兜底。
|
// 默认绑定仅给未显式配置 bindings 的轻量运行时兜底。
|
||||||
// 生产 history app 会显式绑定各协议 event/raw topic,并写入 TDengine raw_frames/locations。
|
// 生产 history app 会显式绑定各协议 event/raw topic,并写入 TDengine raw_frames/locations。
|
||||||
SinkMqProperties.Topics topics = props.getTopics();
|
KafkaSinkProperties.Topics topics = props.getTopics();
|
||||||
Map<String, SinkMqProperties.Binding> defaults = new LinkedHashMap<>();
|
Map<String, KafkaSinkProperties.Binding> defaults = new LinkedHashMap<>();
|
||||||
defaults.put("eventHistoryEnvelopeConsumerProcessor", binding(
|
defaults.put("eventHistoryEnvelopeConsumerProcessor", binding(
|
||||||
"vehicle-event-history",
|
"vehicle-event-history",
|
||||||
topics.getRealtime(), topics.getLocation(), topics.getAlarm(), topics.getSession(), topics.getMediaMeta()));
|
topics.getRealtime(), topics.getLocation(), topics.getAlarm(), topics.getSession(), topics.getMediaMeta()));
|
||||||
@@ -78,8 +78,8 @@ public final class KafkaEnvelopeConsumerFactory {
|
|||||||
return defaults;
|
return defaults;
|
||||||
}
|
}
|
||||||
|
|
||||||
private SinkMqProperties.Binding binding(String groupId, String... topics) {
|
private KafkaSinkProperties.Binding binding(String groupId, String... topics) {
|
||||||
SinkMqProperties.Binding binding = new SinkMqProperties.Binding();
|
KafkaSinkProperties.Binding binding = new KafkaSinkProperties.Binding();
|
||||||
binding.setGroupId(groupId);
|
binding.setGroupId(groupId);
|
||||||
binding.setTopics(List.of(topics));
|
binding.setTopics(List.of(topics));
|
||||||
return binding;
|
return binding;
|
||||||
@@ -106,12 +106,12 @@ public final class KafkaEnvelopeConsumerFactory {
|
|||||||
return List.copyOf(clean);
|
return List.copyOf(clean);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Properties consumerProperties(SinkMqProperties props,
|
private Properties consumerProperties(KafkaSinkProperties props,
|
||||||
SinkMqProperties.Binding binding,
|
KafkaSinkProperties.Binding binding,
|
||||||
String processorBeanName,
|
String processorBeanName,
|
||||||
int workerIndex,
|
int workerIndex,
|
||||||
int concurrency) {
|
int concurrency) {
|
||||||
SinkMqProperties.Consumer consumer = props.getConsumer();
|
KafkaSinkProperties.Consumer consumer = props.getConsumer();
|
||||||
Properties p = new Properties();
|
Properties p = new Properties();
|
||||||
p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers());
|
p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers());
|
||||||
p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
|
p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
|
||||||
@@ -133,7 +133,7 @@ public final class KafkaEnvelopeConsumerFactory {
|
|||||||
return concurrency <= 1 ? base : base + "-" + workerIndex;
|
return concurrency <= 1 ? base : base + "-" + workerIndex;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String groupId(SinkMqProperties.Binding binding, String processorBeanName) {
|
private String groupId(KafkaSinkProperties.Binding binding, String processorBeanName) {
|
||||||
if (binding.getGroupId() != null && !binding.getGroupId().isBlank()) {
|
if (binding.getGroupId() != null && !binding.getGroupId().isBlank()) {
|
||||||
return binding.getGroupId();
|
return binding.getGroupId();
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
|
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
|
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.event.VehicleEvent;
|
import com.lingniu.ingest.api.event.VehicleEvent;
|
||||||
import com.lingniu.ingest.api.sink.EventSink;
|
import com.lingniu.ingest.api.sink.EventSink;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
|
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
|
||||||
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
|
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
|
||||||
@@ -16,40 +16,31 @@ import java.time.Duration;
|
|||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* MQ Sink 自动装配。
|
* Kafka Sink 自动装配。
|
||||||
*
|
*
|
||||||
* <p>装配前提(两个条件都要满足):
|
* <p>{@code lingniu.ingest.sink.kafka.enabled=false} 时本模块完全不装配;Producer 和 Consumer
|
||||||
* <ol>
|
* 是两个独立开关,消费端还需要开启 {@code lingniu.ingest.sink.kafka.consumer.enabled=true}
|
||||||
* <li>{@code lingniu.ingest.sink.mq.enabled=true}(默认 true)—— <b>总开关</b>,
|
* 并配置 bindings。
|
||||||
* 设为 false 时本模块完全不装配,ingest-core 的 DisruptorEventBus 仍然运行但
|
|
||||||
* 没有 Kafka sink。
|
|
||||||
* <li>{@code lingniu.ingest.sink.mq.type=kafka}(默认 kafka)—— 唯一生产 MQ 后端。
|
|
||||||
* </ol>
|
|
||||||
*
|
|
||||||
* <p>Producer 和 Consumer 是两个独立开关:{@code sink.mq.enabled=true} 只表示可以创建
|
|
||||||
* Kafka producer/sink;是否从 Kafka 拉 envelope 还要单独开启
|
|
||||||
* {@code lingniu.ingest.sink.mq.consumer.enabled=true} 并配置 bindings。
|
|
||||||
*/
|
*/
|
||||||
@AutoConfiguration
|
@AutoConfiguration
|
||||||
@EnableConfigurationProperties(SinkMqProperties.class)
|
@EnableConfigurationProperties(KafkaSinkProperties.class)
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||||
public class SinkMqAutoConfiguration {
|
public class KafkaSinkAutoConfiguration {
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
public EnvelopeMapper envelopeMapper(SinkMqProperties props) {
|
public EnvelopeMapper envelopeMapper(KafkaSinkProperties props) {
|
||||||
return new EnvelopeMapper(props.getNodeId());
|
return new EnvelopeMapper(props.getNodeId());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
public TopicRouter topicRouter(SinkMqProperties props) {
|
public TopicRouter topicRouter(KafkaSinkProperties props) {
|
||||||
return new TopicRouter(props.getTopics());
|
return new TopicRouter(props.getTopics());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "type", havingValue = "kafka", matchIfMissing = true)
|
|
||||||
public CircuitBreaker kafkaSinkCircuitBreaker() {
|
public CircuitBreaker kafkaSinkCircuitBreaker() {
|
||||||
return CircuitBreaker.of("kafka-sink", CircuitBreakerConfig.custom()
|
return CircuitBreaker.of("kafka-sink", CircuitBreakerConfig.custom()
|
||||||
.slidingWindowSize(100)
|
.slidingWindowSize(100)
|
||||||
@@ -61,8 +52,7 @@ public class SinkMqAutoConfiguration {
|
|||||||
|
|
||||||
@Bean(destroyMethod = "close")
|
@Bean(destroyMethod = "close")
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "type", havingValue = "kafka", matchIfMissing = true)
|
public KafkaProducer<String, byte[]> kafkaProducer(KafkaSinkProperties props) {
|
||||||
public KafkaProducer<String, byte[]> kafkaProducer(SinkMqProperties props) {
|
|
||||||
Properties p = new Properties();
|
Properties p = new Properties();
|
||||||
p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers());
|
p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers());
|
||||||
p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||||
@@ -77,11 +67,10 @@ public class SinkMqAutoConfiguration {
|
|||||||
|
|
||||||
@Bean(destroyMethod = "close")
|
@Bean(destroyMethod = "close")
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "type", havingValue = "kafka", matchIfMissing = true)
|
|
||||||
public KafkaEventSink kafkaEventSink(KafkaProducer<String, byte[]> producer,
|
public KafkaEventSink kafkaEventSink(KafkaProducer<String, byte[]> producer,
|
||||||
EnvelopeMapper mapper,
|
EnvelopeMapper mapper,
|
||||||
TopicRouter router,
|
TopicRouter router,
|
||||||
SinkMqProperties props,
|
KafkaSinkProperties props,
|
||||||
CircuitBreaker breaker) {
|
CircuitBreaker breaker) {
|
||||||
// KafkaEventSink 是 EventBus 的生产端出口;它不会启动任何 Kafka 消费线程。
|
// KafkaEventSink 是 EventBus 的生产端出口;它不会启动任何 Kafka 消费线程。
|
||||||
return new KafkaEventSink(producer, mapper, router, props.getTopics().getDlq(), breaker);
|
return new KafkaEventSink(producer, mapper, router, props.getTopics().getDlq(), breaker);
|
||||||
@@ -89,9 +78,8 @@ public class SinkMqAutoConfiguration {
|
|||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "type", havingValue = "kafka", matchIfMissing = true)
|
|
||||||
public KafkaEnvelopeDeadLetterSink kafkaEnvelopeDeadLetterSink(KafkaProducer<String, byte[]> producer,
|
public KafkaEnvelopeDeadLetterSink kafkaEnvelopeDeadLetterSink(KafkaProducer<String, byte[]> producer,
|
||||||
SinkMqProperties props) {
|
KafkaSinkProperties props) {
|
||||||
return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq());
|
return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||||
import org.springframework.beans.factory.ListableBeanFactory;
|
import org.springframework.beans.factory.ListableBeanFactory;
|
||||||
@@ -13,15 +13,15 @@ import java.time.Duration;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@AutoConfiguration(after = SinkMqAutoConfiguration.class)
|
@AutoConfiguration(after = KafkaSinkAutoConfiguration.class)
|
||||||
@AutoConfigureAfter(name = {
|
@AutoConfigureAfter(name = {
|
||||||
"com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration",
|
"com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration",
|
||||||
"com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration",
|
"com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration",
|
||||||
"com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration"
|
"com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration"
|
||||||
})
|
})
|
||||||
@EnableConfigurationProperties(SinkMqProperties.class)
|
@EnableConfigurationProperties(KafkaSinkProperties.class)
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||||
public class SinkMqConsumerAutoConfiguration {
|
public class KafkaSinkConsumerAutoConfiguration {
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
@@ -31,10 +31,10 @@ public class SinkMqConsumerAutoConfiguration {
|
|||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@ConditionalOnMissingBean
|
@ConditionalOnMissingBean
|
||||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
|
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka.consumer", name = "enabled", havingValue = "true")
|
||||||
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory,
|
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory,
|
||||||
KafkaEnvelopeConsumerFactory consumerFactory,
|
KafkaEnvelopeConsumerFactory consumerFactory,
|
||||||
SinkMqProperties props) {
|
KafkaSinkProperties props) {
|
||||||
return new KafkaEnvelopeConsumerRunner(
|
return new KafkaEnvelopeConsumerRunner(
|
||||||
() -> createWorkers(beanFactory, consumerFactory, props),
|
() -> createWorkers(beanFactory, consumerFactory, props),
|
||||||
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
|
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
|
||||||
@@ -44,7 +44,7 @@ public class SinkMqConsumerAutoConfiguration {
|
|||||||
|
|
||||||
private List<KafkaEnvelopeConsumerWorker> createWorkers(ListableBeanFactory beanFactory,
|
private List<KafkaEnvelopeConsumerWorker> createWorkers(ListableBeanFactory beanFactory,
|
||||||
KafkaEnvelopeConsumerFactory consumerFactory,
|
KafkaEnvelopeConsumerFactory consumerFactory,
|
||||||
SinkMqProperties props) {
|
KafkaSinkProperties props) {
|
||||||
Map<String, EnvelopeConsumerProcessor> processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class);
|
Map<String, EnvelopeConsumerProcessor> processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class);
|
||||||
return consumerFactory.createWorkers(processors, props);
|
return consumerFactory.createWorkers(processors, props);
|
||||||
}
|
}
|
||||||
@@ -1,26 +1,23 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@ConfigurationProperties(prefix = "lingniu.ingest.sink.mq")
|
@ConfigurationProperties(prefix = "lingniu.ingest.sink.kafka")
|
||||||
public class SinkMqProperties {
|
public class KafkaSinkProperties {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* MQ Sink 总开关。默认 {@code true}。
|
* Kafka Sink 总开关。默认 {@code true}。
|
||||||
* 设为 {@code false} 时 {@link SinkMqAutoConfiguration} 完全不装配任何 Bean(Kafka Producer、
|
* 设为 {@code false} 时 {@link KafkaSinkAutoConfiguration} 完全不装配任何 Bean(Kafka Producer、
|
||||||
* EnvelopeMapper、TopicRouter、KafkaEventSink 都不会创建),ingest-core 的 DisruptorEventBus
|
* EnvelopeMapper、TopicRouter、KafkaEventSink 都不会创建),ingest-core 的 DisruptorEventBus
|
||||||
* 仍然正常运行但没有外部 sink(事件落地到 sink-archive 或 Noop 吞掉)。
|
* 仍然正常运行但没有外部 sink(事件落地到 sink-archive 或 Noop 吞掉)。
|
||||||
*/
|
*/
|
||||||
private boolean enabled = true;
|
private boolean enabled = true;
|
||||||
|
|
||||||
/** MQ 后端类型。生产链路只允许 kafka。 */
|
|
||||||
private String type = "kafka";
|
|
||||||
/** Kafka bootstrap servers;生产环境应通过环境变量覆盖,不建议使用默认开发地址。 */
|
/** Kafka bootstrap servers;生产环境应通过环境变量覆盖,不建议使用默认开发地址。 */
|
||||||
private String bootstrapServers = "114.55.58.251:9092";
|
private String bootstrapServers = "114.55.58.251:9092";
|
||||||
private String compressionType = "zstd";
|
private String compressionType = "zstd";
|
||||||
@@ -35,14 +32,6 @@ public class SinkMqProperties {
|
|||||||
|
|
||||||
public boolean isEnabled() { return enabled; }
|
public boolean isEnabled() { return enabled; }
|
||||||
public void setEnabled(boolean enabled) { this.enabled = enabled; }
|
public void setEnabled(boolean enabled) { this.enabled = enabled; }
|
||||||
public String getType() { return type; }
|
|
||||||
public void setType(String type) {
|
|
||||||
String value = type == null || type.isBlank() ? "kafka" : type.trim().toLowerCase(Locale.ROOT);
|
|
||||||
if (!"kafka".equals(value)) {
|
|
||||||
throw new IllegalStateException("sink.mq.type supports only kafka; configured value: " + type);
|
|
||||||
}
|
|
||||||
this.type = value;
|
|
||||||
}
|
|
||||||
public String getBootstrapServers() { return bootstrapServers; }
|
public String getBootstrapServers() { return bootstrapServers; }
|
||||||
public void setBootstrapServers(String bootstrapServers) { this.bootstrapServers = bootstrapServers; }
|
public void setBootstrapServers(String bootstrapServers) { this.bootstrapServers = bootstrapServers; }
|
||||||
public String getCompressionType() { return compressionType; }
|
public String getCompressionType() { return compressionType; }
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.event.VehicleEvent;
|
import com.lingniu.ingest.api.event.VehicleEvent;
|
||||||
|
|
||||||
@@ -7,9 +7,9 @@ import com.lingniu.ingest.api.event.VehicleEvent;
|
|||||||
*/
|
*/
|
||||||
public final class TopicRouter {
|
public final class TopicRouter {
|
||||||
|
|
||||||
private final SinkMqProperties.Topics topics;
|
private final KafkaSinkProperties.Topics topics;
|
||||||
|
|
||||||
public TopicRouter(SinkMqProperties.Topics topics) {
|
public TopicRouter(KafkaSinkProperties.Topics topics) {
|
||||||
this.topics = topics;
|
this.topics = topics;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
syntax = "proto3";
|
syntax = "proto3";
|
||||||
|
|
||||||
package com.lingniu.ingest.sink.mq.proto;
|
package com.lingniu.ingest.sink.kafka.proto;
|
||||||
|
|
||||||
option java_multiple_files = true;
|
option java_multiple_files = true;
|
||||||
option java_package = "com.lingniu.ingest.sink.mq.proto";
|
option java_package = "com.lingniu.ingest.sink.kafka.proto";
|
||||||
option java_outer_classname = "VehicleEnvelopeProto";
|
option java_outer_classname = "VehicleEnvelopeProto";
|
||||||
|
|
||||||
// 统一消息外壳:所有 Topic 共用此 Envelope,payload 通过 oneof 区分具体事件类型。
|
// 统一消息外壳:所有 Topic 共用此 Envelope,payload 通过 oneof 区分具体事件类型。
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration
|
com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration
|
||||||
com.lingniu.ingest.sink.mq.SinkMqConsumerAutoConfiguration
|
com.lingniu.ingest.sink.kafka.KafkaSinkConsumerAutoConfiguration
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.ProtocolId;
|
import com.lingniu.ingest.api.ProtocolId;
|
||||||
import com.lingniu.ingest.api.event.RawArchiveKeys;
|
import com.lingniu.ingest.api.event.RawArchiveKeys;
|
||||||
import com.lingniu.ingest.api.event.RealtimePayload;
|
import com.lingniu.ingest.api.event.RealtimePayload;
|
||||||
import com.lingniu.ingest.api.event.VehicleEvent;
|
import com.lingniu.ingest.api.event.VehicleEvent;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||||
@@ -23,7 +23,7 @@ class KafkaEnvelopeConsumerFactoryTest {
|
|||||||
created.add(props);
|
created.add(props);
|
||||||
return new MockConsumer<>(OffsetResetStrategy.EARLIEST);
|
return new MockConsumer<>(OffsetResetStrategy.EARLIEST);
|
||||||
});
|
});
|
||||||
SinkMqProperties props = new SinkMqProperties();
|
KafkaSinkProperties props = new KafkaSinkProperties();
|
||||||
props.setBootstrapServers("kafka-1:9092");
|
props.setBootstrapServers("kafka-1:9092");
|
||||||
EnvelopeConsumerProcessor stateProcessor = processor();
|
EnvelopeConsumerProcessor stateProcessor = processor();
|
||||||
EnvelopeConsumerProcessor statProcessor = processor();
|
EnvelopeConsumerProcessor statProcessor = processor();
|
||||||
@@ -51,7 +51,7 @@ class KafkaEnvelopeConsumerFactoryTest {
|
|||||||
created.add(props);
|
created.add(props);
|
||||||
return new MockConsumer<>(OffsetResetStrategy.EARLIEST);
|
return new MockConsumer<>(OffsetResetStrategy.EARLIEST);
|
||||||
});
|
});
|
||||||
SinkMqProperties props = new SinkMqProperties();
|
KafkaSinkProperties props = new KafkaSinkProperties();
|
||||||
props.getConsumer().setConcurrency(3);
|
props.getConsumer().setConcurrency(3);
|
||||||
EnvelopeConsumerProcessor stateProcessor = processor();
|
EnvelopeConsumerProcessor stateProcessor = processor();
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
|
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
|
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.ProtocolId;
|
import com.lingniu.ingest.api.ProtocolId;
|
||||||
import com.lingniu.ingest.api.event.LocationPayload;
|
import com.lingniu.ingest.api.event.LocationPayload;
|
||||||
@@ -26,7 +26,7 @@ class KafkaEventSinkTest {
|
|||||||
KafkaEventSink sink = new KafkaEventSink(
|
KafkaEventSink sink = new KafkaEventSink(
|
||||||
null,
|
null,
|
||||||
new EnvelopeMapper("node-1"),
|
new EnvelopeMapper("node-1"),
|
||||||
new TopicRouter(new SinkMqProperties.Topics()),
|
new TopicRouter(new KafkaSinkProperties.Topics()),
|
||||||
"vehicle.dlq.gb32960.v1",
|
"vehicle.dlq.gb32960.v1",
|
||||||
CircuitBreaker.ofDefaults("kafka-test"));
|
CircuitBreaker.ofDefaults("kafka-test"));
|
||||||
|
|
||||||
@@ -46,7 +46,7 @@ class KafkaEventSinkTest {
|
|||||||
KafkaEventSink sink = new KafkaEventSink(
|
KafkaEventSink sink = new KafkaEventSink(
|
||||||
producer,
|
producer,
|
||||||
new EnvelopeMapper("node-1"),
|
new EnvelopeMapper("node-1"),
|
||||||
new TopicRouter(new SinkMqProperties.Topics()),
|
new TopicRouter(new KafkaSinkProperties.Topics()),
|
||||||
"vehicle.dlq.gb32960.v1",
|
"vehicle.dlq.gb32960.v1",
|
||||||
CircuitBreaker.ofDefaults("kafka-test"));
|
CircuitBreaker.ofDefaults("kafka-test"));
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||||
@@ -15,26 +15,25 @@ import static org.assertj.core.api.Assertions.assertThat;
|
|||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
|
|
||||||
@ExtendWith(OutputCaptureExtension.class)
|
@ExtendWith(OutputCaptureExtension.class)
|
||||||
class SinkMqConsumerAutoConfigurationTest {
|
class KafkaSinkConsumerAutoConfigurationTest {
|
||||||
|
|
||||||
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
|
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
|
||||||
.withUserConfiguration(SinkMqAutoConfiguration.class, SinkMqConsumerAutoConfiguration.class)
|
.withUserConfiguration(KafkaSinkAutoConfiguration.class, KafkaSinkConsumerAutoConfiguration.class)
|
||||||
.withAllowBeanDefinitionOverriding(true)
|
.withAllowBeanDefinitionOverriding(true)
|
||||||
.withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
|
.withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
|
||||||
() -> new EnvelopeConsumerProcessor(
|
() -> new EnvelopeConsumerProcessor(
|
||||||
"vehicle-state",
|
"vehicle-state",
|
||||||
bytes -> EnvelopeIngestResult.processed("evt-1", "VIN001"),
|
bytes -> EnvelopeIngestResult.processed("evt-1", "VIN001"),
|
||||||
record -> {}))
|
record -> {}))
|
||||||
.withBean("kafkaProducer", KafkaProducer.class, SinkMqConsumerAutoConfigurationTest::kafkaProducer)
|
.withBean("kafkaProducer", KafkaProducer.class, KafkaSinkConsumerAutoConfigurationTest::kafkaProducer)
|
||||||
.withBean(KafkaEnvelopeConsumerFactory.class,
|
.withBean(KafkaEnvelopeConsumerFactory.class,
|
||||||
() -> new KafkaEnvelopeConsumerFactory(props -> new MockConsumer<>(OffsetResetStrategy.EARLIEST)))
|
() -> new KafkaEnvelopeConsumerFactory(props -> new MockConsumer<>(OffsetResetStrategy.EARLIEST)))
|
||||||
.withPropertyValues(
|
.withPropertyValues(
|
||||||
"lingniu.ingest.sink.mq.enabled=true",
|
"lingniu.ingest.sink.kafka.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.type=kafka",
|
"lingniu.ingest.sink.kafka.consumer.enabled=true",
|
||||||
"lingniu.ingest.sink.mq.consumer.enabled=true",
|
"lingniu.ingest.sink.kafka.consumer.auto-startup=false",
|
||||||
"lingniu.ingest.sink.mq.consumer.auto-startup=false",
|
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.group-id=vehicle-state",
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.group-id=vehicle-state",
|
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.topics[0]=vehicle.realtime");
|
||||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.topics[0]=vehicle.realtime");
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void createsKafkaEnvelopeConsumerRunnerWhenConsumerBindingIsConfigured(CapturedOutput output) {
|
void createsKafkaEnvelopeConsumerRunnerWhenConsumerBindingIsConfigured(CapturedOutput output) {
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|
||||||
|
class KafkaSinkPropertiesTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void defaultKafkaBrokerUsesProductionAddressButCanStillBeOverriddenByConfigBinding() {
|
||||||
|
KafkaSinkProperties props = new KafkaSinkProperties();
|
||||||
|
|
||||||
|
assertThat(props.getBootstrapServers()).isEqualTo("114.55.58.251:9092");
|
||||||
|
|
||||||
|
props.setBootstrapServers("kafka.internal:9092");
|
||||||
|
assertThat(props.getBootstrapServers()).isEqualTo("kafka.internal:9092");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void exposesNoBackendTypeBecauseKafkaIsTheOnlySink() {
|
||||||
|
assertThat(Arrays.stream(KafkaSinkProperties.class.getMethods()).map(method -> method.getName()))
|
||||||
|
.doesNotContain("getType", "setType");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.api.ProtocolId;
|
import com.lingniu.ingest.api.ProtocolId;
|
||||||
import com.lingniu.ingest.api.event.AlarmPayload;
|
import com.lingniu.ingest.api.event.AlarmPayload;
|
||||||
@@ -18,14 +18,14 @@ class TopicRouterTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void rawArchiveRoutesToVersionedGb32960RawTopicByDefault() {
|
void rawArchiveRoutesToVersionedGb32960RawTopicByDefault() {
|
||||||
TopicRouter router = new TopicRouter(new SinkMqProperties.Topics());
|
TopicRouter router = new TopicRouter(new KafkaSinkProperties.Topics());
|
||||||
|
|
||||||
assertThat(router.route(rawArchive())).isEqualTo("vehicle.raw.gb32960.v1");
|
assertThat(router.route(rawArchive())).isEqualTo("vehicle.raw.gb32960.v1");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void normalizedGb32960EventsRouteToVersionedEventTopicByDefault() {
|
void normalizedGb32960EventsRouteToVersionedEventTopicByDefault() {
|
||||||
TopicRouter router = new TopicRouter(new SinkMqProperties.Topics());
|
TopicRouter router = new TopicRouter(new KafkaSinkProperties.Topics());
|
||||||
|
|
||||||
assertThat(router.route(realtime())).isEqualTo("vehicle.event.gb32960.v1");
|
assertThat(router.route(realtime())).isEqualTo("vehicle.event.gb32960.v1");
|
||||||
assertThat(router.route(location())).isEqualTo("vehicle.event.gb32960.v1");
|
assertThat(router.route(location())).isEqualTo("vehicle.event.gb32960.v1");
|
||||||
@@ -37,7 +37,7 @@ class TopicRouterTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void dlqDefaultsToVersionedGb32960DlqTopic() {
|
void dlqDefaultsToVersionedGb32960DlqTopic() {
|
||||||
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
|
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
|
||||||
|
|
||||||
assertThat(topics.getDlq()).isEqualTo("vehicle.dlq.gb32960.v1");
|
assertThat(topics.getDlq()).isEqualTo("vehicle.dlq.gb32960.v1");
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
package com.lingniu.ingest.sink.kafka;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.DecodedFactPayload;
|
import com.lingniu.ingest.sink.kafka.proto.DecodedFactPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
|
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
|
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import static org.assertj.core.api.Assertions.assertThat;
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package com.lingniu.ingest.sink.mq;
|
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import static org.assertj.core.api.Assertions.assertThat;
|
|
||||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
|
||||||
|
|
||||||
class SinkMqPropertiesTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void defaultKafkaBrokerUsesProductionAddressButCanStillBeOverriddenByConfigBinding() {
|
|
||||||
SinkMqProperties props = new SinkMqProperties();
|
|
||||||
|
|
||||||
assertThat(props.getBootstrapServers()).isEqualTo("114.55.58.251:9092");
|
|
||||||
|
|
||||||
props.setBootstrapServers("kafka.internal:9092");
|
|
||||||
assertThat(props.getBootstrapServers()).isEqualTo("kafka.internal:9092");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void rejectsUnsupportedMqTypeInsteadOfSilentlyDisablingKafkaBeans() {
|
|
||||||
SinkMqProperties props = new SinkMqProperties();
|
|
||||||
|
|
||||||
assertThatThrownBy(() -> props.setType("rocketmq"))
|
|
||||||
.isInstanceOf(IllegalStateException.class)
|
|
||||||
.hasMessageContaining("sink.mq.type supports only kafka")
|
|
||||||
.hasMessageContaining("rocketmq");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,10 +2,10 @@ package com.lingniu.ingest.tdenginehistory;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.lingniu.ingest.sink.mq.proto.LocationPayload;
|
import com.lingniu.ingest.sink.kafka.proto.LocationPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
|
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
|
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
package com.lingniu.ingest.tdenginehistory;
|
package com.lingniu.ingest.tdenginehistory;
|
||||||
|
|
||||||
import com.lingniu.ingest.sink.mq.proto.LocationPayload;
|
import com.lingniu.ingest.sink.kafka.proto.LocationPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
|
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
import com.lingniu.ingest.sink.kafka.proto.RawArchiveRef;
|
||||||
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
|
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
|
||||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
import com.lingniu.ingest.sink.kafka.proto.TelemetrySnapshot;
|
||||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
|
|||||||
Reference in New Issue
Block a user