shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: expose kafka sink configuration

Browse Source
This commit is contained in:
lingniu
2026-07-01 10:50:50 +08:00
parent 303b2eb47e
commit 849e8f8a8b
67 changed files with 350 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
docs/superpowers/plans/2026-06-23-gb32960-service-split.md
View File

@@ -18,7 +18,7 @@
- Existing handler currently writes report ACK before `dispatcher.dispatch(rf)`.
- Existing `DisruptorEventBus` publishes to sinks asynchronously and does not wait for sink completion.
- Existing `KafkaEventSink.accepts()` rejects `VehicleEvent.RawArchive`.
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.mq.topics`.
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.kafka.topics`.
- Existing history consumer logic: `modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java`.
- Existing analytics consumers: `VehicleStateEnvelopeIngestor` and `VehicleStatEnvelopeIngestor`.
- There is an unrelated working-tree change in `modules/apps/bootstrap-all/src/main/resources/application.yml` for GB32960 diagnostics capacity. Do not revert it. Do not include it in service-split commits unless the user explicitly asks.
@@ -48,7 +48,7 @@ Create app modules:
Modify shared build and Kafka contract files:
- `pom.xml`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
@@ -466,16 +466,19 @@ lingniu:
rate-limit:
per-vin-qps: 50
session:
store: ${SESSION_STORE:memory}
store: ${SESSION_STORE:redis}
ttl: ${SESSION_TTL:30m}
identity:
store: ${VEHICLE_IDENTITY_STORE:file}
file:
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
store: ${VEHICLE_IDENTITY_STORE:mysql}
mysql:
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd
linger-ms: 20
@@ -568,9 +571,8 @@ lingniu:
gb32960:
enabled: false
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
@@ -666,9 +668,8 @@ lingniu:
gb32960:
enabled: false
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
@@ -754,7 +755,7 @@ import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
@@ -770,7 +771,7 @@ class Gb32960IngestAppCompositionTest {
.withPropertyValues(
"lingniu.ingest.gb32960.enabled=true",
"lingniu.ingest.gb32960.port=0",
"lingniu.ingest.sink.mq.enabled=false",
"lingniu.ingest.sink.kafka.enabled=false",
"lingniu.ingest.sink.archive.enabled=false",
"lingniu.ingest.event-file-store.enabled=false",
"lingniu.ingest.event-history.enabled=false",
@@ -835,7 +836,7 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=false",
"lingniu.ingest.sink.mq.enabled=false")
"lingniu.ingest.sink.kafka.enabled=false")
.withConfiguration(AutoConfigurations.of(
com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration.class,
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration.class,
@@ -896,7 +897,7 @@ class VehicleAnalyticsAppCompositionTest {
"lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=true",
"lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"),
"lingniu.ingest.sink.mq.enabled=false")
"lingniu.ingest.sink.kafka.enabled=false")
.withConfiguration(AutoConfigurations.of(
com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration.class,
com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration.class))
@@ -932,7 +933,7 @@ git commit -m "test: cover split service composition"
### Task 4: Version Kafka Topics for GB32960 Raw/Event/DLQ
**Files:**
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java`
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
@@ -947,7 +948,7 @@ Expected test names:
```java
@Test
void rawArchiveRoutesToVersionedGb32960RawTopic() {
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
topics.setRawArchive("vehicle.raw.gb32960.v1");
TopicRouter router = new TopicRouter(topics);
@@ -970,7 +971,7 @@ Also add a normalized realtime event test:
```java
@Test
void realtimeRoutesToVersionedGb32960EventTopic() {
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
topics.setRealtime("vehicle.event.gb32960.v1");
TopicRouter router = new TopicRouter(topics);
@@ -1000,9 +1001,9 @@ mvn -pl :sink-mq -Dtest=TopicRouterTest test
Expected before implementation: compile failure if constructors are wrong or assertion failure if defaults are old. Fix constructors first; keep routing assertions.
- [ ] **Step 3: Update `SinkMqProperties.Topics` defaults**
- [ ] **Step 3: Update `KafkaSinkProperties.Topics` defaults**
In `SinkMqProperties.Topics`, change defaults:
In `KafkaSinkProperties.Topics`, change defaults:
```java
private String realtime = "vehicle.event.gb32960.v1";
@@ -1040,7 +1041,7 @@ Expected: PASS.
- [ ] **Step 6: Commit**
```bash
git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java \
git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaSinkProperties.java \
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java \
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java \
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java \
@@ -1353,8 +1354,8 @@ void gb32960IngestDefaultsOnlyEnableProtocolAndKafkaProducer() {
assertThat(props.getProperty("spring.application.name")).isEqualTo("gb32960-ingest-app");
assertThat(props.getProperty("lingniu.ingest.gb32960.enabled")).isEqualTo("true");
assertThat(props.getProperty("lingniu.ingest.gb32960.port")).isEqualTo("${GB32960_PORT:32960}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.sink.kafka.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
@@ -1378,7 +1379,7 @@ void historyDefaultsEnableStorageAndHistoryConsumerOnly() {
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("true");
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
.isEqualTo("${KAFKA_GROUP_HISTORY:vehicle-history}");
}
```
@@ -1398,7 +1399,7 @@ void analyticsDefaultsEnableStatConsumerAndDisableProtocolAndHistoryStorage() {
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("${VEHICLE_STAT_ENABLED:true}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
.isEqualTo("${KAFKA_GROUP_STAT:vehicle-stat}");
}
```

10
docs/superpowers/plans/2026-06-29-vehicle-ingest-redesign-phase1.md
View File

@@ -1269,12 +1269,12 @@ git commit -m "feat: add raw archive store contract"
Create `VehicleEnvelopeProtoCompatibilityTest.java`:
```java
package com.lingniu.ingest.sink.mq;
package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.sink.mq.proto.DecodedFactPayload;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.lingniu.ingest.sink.kafka.proto.DecodedFactPayload;
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;

2
docs/superpowers/specs/2026-06-23-gb32960-service-split-design.md
View File

@@ -133,7 +133,7 @@ Use versioned topics:
- `vehicle.event.gb32960.v1`
- `vehicle.dlq.gb32960.v1`
The current topic names under `lingniu.ingest.sink.mq.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
The current topic names under `lingniu.ingest.sink.kafka.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
### Keys

2
docs/target-architecture.md
View File

@@ -71,7 +71,7 @@ repositories, and statistic calculators.
### Event Contract
`sink-mq` owns the Kafka protobuf envelope and consumer plumbing.
The Kafka sink owns the protobuf envelope and consumer plumbing.
Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped
envelopes, and downstream failures are converted to structured results and DLQ
records instead of blocking a partition.