shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: productionize raw history ingestion

Browse Source
This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
modules/apps/jt808-ingest-app/src/main/resources/application.yml
View File

@@ -52,12 +52,11 @@ lingniu:
file:
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
mysql:
table: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:}
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
driver-class-name: ${VEHICLE_IDENTITY_MYSQL_DRIVER_CLASS_NAME:com.mysql.cj.jdbc.Driver}
refresh-interval: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
enabled: ${KAFKA_ENABLED:true}
@@ -79,10 +78,6 @@ lingniu:
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
consumer:
enabled: false
archive:
enabled: ${SINK_ARCHIVE_ENABLED:true}
type: local
path: ${SINK_ARCHIVE_PATH:./archive/}
event-file-store:
enabled: false
event-history:

14
modules/apps/jt808-ingest-app/src/test/java/com/lingniu/ingest/jt808app/Jt808IngestAppDefaultsTest.java
View File

@@ -29,12 +29,14 @@ class Jt808IngestAppDefaultsTest {
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}")
.containsEntry("lingniu.ingest.identity.mysql.table", "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}")
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url", "${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}")
.containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:}")
.containsEntry("lingniu.ingest.identity.mysql.refresh-interval", "${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}")
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
.containsEntry("lingniu.ingest.identity.mysql.table-name",
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url",
"${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}")
.containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:root}")
.containsEntry("lingniu.ingest.identity.mysql.password", "${VEHICLE_IDENTITY_MYSQL_PASSWORD:}")
.containsEntry("lingniu.ingest.identity.mysql.initialize-schema",
"${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}")
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
.containsEntry("lingniu.ingest.event-history.enabled", false)
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)

4
modules/apps/vehicle-analytics-app/pom.xml
View File

@@ -33,6 +33,10 @@
<groupId>com.lingniu.ingest</groupId>
<artifactId>vehicle-stat-service</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>

2
modules/apps/vehicle-analytics-app/src/main/resources/application.yml
View File

@@ -26,6 +26,7 @@ spring:
redis:
host: ${REDIS_HOST:}
port: ${REDIS_PORT:6379}
username: ${REDIS_USERNAME:}
password: ${REDIS_PASSWORD:}
database: ${REDIS_DATABASE:50}
@@ -55,6 +56,7 @@ lingniu:
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:500}
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:900000}
bindings:
vehicleStateEnvelopeConsumerProcessor:
enabled: ${VEHICLE_STATE_ENABLED:false}

2
modules/apps/vehicle-history-app/src/main/resources/application.yml
View File

@@ -46,7 +46,7 @@ lingniu:
enabled: ${KAFKA_CONSUMER_ENABLED:true}
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}
concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3}
bindings:

2
modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java
View File

@@ -43,7 +43,7 @@ class VehicleHistoryAppDefaultsTest {
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}")
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
.containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.max-poll-interval-millis",