Compare commits
228 Commits
3e19c72915
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbb6f3b741 | ||
|
|
42de422935 | ||
|
|
318486b1e9 | ||
|
|
0a024f9a76 | ||
|
|
ea9d9f5d35 | ||
|
|
409f55b5ae | ||
|
|
8def635bf5 | ||
|
|
c211447747 | ||
|
|
1741aacaaa | ||
|
|
27f4ad3468 | ||
|
|
601150b364 | ||
|
|
5b7f386c5a | ||
|
|
f4d9c9ef5a | ||
|
|
96514898eb | ||
|
|
0ef88e1a4c | ||
|
|
0fd4d319f9 | ||
|
|
4e8434c511 | ||
|
|
576644eb65 | ||
|
|
9bd48c5781 | ||
|
|
2214abe9c5 | ||
|
|
1477b997c9 | ||
|
|
4ea8d18972 | ||
|
|
947e645090 | ||
|
|
78b805e3a2 | ||
|
|
a7b24f4c0b | ||
|
|
eb124d929b | ||
|
|
4711287655 | ||
|
|
2c224396c9 | ||
|
|
4f2cded760 | ||
|
|
7ad1a25e8e | ||
|
|
3ee1d6f52b | ||
|
|
6d1d0aa85e | ||
|
|
055373c405 | ||
|
|
e37dd39c89 | ||
|
|
dd73fdbb5b | ||
|
|
fb9ab8525a | ||
|
|
cc89e0d537 | ||
|
|
f3ecf430cc | ||
|
|
bcf7cdae2e | ||
|
|
9045b871d6 | ||
|
|
b0150a5503 | ||
|
|
8f3b337fd6 | ||
|
|
594ab2d9c7 | ||
|
|
229ffcf61f | ||
|
|
593713283c | ||
|
|
6bee0499ae | ||
|
|
59c1c9df0f | ||
|
|
56f4811c68 | ||
|
|
1a53011408 | ||
|
|
c2d70dad86 | ||
|
|
91db6c959e | ||
|
|
7b44f97ddb | ||
|
|
a72977cd79 | ||
|
|
dd95101499 | ||
|
|
d347525a75 | ||
|
|
19008e840c | ||
|
|
c803a19bf5 | ||
|
|
243de5c2b2 | ||
|
|
4a32e2181a | ||
|
|
7e7066dd86 | ||
|
|
e7c8c57296 | ||
|
|
6b55694c43 | ||
|
|
eb48d16181 | ||
|
|
a15de91912 | ||
|
|
796d79dd30 | ||
|
|
5f4f4fd124 | ||
|
|
e0068a750d | ||
|
|
665122b23b | ||
|
|
60ca42e5d5 | ||
|
|
80474eeccb | ||
|
|
17024651c8 | ||
|
|
d8c77d7882 | ||
|
|
b55b075a97 | ||
|
|
bac8864e79 | ||
|
|
5d844701ff | ||
|
|
967981c040 | ||
|
|
1fdbc29e27 | ||
|
|
d148a4e3af | ||
|
|
cde55e1ae6 | ||
|
|
e06534d6a5 | ||
|
|
4eaaa88e9c | ||
|
|
f89ac1c23f | ||
|
|
c8abc721e1 | ||
|
|
c8dac638b2 | ||
|
|
4591919a31 | ||
|
|
6e24bf615d | ||
|
|
96b8474182 | ||
|
|
fe4ed5e88f | ||
|
|
71353a3cb4 | ||
|
|
37844e4425 | ||
|
|
ec4aed3ca9 | ||
|
|
6ac77b2f7d | ||
|
|
5deadd06af | ||
|
|
ccf77b4596 | ||
|
|
f4ce4557c0 | ||
|
|
2e6a982af5 | ||
|
|
68253901cf | ||
|
|
fc81693285 | ||
|
|
61cc15ab07 | ||
|
|
b7bb93de8e | ||
|
|
5971d1a8c3 | ||
|
|
c359a0683e | ||
|
|
bf6f94afbb | ||
|
|
4d9779e088 | ||
|
|
80c051e056 | ||
|
|
53f8f79c3e | ||
|
|
dc56a664f4 | ||
|
|
9cd5c5a35d | ||
|
|
192c5eff00 | ||
|
|
329f106e3c | ||
|
|
15c57c33b9 | ||
|
|
664a0c801b | ||
|
|
4ea4469640 | ||
|
|
a6d0236fc4 | ||
|
|
3161e98eff | ||
|
|
5910d23dd9 | ||
|
|
faeee0c2a8 | ||
|
|
90791d28bd | ||
|
|
3795e43a6a | ||
|
|
aa30a413c5 | ||
|
|
0d77f7777f | ||
|
|
b979d7a020 | ||
|
|
f7bb5b1554 | ||
|
|
cd12a84e51 | ||
|
|
1eed7623d1 | ||
|
|
1cca142550 | ||
|
|
a6784bc8c3 | ||
|
|
eeba8fb754 | ||
|
|
476d5a7a14 | ||
|
|
c3d67e4339 | ||
|
|
68f45ee79c | ||
|
|
a5cb0c9705 | ||
|
|
932b386c69 | ||
|
|
981879136f | ||
|
|
27f9cbf0d2 | ||
|
|
cea8ceeabe | ||
|
|
97ea77ed27 | ||
|
|
85013a0ebe | ||
|
|
57ab57c42b | ||
|
|
28bde233a1 | ||
|
|
4134824caf | ||
|
|
422a24684d | ||
|
|
78c39036cd | ||
|
|
495ded0547 | ||
|
|
252aff984c | ||
|
|
dc481e8653 | ||
|
|
9fb77a1aa2 | ||
|
|
afed6cd4f3 | ||
|
|
fd76bfdf61 | ||
|
|
3185696b6a | ||
|
|
641114696c | ||
|
|
2deea2d072 | ||
|
|
3d9a520c3c | ||
|
|
e81d8dcab5 | ||
|
|
a5b39e22b6 | ||
|
|
bc60017c30 | ||
|
|
5ff7cfb9bb | ||
|
|
59edc7a8d4 | ||
|
|
a76cd3a8d6 | ||
|
|
2d65a2555f | ||
|
|
5ede13e6d4 | ||
|
|
5ec3be97aa | ||
|
|
12423efd64 | ||
|
|
5beb81f8cd | ||
|
|
77d5aa4ef7 | ||
|
|
c6347c50a4 | ||
|
|
d4c54b5d0f | ||
|
|
ebbf2ca8fb | ||
|
|
61ad4a2439 | ||
|
|
986fc2fac3 | ||
|
|
7ce64e9963 | ||
|
|
e80fa6f38c | ||
|
|
849e8f8a8b | ||
|
|
303b2eb47e | ||
|
|
e2b6ab63d6 | ||
|
|
87830fee09 | ||
|
|
53a9b3b3b7 | ||
|
|
163f313066 | ||
|
|
7b0b69295f | ||
|
|
74e01acc7a | ||
|
|
418aefc189 | ||
|
|
da17acb09a | ||
|
|
136b8a040c | ||
|
|
0236884f81 | ||
|
|
bb051391c2 | ||
|
|
ce93236a5a | ||
|
|
ba1ec9a012 | ||
|
|
072fa00ea9 | ||
|
|
74fbdd2b1a | ||
|
|
d9329f4364 | ||
|
|
bdc035185e | ||
|
|
d491f543b5 | ||
|
|
5d082d50d5 | ||
|
|
6faf3533a5 | ||
|
|
ce3f2ffb90 | ||
|
|
42e8742b10 | ||
|
|
a321b6351d | ||
|
|
7c51c83220 | ||
|
|
9db8b15ae3 | ||
|
|
2ab1eeb1f8 | ||
|
|
dcd5f2a1a2 | ||
|
|
625d09c856 | ||
|
|
f80c47df0f | ||
|
|
a08a132a3a | ||
|
|
bb6ffa86fd | ||
|
|
62ef3f5bc1 | ||
|
|
8c61ae7139 | ||
|
|
37902be034 | ||
|
|
e745419f76 | ||
|
|
80d3afa80a | ||
|
|
59e8315fe6 | ||
|
|
56a6c01f95 | ||
|
|
51b1dd4174 | ||
|
|
ba4bb51a07 | ||
|
|
0987bb4718 | ||
|
|
4f83cd2792 | ||
|
|
e47e3eebb4 | ||
|
|
cf4d369335 | ||
|
|
9bd61ac91b | ||
|
|
18aa32a721 | ||
|
|
40dea58278 | ||
|
|
e03d1d7629 | ||
|
|
6ccf8fdc26 | ||
|
|
442898cc0b | ||
|
|
ef1fd6e4f5 | ||
|
|
7c3f25eaf6 | ||
|
|
d8f8235544 | ||
|
|
be2daf986a |
@@ -54,13 +54,16 @@
|
||||
V2016/V2025 双版本信息体 parser、平台登入鉴权、VIN 白名单、读空闲告警。
|
||||
- **JT/T 808 / JT/T 1078 / JSATL12 接入**(`protocol-jt808` / `protocol-jt1078` /
|
||||
`protocol-jsatl12`)。
|
||||
- **MQTT 入站**与**信达平台 push 接入**(`inbound-mqtt` / `inbound-xinda-push`)。
|
||||
- **MQTT 入站**(`inbound-mqtt`)。
|
||||
- **Disruptor 事件总线**(`ingest-core`):高吞吐解耦解码与下游 sink。
|
||||
- **Sink**:本地归档(`sink-archive`)、Kafka(`sink-mq`,含 protobuf envelope)。
|
||||
- **Sink**:本地归档(`sink-archive`)、Kafka(`sink-kafka`,含 protobuf envelope)。
|
||||
- **会话状态**(`session-core`)。
|
||||
- **可观测性**:Micrometer/Actuator 装配(`observability`)。
|
||||
- **终端控制命令网关**(`command-gateway`)。
|
||||
|
||||
### Removed
|
||||
- 旧文件型事件索引模块和 DuckDB 依赖;历史查询统一收敛到 TDengine 与 `archive://...` 原始报文引用。
|
||||
|
||||
### Fixed —— GB/T 32960 应答帧时间字段
|
||||
- `Gb32960MessageDecoder` 对 `VEHICLE_LOGIN` / `VEHICLE_LOGOUT` /
|
||||
`PLATFORM_LOGIN` / `PLATFORM_LOGOUT` 命令预解析 body 首 6B 采集时间,写入
|
||||
|
||||
80
DECISIONS.md
80
DECISIONS.md
@@ -2,11 +2,11 @@
|
||||
|
||||
> 本文件记录 lingniu-vehicle-ingest v2 的关键架构决策,后续每次重大调整追加条目(不删除旧条目)。
|
||||
|
||||
## ADR-001 MQ 选型:Kafka
|
||||
## ADR-001 消息通道:Kafka
|
||||
- **Status**: Accepted
|
||||
- **Context**: 需要高吞吐、严格单车有序、成熟生态
|
||||
- **Decision**: 使用 Kafka,分区 key = VIN
|
||||
- **Consequences**: 下游消费者需使用消费组 + 分区内顺序消费;不使用 RocketMQ 的事务/顺序特性
|
||||
- **Consequences**: 下游消费者需使用消费组 + 分区内顺序消费;生产链路只支持 Kafka。
|
||||
|
||||
## ADR-002 线上消息格式:Protobuf
|
||||
- **Status**: Accepted
|
||||
@@ -20,27 +20,22 @@
|
||||
- **Decision**: 拆出独立 `command-gateway` 模块,复用 `session-core`;本次重构同步交付
|
||||
- **Consequences**: HTTP 对外接口(原 JT808Controller / JT1078Controller)迁移到此模块
|
||||
|
||||
## ADR-004 信达 Push:保留模块但彻底重写
|
||||
- **Status**: Accepted
|
||||
## ADR-004 信达 Push:Removed
|
||||
- **Status**: Superseded by ADR-011
|
||||
- **Context**: 旧实现硬编码凭证、无重连、0300/0401 空实现
|
||||
- **Decision**:
|
||||
1. 新模块 `inbound-xinda-push` 隔离第三方库 `gps-push-client`
|
||||
2. 配置外置(yml + env)
|
||||
3. 自动重连 + Resilience4j 熔断
|
||||
4. 0200/0300/0401 **仅做协议解析**,转换为 `VehicleEvent` 后直接投 Kafka
|
||||
5. **业务处理全部下沉到下游消费者**,本服务不再实现报警/透传业务逻辑
|
||||
- **Consequences**: 旧代码 messageReceived 里的 TODO 不再回到本仓库;需要协调下游消费者团队认领
|
||||
- **Decision**: 删除信达 Push 源码、Maven profile、私仓依赖和部署入口。
|
||||
- **Consequences**: 信达 Push 后续已从源码和构建面删除,优化优先投向 GB32960、JT808、宇通 MQTT、历史和统计链路。
|
||||
|
||||
## ADR-005 JT1078 / JSATL12 进第一批
|
||||
- **Status**: Accepted
|
||||
- **Status**: Superseded by ADR-012
|
||||
- **Decision**: 第一批迁移即覆盖 JT1078 信令 + JSATL12 报警附件上传
|
||||
- **Consequences**: Phase 2 工期相应拉长;JSATL12 需要对象存储后端(本地 / S3 / OSS)
|
||||
|
||||
## ADR-006 部署形态:GB32960 三应用拆分
|
||||
- **Status**: Superseded
|
||||
- **Status**: Superseded by ADR-011
|
||||
- **Context**: 原 `bootstrap-all` 一体化启动已被 GB32960 拆分部署替代
|
||||
- **Decision**: 生产默认部署 `gb32960-ingest-app`、`vehicle-history-app`、`vehicle-analytics-app` 三个应用;旧 `bootstrap-all` 模块删除
|
||||
- **Consequences**: CI/CD 分别构建三个镜像;协议接入、历史查询、统计消费可以独立发布和回滚
|
||||
- **Decision**: 当时生产默认部署 `gb32960-ingest-app`、`vehicle-history-app`、`vehicle-analytics-app`;旧 `bootstrap-all` 模块删除
|
||||
- **Consequences**: 该三应用边界已由 ADR-011 的三协议接入 + 历史 + 统计生产面取代;协议接入、历史查询、统计消费仍保持独立发布和回滚。
|
||||
|
||||
## ADR-007 Java 25 + 虚拟线程 + Disruptor
|
||||
- **Status**: Accepted
|
||||
@@ -51,15 +46,60 @@
|
||||
- 禁用 `synchronized`,使用 `ReentrantLock` 避免虚拟线程 pinning
|
||||
- **Consequences**: 需开启 `jdk.VirtualThreadPinned` JFR 监控
|
||||
|
||||
## ADR-008 本服务不写业务库
|
||||
## ADR-008 协议接入应用不直接写业务库
|
||||
- **Status**: Accepted
|
||||
- **Decision**: 所有业务表(vehicle_data_actual / history / travel / gps_data 等)不再由本服务写入
|
||||
- **Consequences**: 需要新建消费服务 `vehicle-analytics-service` 承接;迁移期采用双跑对账
|
||||
- **Decision**: GB32960、JT808、Yutong MQTT 等协议接入应用只负责收、解析、校验、规整和投递 Kafka,不直接写业务表。
|
||||
- **Consequences**: 业务落库由独立 Kafka 消费应用承接:`vehicle-history-app` 写入 TDengine 历史与 RAW JSON,`vehicle-analytics-app` 写入 MySQL `vehicle_stat_metric`。
|
||||
|
||||
## ADR-009 构建工具:Maven
|
||||
- **Status**: Accepted
|
||||
- **Decision**: 沿用 Maven,统一 BOM 管理版本,Spotless 格式化,ArchUnit 守护分层
|
||||
|
||||
## ADR-010 框架:Spring Boot 3.4
|
||||
## ADR-010 框架:Spring Boot 3.5
|
||||
- **Status**: Accepted
|
||||
- **Decision**: Spring Boot 3.4.x,支持 JDK 25;AutoConfiguration 用于按需启停
|
||||
- **Decision**: Spring Boot 3.5.x,支持 JDK 25;AutoConfiguration 用于按需启停
|
||||
|
||||
## ADR-011 默认生产面:三协议接入 + 历史 + 统计
|
||||
- **Status**: Accepted
|
||||
- **Context**: 生产接入范围已经从 GB32960 三应用拆分,扩展为三条活跃接入链路和两个消费应用;信达 Push 已废弃并删除,不应再出现在构建、部署或优化目标里。
|
||||
- **Decision**:
|
||||
1. 默认生产面包含 GB32960、JT808、Yutong MQTT、vehicle-history-app、vehicle-analytics-app。
|
||||
2. Xinda Push 源码、Maven profile、Woodpecker 镜像发布和历史消费绑定全部移除。
|
||||
3. `command-gateway` 和 JT1078 继续作为可选能力,通过 `optional-command-gateway` profile 显式启用。
|
||||
- **Consequences**: 默认构建和部署保持精简;后续性能、可靠性、字段解析和存储优化都服务三条活跃协议链路。
|
||||
|
||||
## ADR-012 JSATL12 附件上传:Optional only
|
||||
- **Status**: Accepted
|
||||
- **Context**: 当前默认生产面只部署 GB32960、JT808、Yutong MQTT、vehicle-history-app、vehicle-analytics-app;JSATL12 附件上传没有独立生产 app,也不在 Portainer 和 Woodpecker 活跃镜像列表中。
|
||||
- **Decision**:
|
||||
1. `protocol-jsatl12` 不进入默认 Maven reactor。
|
||||
2. JSATL12 仅保留在 `optional-attachments` profile,需要附件上传能力时显式构建。
|
||||
3. 默认优化和验证优先覆盖活跃接入链路,附件上传能力保持源码可用但不增加默认构建面。
|
||||
- **Consequences**: 默认构建更轻,生产部署边界更清晰;附件上传相关测试需要通过 `-Poptional-attachments` 显式运行。
|
||||
|
||||
## ADR-013 最新状态服务:Optional only
|
||||
- **Status**: Accepted
|
||||
- **Context**: Redis 最新状态查询是独立消费能力,但当前默认 Portainer 部署和 Woodpecker 镜像发布只包含三条接入链路、history 和 analytics;`vehicle-state-service` 没有独立 app,也不应增加默认构建面。
|
||||
- **Decision**:
|
||||
1. `vehicle-state-service` 不进入默认 Maven reactor。
|
||||
2. vehicle-state-service 仅保留在 `optional-latest-state` profile,需要 Redis 最新状态查询能力时显式构建。
|
||||
3. 默认优化和验证优先覆盖活跃接入、TDengine 历史和 MySQL 指标链路。
|
||||
- **Consequences**: 默认构建和部署边界继续收窄;最新状态能力保持源码可用,但其测试需要通过 `-Poptional-latest-state` 显式运行。
|
||||
|
||||
## ADR-014 raw-archive-store 原型已删除
|
||||
- **Status**: Accepted
|
||||
- **Context**: 默认生产 raw bytes 写入由 `sink-archive` 负责,历史查询通过 TDengine `raw_frames` 和 `archive://...` 引用追溯;`raw-archive-store` 只是未部署的独立读写原型,继续保留会造成 raw archive 路径歧义。
|
||||
- **Decision**:
|
||||
1. 删除 `raw-archive-store` 模块和对应 optional profile。
|
||||
2. 默认生产链路只保留 `sink-archive`、Kafka raw topic、TDengine raw_frames 这条 raw 路径。
|
||||
3. 如需新的 archive store 能力,先以生产链路需求重新设计,不恢复旧原型。
|
||||
- **Consequences**: 默认构建面和可选构建面继续收窄;raw bytes 写入职责集中在 `sink-archive`。
|
||||
|
||||
## ADR-015 文件型事件索引:Removed
|
||||
- **Status**: Accepted
|
||||
- **Context**: 默认历史查询已收敛到 TDengine `raw_frames`、`vehicle_locations` 和按需解码;旧文件型索引会增加一套无生产部署的查询和依赖边界。
|
||||
- **Decision**:
|
||||
1. 删除旧文件型事件索引模块和 profile。
|
||||
2. 父 POM 不再管理旧索引驱动依赖。
|
||||
3. 历史查询和 RAW 回放统一以 TDengine + `archive://...` 引用为准。
|
||||
- **Consequences**: 默认构建面和可选构建面都更小;需要历史查询时只维护 TDengine 一条路径。
|
||||
|
||||
10
Dockerfile
10
Dockerfile
@@ -5,11 +5,15 @@ ARG APP_VERSION
|
||||
|
||||
ENV APP_NAME=${APP_NAME}
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
ENV JAVA_OPTS=""
|
||||
ENV JAVA_OPTS="--sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true"
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY modules/apps/${APP_NAME}/target/${APP_NAME}.jar /app/app.jar
|
||||
|
||||
EXPOSE 20100 20200 20300 20400 20500 32960 10808
|
||||
EXPOSE 20100 20200 20310 20400 20500 32960 808
|
||||
|
||||
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app/app.jar"]
|
||||
ENTRYPOINT ["sh", "-c", "exec java $JAVA_OPTS -jar /app/app.jar"]
|
||||
|
||||
75
README.md
75
README.md
@@ -4,9 +4,9 @@
|
||||
|
||||
## 设计目标
|
||||
|
||||
- **协议接入统一抽象**:GB/T 32960、JT/T 808、JT/T 1078、JSATL12、MQTT 全部变成可插拔 Inbound Adapter;信达 Push 仅保留为废弃兼容模块
|
||||
- **协议接入统一抽象**:GB/T 32960、JT/T 808、Yutong MQTT 是默认生产接入;JT/T 1078、JSATL12 作为显式 profile 的可选能力;信达 Push 已废弃并删除源码
|
||||
- **原子能力化**:每个协议一个独立 Maven 模块 + 独立 AutoConfiguration + 独立配置开关
|
||||
- **业务 / 实时彻底解耦**:本服务只负责 **收 → 解析 → 校验 → 规整 → 投递 Kafka**,不碰业务库
|
||||
- **业务 / 实时彻底解耦**:协议接入应用只负责收、解析、校验、规整和投递 Kafka;历史与统计由独立 Kafka 消费应用落库
|
||||
- **高并发低延迟**:Netty + Disruptor + Java 25 虚拟线程;目标单节点 ≥ 5 万 msg/s,P99 < 50 ms
|
||||
- **可观测、可回放、可灰度**:统一 traceId、Envelope、幂等键、DLQ、原始报文冷存
|
||||
|
||||
@@ -15,17 +15,17 @@
|
||||
| 类别 | 选型 |
|
||||
|---|---|
|
||||
| 语言 | Java **25** |
|
||||
| 框架 | Spring Boot 3.4.x |
|
||||
| 网络 | Netty 4.1 |
|
||||
| 并发 | Disruptor 4 + 虚拟线程 + `StructuredTaskScope` |
|
||||
| 消息队列 | **Kafka**(vin 分区,保证单车有序) |
|
||||
| 框架 | Spring Boot 3.5.3 |
|
||||
| 网络 | Netty 4.2.9.Final |
|
||||
| 并发 | Disruptor 4 + 虚拟线程 |
|
||||
| 事件流 | **Kafka**(唯一生产消息通道,vin 分区,保证单车有序) |
|
||||
| 线上消息格式 | **Protobuf**,调试走 JSON |
|
||||
| MQTT 客户端 | HiveMQ MQTT Client |
|
||||
| 会话缓存 | Redis 生产索引 + Caffeine 内存降级 |
|
||||
| MQTT 客户端 | Eclipse Paho 1.2.5 |
|
||||
| 会话索引 | Redis |
|
||||
| 熔断/限流 | Resilience4j 2 |
|
||||
| 可观测 | Micrometer + Prometheus + OpenTelemetry |
|
||||
| 冷存 | S3 / OSS / 本地 |
|
||||
| 构建 | Maven + Spotless + ArchUnit |
|
||||
| 可观测 | Micrometer + Prometheus |
|
||||
| 冷存 | 本地文件系统 |
|
||||
| 构建 | Maven |
|
||||
|
||||
## 模块划分
|
||||
|
||||
@@ -36,32 +36,32 @@ lingniu-vehicle-ingest/
|
||||
│ │ ├── ingest-api/ SPI + sealed 领域事件 + 注解
|
||||
│ │ ├── ingest-codec-common/ 公共编解码工具(BCD/CRC/BCC/bit utils)
|
||||
│ │ ├── ingest-core/ Pipeline / Dispatcher / Disruptor / Session 桥
|
||||
│ │ ├── session-core/ 设备会话 + 鉴权 + Token + Redis/Memory SessionStore
|
||||
│ │ ├── session-core/ 设备会话 + 鉴权 + Token + Redis SessionStore
|
||||
│ │ ├── vehicle-identity/ 跨协议车辆身份解析 + MySQL 外部标识绑定
|
||||
│ │ └── observability/ metrics / tracing / health
|
||||
│ │ └── observability/ metrics / health
|
||||
│ ├── protocols/
|
||||
│ │ ├── protocol-gb32960/ GB/T 32960
|
||||
│ │ ├── protocol-jt808/ JT/T 808(统一身份映射 + 事件 metadata 内部 VIN + 注册/鉴权/心跳/注销/位置/批量位置/参数/属性/媒体/透传/未知上行和坏帧兜底/断链清理会话/下行分包)
|
||||
│ │ ├── protocol-jt1078/ JT/T 1078(808 信令按需桥接 + 常用下行信令编码 + TCP/UDP RTP 媒体流分段归档 + 事件 metadata 内部 VIN + 坏 RTP 统一 RawArchive/Passthrough + 归档失败兜底 + SIM 身份映射)
|
||||
│ │ └── protocol-jsatl12/ 苏标主动安全报警附件(真实 DataPacket 分帧 + 附件流归档 + T1210 文件清单身份继承 + 9212 补传应答 + MediaMeta 引用事件 + 坏帧/坏信令/归档失败兜底)
|
||||
│ │ ├── protocol-jt1078/ JT/T 1078(optional-command-gateway profile;808 信令按需桥接 + 常用下行信令编码 + TCP/UDP RTP 媒体流分段归档 + 事件 metadata 内部 VIN + 坏 RTP 统一 RawArchive/Passthrough + 归档失败兜底 + SIM 身份映射)
|
||||
│ │ └── protocol-jsatl12/ 苏标主动安全报警附件(optional-attachments profile)
|
||||
│ ├── inbound/
|
||||
│ │ ├── inbound-mqtt/ MQTT 接入(endpoint 生命周期 + profile 注册扩展 + 统一身份映射 + PEM 双向 TLS + 未知 profile/解析失败/profile异常/连接订阅失败兜底 + 统一 UNKNOWN 身份 metadata)
|
||||
│ │ └── inbound-xinda-push/ 信达 Push 接入(废弃兼容模块,仅 -Plegacy-xinda 显式构建)
|
||||
│ │ └── inbound-mqtt/ MQTT 接入(endpoint 生命周期 + profile 注册扩展 + 统一身份映射 + PEM 双向 TLS + 未知 profile/解析失败/profile异常/连接订阅失败兜底 + 统一 UNKNOWN 身份 metadata)
|
||||
│ ├── sinks/
|
||||
│ │ ├── sink-mq/ Kafka producer + Protobuf Envelope
|
||||
│ │ ├── sink-archive/ 原始报文冷存
|
||||
│ │ └── event-file-store/ Parquet + DuckDB 文件型明细库
|
||||
│ │ ├── sink-kafka/ Kafka producer + Protobuf Envelope
|
||||
│ │ └── sink-archive/ 原始报文冷存
|
||||
│ ├── services/
|
||||
│ │ ├── event-history-service/ Kafka 全字段事件消费 + 历史查询/导出
|
||||
│ │ ├── vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询
|
||||
│ │ ├── event-history-service/ Kafka 全字段事件消费 + 历史查询
|
||||
│ │ ├── vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询(optional-latest-state profile)
|
||||
│ │ └── vehicle-stat-service/ Kafka 全字段事件消费 + 可配置日统计
|
||||
│ ├── testing/
|
||||
│ │ └── vehicle-identity-test-support/ 协议测试共享身份解析夹具
|
||||
│ └── apps/
|
||||
│ ├── command-gateway/ HTTP → 设备下行命令(808 位置/参数/属性/控制/区域删除/报警确认 + 1078 音视频控制)
|
||||
│ ├── command-gateway/ 可选 HTTP → 设备下行命令(optional-command-gateway profile)
|
||||
│ ├── gb32960-ingest-app/ GB32960 TCP 接入 + Kafka 投递
|
||||
│ ├── jt808-ingest-app/ JT808 TCP 接入 + Kafka 投递
|
||||
│ ├── yutong-mqtt-app/ 宇通 MQTT 接入 + Kafka 投递
|
||||
│ ├── vehicle-history-app/ 历史查询 + 原始归档索引消费
|
||||
│ └── vehicle-analytics-app/ 车辆状态/日统计消费
|
||||
│ ├── vehicle-history-app/ TDengine 历史查询 + RAW JSON
|
||||
│ └── vehicle-analytics-app/ JT808 每日里程指标消费
|
||||
├── docs/ 架构文档、模块图、实施计划
|
||||
└── reference/ 参考资料
|
||||
```
|
||||
@@ -71,12 +71,18 @@ lingniu-vehicle-ingest/
|
||||
```bash
|
||||
# 要求:JDK 25, Maven 3.9+
|
||||
mvn -v
|
||||
mvn clean install -DskipTests
|
||||
mvn -pl :gb32960-ingest-app -am spring-boot:run
|
||||
mvn -pl :jt808-ingest-app -am spring-boot:run
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
信达 Push 已废弃,不参与默认 Maven reactor、CI、Portainer 部署和历史消费链路;如需兼容排查,可显式使用 `-Plegacy-xinda` 构建。
|
||||
生产服务只在 ECS/Portainer 上运行;本机只用于源码检查、Maven 构建和单元测试。部署、健康检查和真实流量验证步骤见 `docs/operations/gb32960-service-split-runbook.md` 与 `docs/operations/vehicle-ingest-tdengine-verification.md`。
|
||||
|
||||
信达 Push 已废弃并删除源码,不参与 Maven reactor、CI、Portainer 部署和历史消费链路。
|
||||
|
||||
JSATL12 附件上传不在默认 Maven reactor 中;需要时使用 `-Poptional-attachments` 构建。
|
||||
|
||||
Command Gateway/JT1078 下行与音视频信令不在默认 Maven reactor 中;需要时使用 `-Poptional-command-gateway` 构建。
|
||||
|
||||
Redis 最新状态查询不在默认 Maven reactor 中;需要时使用 `-Poptional-latest-state` 构建 `vehicle-state-service`。
|
||||
|
||||
## 迁移说明
|
||||
|
||||
@@ -90,8 +96,9 @@ mvn -pl :jt808-ingest-app -am spring-boot:run
|
||||
|
||||
## 核心原则
|
||||
|
||||
1. **本服务不写业务库**:历史、行程、在线检测、里程统计全部由 Kafka 下游消费者实现
|
||||
2. **协议即插拔**:每个 `protocol-*` / `inbound-*` 都可独立开关(`lingniu.ingest.<name>.enabled`)
|
||||
3. **顺序保证**:同一 VIN 严格有序(Disruptor hash + Kafka 分区 key)
|
||||
4. **幂等消费**:Envelope 带 `eventId`,下游去重
|
||||
5. **原始可回放**:Dispatcher 会先发布 `RawArchive`,并把同一份 `rawArchiveKey/rawArchiveUri` 追加到业务事件 metadata;下游明细库、Kafka Envelope 和导出都使用 `archive://...` 逻辑 URI 追溯原始 bytes,实际文件由 `sink-archive`/`ArchiveStore` 管理;`event-file-store` 只额外保存 RawArchive 查询索引,不嵌入原始 bytes
|
||||
1. **接入与业务落库分离**:协议接入应用只负责收、解析、校验、规整和投递 Kafka;`vehicle-history-app` 写入 TDengine 历史与 RAW JSON,`vehicle-analytics-app` 将 JT808 `daily_mileage_km` 写入 MySQL `vehicle_stat_metric`
|
||||
2. **Kafka 唯一消息通道**:生产链路只支持 Kafka;接入、历史、统计统一通过 Kafka Envelope 解耦
|
||||
3. **协议即插拔**:每个 `protocol-*` / `inbound-*` 都可独立开关(`lingniu.ingest.<name>.enabled`)
|
||||
4. **顺序保证**:同一 VIN 严格有序(Disruptor hash + Kafka 分区 key)
|
||||
5. **幂等消费**:Envelope 带 `eventId`,下游去重
|
||||
6. **原始可回放**:协议接入会把 `rawArchiveKey/rawArchiveUri` 追加到事件 metadata;Kafka Envelope、TDengine `raw_frames` 和导出都使用 `archive://...` 逻辑 URI 追溯原始 bytes,实际文件由 `sink-archive` 管理
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
# macOS launchctl 本机部署模板
|
||||
|
||||
本目录用于把本机 32960/JT808/history 三个服务的启动方式仓库化。模板面向当前生产验证链路:
|
||||
|
||||
- `gb32960-ingest-app`: TCP `32960`,HTTP `20100`
|
||||
- `jt808-ingest-app`: TCP `808`,HTTP `20482`
|
||||
- `vehicle-history-app`: HTTP `20200`
|
||||
|
||||
`vehicle-history-app` 的热历史查询只依赖 Kafka 与 TDengine:RAW 帧的 `parsedJson`、`metadataJson`、`rawUri` 已进入 TDengine `raw_frames`。GB32960/JT808 接入服务可以继续保留 `SINK_ARCHIVE_PATH` 作为原始 `.bin` 冷备,history 本机模板不再要求共享 archive root。
|
||||
|
||||
`vehicle-history-app` 的生产高吞吐模板是 TDengine-only:
|
||||
|
||||
- 不暴露 DuckDB/EventFileStore 启动变量,避免文件型明细库进入实时消费热路径。
|
||||
- `TDENGINE_TELEMETRY_FIELDS_ENABLED=false`:默认只写 `raw_frames` 和协议位置表,避免逐字段写放大。
|
||||
- 通用 RAW 查询默认直接返回 TDengine `raw_frames` 中的 `parsedJson`/`parsedFields`;后续解析器新增字段后,优先通过 replay 或专用解析任务补写 TDengine,而不是让 history 服务依赖本地 archive 挂载。
|
||||
- 如果需要兼容旧 `/api/event-history/records` 低层记录接口或逐字段趋势宽表,可以显式打开对应开关;开启后要重新压测 Kafka lag 和 TDengine 写入吞吐。
|
||||
|
||||
## 生成 plist
|
||||
|
||||
在仓库根目录执行:
|
||||
|
||||
```bash
|
||||
export PROJECT_ROOT="$(pwd)"
|
||||
export JAVA_BIN="$(/usr/libexec/java_home 2>/dev/null)/bin/java"
|
||||
export INGEST_ARCHIVE_PATH="$PROJECT_ROOT/data/archive-live"
|
||||
export TDENGINE_JDBC_URL='jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts'
|
||||
export TDENGINE_USERNAME='root'
|
||||
export TDENGINE_PASSWORD='<tdengine-password>'
|
||||
export VEHICLE_IDENTITY_MYSQL_JDBC_URL='jdbc:mysql://<mysql-host>:3306/vehicle_ingest'
|
||||
export VEHICLE_IDENTITY_MYSQL_USERNAME='<mysql-user>'
|
||||
export VEHICLE_IDENTITY_MYSQL_PASSWORD='<mysql-password>'
|
||||
export VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL='60s'
|
||||
|
||||
mkdir -p "$HOME/Library/LaunchAgents" "$PROJECT_ROOT/data"
|
||||
mkdir -p /tmp/lingniu-gb32960-live /tmp/lingniu-jt808-live /tmp/lingniu-history-live
|
||||
|
||||
for service in gb32960 jt808 vehicle-history; do
|
||||
sed \
|
||||
-e "s#__PROJECT_ROOT__#$PROJECT_ROOT#g" \
|
||||
-e "s#__JAVA_BIN__#$JAVA_BIN#g" \
|
||||
-e "s#__SHARED_ARCHIVE_PATH__#$INGEST_ARCHIVE_PATH#g" \
|
||||
-e "s#__TDENGINE_JDBC_URL__#$TDENGINE_JDBC_URL#g" \
|
||||
-e "s#__TDENGINE_USERNAME__#$TDENGINE_USERNAME#g" \
|
||||
-e "s#__TDENGINE_PASSWORD__#$TDENGINE_PASSWORD#g" \
|
||||
-e "s#__VEHICLE_IDENTITY_MYSQL_JDBC_URL__#$VEHICLE_IDENTITY_MYSQL_JDBC_URL#g" \
|
||||
-e "s#__VEHICLE_IDENTITY_MYSQL_USERNAME__#$VEHICLE_IDENTITY_MYSQL_USERNAME#g" \
|
||||
-e "s#__VEHICLE_IDENTITY_MYSQL_PASSWORD__#$VEHICLE_IDENTITY_MYSQL_PASSWORD#g" \
|
||||
-e "s#__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__#$VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL#g" \
|
||||
"deploy/local/launchctl/com.lingniu.${service}.plist.template" \
|
||||
> "$HOME/Library/LaunchAgents/com.lingniu.${service}.plist"
|
||||
done
|
||||
```
|
||||
|
||||
如果生产配置由 Nacos 下发,可以把模板里的 `NACOS_CONFIG_ENABLED` 改成 `true`,并补充 `NACOS_SERVER_ADDR`、`NACOS_NAMESPACE`、`NACOS_GROUP`、`NACOS_USERNAME`、`NACOS_PASSWORD`。端口、Kafka topic、接入服务冷备 archive 路径、TDengine 连接建议仍保留为启动环境变量,便于 Portainer、launchctl 和临时压测保持一致。
|
||||
|
||||
history 消费者会按协议和 raw/event 自动拆分 consumer group。设置 `KAFKA_GROUP_HISTORY=vehicle-history-live` 后,实际 group 为:
|
||||
|
||||
- `vehicle-history-live-gb32960-event`
|
||||
- `vehicle-history-live-gb32960-raw`
|
||||
- `vehicle-history-live-jt808-event`
|
||||
- `vehicle-history-live-jt808-raw`
|
||||
|
||||
本地模板与 ECS 生产口径一致,身份绑定只使用 MySQL。启动前必须提供 `VEHICLE_IDENTITY_MYSQL_*` 连接参数;服务会自动创建 `vehicle_identity_binding` 和 `vehicle_identity_binding_registration` 两张表:前者只保存 `plate`、`vin`,后者保存注册字段;服务按刷新周期用注册表里的车牌关联 VIN,后续 raw/event 会带真实 VIN。
|
||||
|
||||
## 启停
|
||||
|
||||
```bash
|
||||
launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.gb32960.plist"
|
||||
launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.jt808.plist"
|
||||
launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.vehicle-history.plist"
|
||||
|
||||
launchctl list | rg 'com.lingniu.(gb32960|jt808|vehicle-history)'
|
||||
```
|
||||
|
||||
替换 jar 前先停止服务,避免 KeepAlive 在 jar 拷贝中途拉起半包:
|
||||
|
||||
```bash
|
||||
launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.gb32960.plist" || true
|
||||
launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.jt808.plist" || true
|
||||
launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.vehicle-history.plist" || true
|
||||
```
|
||||
|
||||
## 验证地址
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20482/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
```
|
||||
|
||||
Swagger:
|
||||
|
||||
- GB32960 ingest: `http://127.0.0.1:20100/swagger-ui/index.html`
|
||||
- JT808 ingest: `http://127.0.0.1:20482/swagger-ui/index.html`
|
||||
- History query: `http://127.0.0.1:20200/swagger-ui/index.html`
|
||||
@@ -1,65 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.lingniu.gb32960</string>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>__PROJECT_ROOT__</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>__JAVA_BIN__</string>
|
||||
<string>-jar</string>
|
||||
<string>__PROJECT_ROOT__/modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar</string>
|
||||
</array>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>HTTP_PORT</key>
|
||||
<string>20100</string>
|
||||
<key>GB32960_PORT</key>
|
||||
<string>32960</string>
|
||||
<key>GB32960_AUTH_ENABLED</key>
|
||||
<string>false</string>
|
||||
<key>GB32960_PLATFORM_IP_HYUNDAI</key>
|
||||
<string>115.29.187.205</string>
|
||||
<key>KAFKA_ENABLED</key>
|
||||
<string>true</string>
|
||||
<key>KAFKA_BROKERS</key>
|
||||
<string>114.55.58.251:9092</string>
|
||||
<key>KAFKA_NODE_ID</key>
|
||||
<string>gb32960-live-32960</string>
|
||||
<key>KAFKA_TOPIC_GB32960_EVENT</key>
|
||||
<string>vehicle.event.gb32960.v1</string>
|
||||
<key>KAFKA_TOPIC_GB32960_RAW</key>
|
||||
<string>vehicle.raw.gb32960.v1</string>
|
||||
<key>KAFKA_TOPIC_GB32960_DLQ</key>
|
||||
<string>vehicle.dlq.gb32960.v1</string>
|
||||
<key>SINK_ARCHIVE_PATH</key>
|
||||
<string>__SHARED_ARCHIVE_PATH__</string>
|
||||
<key>VEHICLE_IDENTITY_STORE</key>
|
||||
<string>mysql</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_TABLE</key>
|
||||
<string>vehicle_identity_binding</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_JDBC_URL</key>
|
||||
<string>__VEHICLE_IDENTITY_MYSQL_JDBC_URL__</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_USERNAME</key>
|
||||
<string>__VEHICLE_IDENTITY_MYSQL_USERNAME__</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_PASSWORD</key>
|
||||
<string>__VEHICLE_IDENTITY_MYSQL_PASSWORD__</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL</key>
|
||||
<string>__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__</string>
|
||||
<key>NACOS_CONFIG_ENABLED</key>
|
||||
<string>false</string>
|
||||
<key>MANAGEMENT_HEALTH_REDIS_ENABLED</key>
|
||||
<string>false</string>
|
||||
</dict>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/lingniu-gb32960-live/launchctl-gb32960.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/lingniu-gb32960-live/launchctl-gb32960.err.log</string>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -1,61 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.lingniu.jt808</string>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>__PROJECT_ROOT__</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>__JAVA_BIN__</string>
|
||||
<string>-jar</string>
|
||||
<string>__PROJECT_ROOT__/modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar</string>
|
||||
</array>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>HTTP_PORT</key>
|
||||
<string>20482</string>
|
||||
<key>JT808_PORT</key>
|
||||
<string>808</string>
|
||||
<key>KAFKA_ENABLED</key>
|
||||
<string>true</string>
|
||||
<key>KAFKA_BROKERS</key>
|
||||
<string>114.55.58.251:9092</string>
|
||||
<key>KAFKA_NODE_ID</key>
|
||||
<string>jt808-live-808</string>
|
||||
<key>KAFKA_TOPIC_JT808_EVENT</key>
|
||||
<string>vehicle.event.jt808.v1</string>
|
||||
<key>KAFKA_TOPIC_JT808_RAW</key>
|
||||
<string>vehicle.raw.jt808.v1</string>
|
||||
<key>KAFKA_TOPIC_JT808_DLQ</key>
|
||||
<string>vehicle.dlq.jt808.v1</string>
|
||||
<key>SINK_ARCHIVE_PATH</key>
|
||||
<string>__SHARED_ARCHIVE_PATH__</string>
|
||||
<key>VEHICLE_IDENTITY_STORE</key>
|
||||
<string>mysql</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_TABLE</key>
|
||||
<string>vehicle_identity_binding</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_JDBC_URL</key>
|
||||
<string>__VEHICLE_IDENTITY_MYSQL_JDBC_URL__</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_USERNAME</key>
|
||||
<string>__VEHICLE_IDENTITY_MYSQL_USERNAME__</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_PASSWORD</key>
|
||||
<string>__VEHICLE_IDENTITY_MYSQL_PASSWORD__</string>
|
||||
<key>VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL</key>
|
||||
<string>__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__</string>
|
||||
<key>NACOS_CONFIG_ENABLED</key>
|
||||
<string>false</string>
|
||||
<key>MANAGEMENT_HEALTH_REDIS_ENABLED</key>
|
||||
<string>false</string>
|
||||
</dict>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/lingniu-jt808-live/launchctl-jt808.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/lingniu-jt808-live/launchctl-jt808.err.log</string>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -1,75 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.lingniu.vehicle-history</string>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>__PROJECT_ROOT__</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>__JAVA_BIN__</string>
|
||||
<string>-jar</string>
|
||||
<string>__PROJECT_ROOT__/modules/apps/vehicle-history-app/target/vehicle-history-app.jar</string>
|
||||
</array>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>HTTP_PORT</key>
|
||||
<string>20200</string>
|
||||
<key>KAFKA_ENABLED</key>
|
||||
<string>true</string>
|
||||
<key>KAFKA_CONSUMER_ENABLED</key>
|
||||
<string>true</string>
|
||||
<key>KAFKA_CONSUMER_AUTO_OFFSET_RESET</key>
|
||||
<string>latest</string>
|
||||
<key>KAFKA_CONSUMER_CLIENT_ID_PREFIX</key>
|
||||
<string>vehicle-history-live</string>
|
||||
<key>KAFKA_GROUP_HISTORY</key>
|
||||
<string>vehicle-history-live</string>
|
||||
<key>KAFKA_CONSUMER_CONCURRENCY</key>
|
||||
<string>3</string>
|
||||
<key>KAFKA_BROKERS</key>
|
||||
<string>114.55.58.251:9092</string>
|
||||
<key>KAFKA_TOPIC_GB32960_EVENT</key>
|
||||
<string>vehicle.event.gb32960.v1</string>
|
||||
<key>KAFKA_TOPIC_GB32960_RAW</key>
|
||||
<string>vehicle.raw.gb32960.v1</string>
|
||||
<key>KAFKA_TOPIC_JT808_EVENT</key>
|
||||
<string>vehicle.event.jt808.v1</string>
|
||||
<key>KAFKA_TOPIC_JT808_RAW</key>
|
||||
<string>vehicle.raw.jt808.v1</string>
|
||||
<key>TDENGINE_HISTORY_ENABLED</key>
|
||||
<string>true</string>
|
||||
<key>TDENGINE_HISTORY_DATABASE</key>
|
||||
<string>vehicle_ts</string>
|
||||
<key>TDENGINE_JDBC_URL</key>
|
||||
<string>__TDENGINE_JDBC_URL__</string>
|
||||
<key>TDENGINE_DRIVER_CLASS_NAME</key>
|
||||
<string>com.taosdata.jdbc.ws.WebSocketDriver</string>
|
||||
<key>TDENGINE_USERNAME</key>
|
||||
<string>__TDENGINE_USERNAME__</string>
|
||||
<key>TDENGINE_PASSWORD</key>
|
||||
<string>__TDENGINE_PASSWORD__</string>
|
||||
<key>TDENGINE_MIN_IDLE</key>
|
||||
<string>0</string>
|
||||
<key>TDENGINE_MAX_POOL_SIZE</key>
|
||||
<string>16</string>
|
||||
<key>TDENGINE_INITIALIZE_SCHEMA</key>
|
||||
<string>true</string>
|
||||
<key>TDENGINE_TELEMETRY_FIELDS_ENABLED</key>
|
||||
<string>false</string>
|
||||
<key>NACOS_CONFIG_ENABLED</key>
|
||||
<string>false</string>
|
||||
<key>MANAGEMENT_HEALTH_REDIS_ENABLED</key>
|
||||
<string>false</string>
|
||||
</dict>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/lingniu-history-live/vehicle-history.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/lingniu-history-live/vehicle-history.err.log</string>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
||||
117
deploy/portainer/docker-compose-go.yml
Normal file
117
deploy/portainer/docker-compose-go.yml
Normal file
@@ -0,0 +1,117 @@
|
||||
version: "3.8"
|
||||
|
||||
# Go sidecar stack for the vehicle ingest redesign.
|
||||
# It is intentionally separate from deploy/portainer/docker-compose.yml so the
|
||||
# current Java production stack can keep running until Go evidence is captured.
|
||||
|
||||
x-common-env: &common-env
|
||||
TZ: Asia/Shanghai
|
||||
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.yutong-mqtt.v1}
|
||||
KAFKA_TOPIC_UNIFIED: ${KAFKA_TOPIC_UNIFIED:-vehicle.event.unified.v1}
|
||||
|
||||
x-restart: &restart-policy
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "100m"
|
||||
max-file: "5"
|
||||
|
||||
services:
|
||||
go-vehicle-gateway:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-vehicle-gateway
|
||||
command: ["/app/gateway"]
|
||||
mem_limit: ${GO_GATEWAY_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
GB32960_TCP_ADDR: ":32960"
|
||||
JT808_TCP_ADDR: ":808"
|
||||
TCP_READ_BUFFER_BYTES: ${TCP_READ_BUFFER_BYTES:-65536}
|
||||
TCP_IDLE_TIMEOUT_SECONDS: ${TCP_IDLE_TIMEOUT_SECONDS:-180}
|
||||
TCP_MAX_CONNECTIONS: ${TCP_MAX_CONNECTIONS:-20000}
|
||||
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
|
||||
YUTONG_MQTT_ENDPOINT: ${YUTONG_MQTT_ENDPOINT:-yutong}
|
||||
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
|
||||
YUTONG_MQTT_TOPICS: ${YUTONG_MQTT_TOPICS:-/ytforward/shln/+}
|
||||
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-2}
|
||||
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt}
|
||||
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
|
||||
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
|
||||
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
|
||||
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
|
||||
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
|
||||
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
|
||||
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
|
||||
IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-}
|
||||
VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding}
|
||||
ports:
|
||||
- "${GO_GB32960_TCP_PORT:-32960}:32960"
|
||||
- "${GO_JT808_TCP_PORT:-808}:808"
|
||||
volumes:
|
||||
- "${KAFKA_SPOOL_HOST_DIR:-/opt/lingniu-go/spool/gateway}:${KAFKA_SPOOL_DIR:-/data/spool/gateway}"
|
||||
- "${YUTONG_MQTT_CERT_HOST_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:${YUTONG_MQTT_CERT_CONTAINER_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:ro"
|
||||
|
||||
go-history-writer:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-history-writer
|
||||
command: ["/app/history-writer"]
|
||||
mem_limit: ${GO_HISTORY_WRITER_MEM_LIMIT:-768m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_HISTORY:-go-history-writer}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_HISTORY:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1}
|
||||
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
|
||||
TDENGINE_DSN: ${TDENGINE_DSN:?set TDENGINE_DSN, example root:password@ws(172.17.111.57:6041)/lingniu_vehicle_ts}
|
||||
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
|
||||
TDENGINE_ENSURE_SCHEMA: ${TDENGINE_ENSURE_SCHEMA:-true}
|
||||
|
||||
go-stat-writer:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-stat-writer
|
||||
command: ["/app/stat-writer"]
|
||||
mem_limit: ${GO_STAT_WRITER_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_STAT:-go-stat-writer}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_STAT:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1}
|
||||
MYSQL_DSN: ${MYSQL_DSN:?set MYSQL_DSN}
|
||||
MYSQL_ENSURE_SCHEMA: ${MYSQL_ENSURE_SCHEMA:-true}
|
||||
LOCAL_TZ: ${LOCAL_TZ:-Asia/Shanghai}
|
||||
|
||||
go-realtime-api:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-realtime-api
|
||||
command: ["/app/realtime-api"]
|
||||
mem_limit: ${GO_REALTIME_API_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
HTTP_ADDR: ":20210"
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_REALTIME:-go-realtime-api}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_REALTIME:-vehicle.event.unified.v1}
|
||||
REDIS_ADDR: ${REDIS_ADDR:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379}
|
||||
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
|
||||
REDIS_DB: ${REDIS_DB:-50}
|
||||
ONLINE_TTL_SECONDS: ${ONLINE_TTL_SECONDS:-600}
|
||||
MYSQL_DSN: ${MYSQL_DSN:-}
|
||||
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
|
||||
TDENGINE_DSN: ${TDENGINE_DSN:-}
|
||||
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
|
||||
ports:
|
||||
- "${GO_REALTIME_HTTP_PORT:-20210}:20210"
|
||||
|
||||
networks:
|
||||
vehicle-ingest:
|
||||
name: vehicle-ingest
|
||||
@@ -1,10 +1,13 @@
|
||||
version: "3.8"
|
||||
|
||||
# Default production stack: GB32960, JT808, Yutong MQTT, history, analytics.
|
||||
# Legacy and optional services stay outside this compose file.
|
||||
# Set LINGNIU_IMAGE_VERSION in Portainer Stack env, for example:
|
||||
# main-0.1.0-SNAPSHOT-22468e7
|
||||
# Override LINGNIU_IMAGE_REGISTRY / LINGNIU_IMAGE_NAMESPACE only when publishing to another registry.
|
||||
x-common-env: &common-env
|
||||
TZ: Asia/Shanghai
|
||||
JAVA_OPTS: ${JAVA_OPTS:--XX:+UseZGC -XX:MaxRAMPercentage=75}
|
||||
JAVA_OPTS: ${JAVA_OPTS:---sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true -XX:+UseZGC -XX:MaxRAMPercentage=75}
|
||||
NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true}
|
||||
NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848}
|
||||
NACOS_NAMESPACE: ${NACOS_NAMESPACE:-}
|
||||
@@ -14,48 +17,51 @@ x-common-env: &common-env
|
||||
NACOS_USERNAME: ${NACOS_USERNAME:-}
|
||||
NACOS_PASSWORD: ${NACOS_PASSWORD:-}
|
||||
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
|
||||
KAFKA_ENABLED: ${KAFKA_ENABLED:-true}
|
||||
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true}
|
||||
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:-900000}
|
||||
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
|
||||
MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
|
||||
x-identity-mysql-env: &identity-mysql-env
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
|
||||
x-redis-session-env: &redis-session-env
|
||||
REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DATABASE: ${REDIS_DATABASE:-50}
|
||||
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
|
||||
|
||||
x-readiness-healthcheck: &readiness-healthcheck
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
services:
|
||||
gb32960-ingest-app:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: gb32960-ingest-app
|
||||
restart: unless-stopped
|
||||
mem_limit: ${GB32960_MEM_LIMIT:-768m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
<<:
|
||||
- *common-env
|
||||
- *identity-mysql-env
|
||||
- *redis-session-env
|
||||
HTTP_PORT: 20100
|
||||
GB32960_PORT: 32960
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID:-gb32960-ingest-portainer}
|
||||
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
|
||||
GB32960_AUTH_ENABLED: ${GB32960_AUTH_ENABLED:-false}
|
||||
GB32960_PLATFORM_USER_HYUNDAI: ${GB32960_PLATFORM_USER_HYUNDAI:-Hyundai}
|
||||
GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}
|
||||
GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}
|
||||
GB32960_PLATFORM_USER_YUEJIN: ${GB32960_PLATFORM_USER_YUEJIN:-YueJin}
|
||||
GB32960_PLATFORM_PWD_YUEJIN: ${GB32960_PLATFORM_PWD_YUEJIN:-}
|
||||
GB32960_PLATFORM_IP_YUEJIN: ${GB32960_PLATFORM_IP_YUEJIN:-}
|
||||
GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false}
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481pd.redis.rds.aliyuncs.com}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DATABASE: ${REDIS_DATABASE:-50}
|
||||
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
|
||||
SINK_ARCHIVE_PATH: ${GB32960_ARCHIVE_PATH:-/data/archive}
|
||||
ports:
|
||||
- "${GB32960_HTTP_PORT:-20100}:20100"
|
||||
- "${GB32960_TCP_PORT:-32960}:32960"
|
||||
@@ -63,27 +69,27 @@ services:
|
||||
- gb32960-ingest-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20100/actuator/health/readiness >/dev/null"]
|
||||
|
||||
jt808-ingest-app:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: jt808-ingest-app
|
||||
restart: unless-stopped
|
||||
mem_limit: ${JT808_MEM_LIMIT:-768m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
<<:
|
||||
- *common-env
|
||||
- *identity-mysql-env
|
||||
- *redis-session-env
|
||||
HTTP_PORT: 20400
|
||||
JT808_PORT: 808
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer}
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481pd.redis.rds.aliyuncs.com}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DATABASE: ${REDIS_DATABASE:-50}
|
||||
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
|
||||
SINK_ARCHIVE_PATH: ${JT808_ARCHIVE_PATH:-/data/archive}
|
||||
ports:
|
||||
- "${JT808_HTTP_PORT:-20400}:20400"
|
||||
- "${JT808_TCP_PORT:-808}:808"
|
||||
@@ -91,23 +97,26 @@ services:
|
||||
- jt808-ingest-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20400/actuator/health/readiness >/dev/null"]
|
||||
|
||||
yutong-mqtt-app:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: yutong-mqtt-app
|
||||
restart: unless-stopped
|
||||
mem_limit: ${YUTONG_MQTT_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
<<:
|
||||
- *common-env
|
||||
- *identity-mysql-env
|
||||
HTTP_PORT: 20500
|
||||
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
|
||||
YUTONG_MQTT_AUTO_STARTUP: ${YUTONG_MQTT_AUTO_STARTUP:-true}
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_MQTT:-yutong-mqtt-portainer}
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
|
||||
YUTONG_MQTT_ENDPOINT_NAME: ${YUTONG_MQTT_ENDPOINT_NAME:-yutong}
|
||||
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
|
||||
YUTONG_MQTT_TOPIC: "${YUTONG_MQTT_TOPIC:-#}"
|
||||
@@ -116,32 +125,43 @@ services:
|
||||
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
|
||||
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
|
||||
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
|
||||
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
|
||||
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
|
||||
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
|
||||
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
|
||||
SINK_ARCHIVE_PATH: ${YUTONG_MQTT_ARCHIVE_PATH:-/data/archive}
|
||||
ports:
|
||||
- "${YUTONG_MQTT_HTTP_PORT:-20500}:20500"
|
||||
volumes:
|
||||
- yutong-mqtt-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20500/actuator/health/readiness >/dev/null"]
|
||||
|
||||
vehicle-history-app:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: vehicle-history-app
|
||||
restart: unless-stopped
|
||||
mem_limit: ${VEHICLE_HISTORY_MEM_LIMIT:-1536m}
|
||||
depends_on:
|
||||
- gb32960-ingest-app
|
||||
- jt808-ingest-app
|
||||
- yutong-mqtt-app
|
||||
environment:
|
||||
<<: *common-env
|
||||
HTTP_PORT: 20200
|
||||
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_HISTORY:-true}
|
||||
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history}
|
||||
KAFKA_CONSUMER_MAX_POLL_RECORDS: ${KAFKA_CONSUMER_MAX_POLL_RECORDS_HISTORY:-2000}
|
||||
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_HISTORY:-1800000}
|
||||
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_HISTORY_DLQ: ${KAFKA_TOPIC_HISTORY_DLQ:-vehicle.dlq.history.v1}
|
||||
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
|
||||
KAFKA_GROUP_HISTORY_GB32960_EVENT: ${KAFKA_GROUP_HISTORY_GB32960_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-gb32960-event}
|
||||
KAFKA_GROUP_HISTORY_JT808_EVENT: ${KAFKA_GROUP_HISTORY_JT808_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-jt808-event}
|
||||
KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-yutong-mqtt-event}
|
||||
KAFKA_GROUP_HISTORY_GB32960_RAW: ${KAFKA_GROUP_HISTORY_GB32960_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-gb32960-raw}
|
||||
KAFKA_GROUP_HISTORY_JT808_RAW: ${KAFKA_GROUP_HISTORY_JT808_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-jt808-raw}
|
||||
KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-yutong-mqtt-raw}
|
||||
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}
|
||||
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts}
|
||||
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}
|
||||
@@ -154,19 +174,23 @@ services:
|
||||
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20200/actuator/health/readiness >/dev/null"]
|
||||
|
||||
vehicle-analytics-app:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: vehicle-analytics-app
|
||||
restart: unless-stopped
|
||||
mem_limit: ${VEHICLE_ANALYTICS_MEM_LIMIT:-1024m}
|
||||
depends_on:
|
||||
- gb32960-ingest-app
|
||||
- jt808-ingest-app
|
||||
environment:
|
||||
<<: *common-env
|
||||
HTTP_PORT: 20300
|
||||
HTTP_PORT: 20310
|
||||
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_ANALYTICS:-true}
|
||||
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics}
|
||||
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_ANALYTICS:-900000}
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
|
||||
KAFKA_GROUP_STAT: ${KAFKA_GROUP_STAT:-vehicle-stat}
|
||||
VEHICLE_STAT_ENABLED: ${VEHICLE_STAT_ENABLED:-true}
|
||||
VEHICLE_STAT_ZONE_ID: ${VEHICLE_STAT_ZONE_ID:-Asia/Shanghai}
|
||||
@@ -175,9 +199,12 @@ services:
|
||||
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
ports:
|
||||
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300"
|
||||
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20310}:20310"
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20310/actuator/health/readiness >/dev/null"]
|
||||
|
||||
networks:
|
||||
vehicle-ingest:
|
||||
@@ -186,3 +213,4 @@ networks:
|
||||
volumes:
|
||||
gb32960-ingest-data:
|
||||
jt808-ingest-data:
|
||||
yutong-mqtt-data:
|
||||
|
||||
66
deploy/systemd/README.md
Normal file
66
deploy/systemd/README.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Go Vehicle Gateway Native Deployment
|
||||
|
||||
本目录用于 ECS 原生部署。当前 goal 后续不再使用 Docker/Portainer 作为 Go 接入链路的部署方式。
|
||||
|
||||
## Runtime Layout
|
||||
|
||||
```text
|
||||
/opt/lingniu-go-native/
|
||||
current -> /opt/lingniu-go-native/releases/<git-short-sha>
|
||||
releases/<git-short-sha>/
|
||||
gateway
|
||||
history-writer
|
||||
stat-writer
|
||||
realtime-api
|
||||
env/
|
||||
gateway.env
|
||||
history-writer.env
|
||||
stat-writer.env
|
||||
realtime-api.env
|
||||
spool/gateway/
|
||||
```
|
||||
|
||||
## Services
|
||||
|
||||
| systemd unit | Binary | Purpose |
|
||||
|---|---|---|
|
||||
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP `32960`、JT808 TCP `808`、宇通 MQTT 接入,写 Kafka RAW/unified |
|
||||
| `lingniu-go-history-writer.service` | `history-writer` | 消费 RAW topic,写 TDengine `raw_frames` 和核心时序表 |
|
||||
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 RAW topic,写 MySQL `vehicle_daily_metric` |
|
||||
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费 unified topic,写 Redis,并提供 realtime/raw/stat 查询 API |
|
||||
|
||||
## Build
|
||||
|
||||
在开发机或 CI 上构建 Linux amd64 二进制:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/gateway ./cmd/gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/history-writer ./cmd/history-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/stat-writer ./cmd/stat-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/realtime-api ./cmd/realtime-api
|
||||
```
|
||||
|
||||
## Cutover Notes
|
||||
|
||||
1. 先上传二进制到新 release 目录并更新 `current` symlink。
|
||||
2. 从现有生产 env 生成四个 systemd 专用 env 文件;不要把密钥写入仓库。
|
||||
3. 停止旧 Docker Go 容器或 Java 容器,释放 `808`、`32960`、`20210`。
|
||||
4. 执行:
|
||||
|
||||
```bash
|
||||
systemctl daemon-reload
|
||||
systemctl enable lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=JT808&limit=1'
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=GB32960&limit=1'
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=YUTONG_MQTT&limit=1'
|
||||
```
|
||||
18
deploy/systemd/lingniu-go-gateway.service
Normal file
18
deploy/systemd/lingniu-go-gateway.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Vehicle Gateway
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/gateway.env
|
||||
ExecStart=/opt/lingniu-go-native/current/gateway
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-history-writer.service
Normal file
18
deploy/systemd/lingniu-go-history-writer.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go History Writer
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/history-writer.env
|
||||
ExecStart=/opt/lingniu-go-native/current/history-writer
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-realtime-api.service
Normal file
18
deploy/systemd/lingniu-go-realtime-api.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Realtime API
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/realtime-api.env
|
||||
ExecStart=/opt/lingniu-go-native/current/realtime-api
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-stat-writer.service
Normal file
18
deploy/systemd/lingniu-go-stat-writer.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Stat Writer
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/stat-writer.env
|
||||
ExecStart=/opt/lingniu-go-native/current/stat-writer
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -257,14 +257,15 @@
|
||||
<section>
|
||||
<h2>一、系统定位</h2>
|
||||
<p>
|
||||
当前项目不是业务库写入服务。它的核心职责是把 GB/T 32960、JT/T 808、JT/T 1078、JSATL12、
|
||||
MQTT 等不同来源的车辆数据统一转换为 <code>VehicleEvent</code> 和全字段 <code>TelemetrySnapshot</code>,
|
||||
当前项目不是业务库写入服务。它的核心职责是把 GB/T 32960、JT/T 808、Yutong MQTT
|
||||
等默认生产来源的车辆数据统一转换为 <code>VehicleEvent</code> 和全字段 <code>TelemetrySnapshot</code>;
|
||||
JT/T 1078、JSATL12 作为显式 profile 的可选能力保留,
|
||||
再通过 Kafka、原始报文归档、TDengine 历史索引、Redis 热状态和独立统计模块交给下游业务。
|
||||
</p>
|
||||
<div class="summary-grid">
|
||||
<div class="summary-card">
|
||||
<strong>接入方式</strong>
|
||||
<span>Netty TCP、MQTT、后续可扩展更多 Inbound Adapter;Xinda Push 不在默认生产面。</span>
|
||||
<span>Netty TCP、MQTT、后续可扩展更多 Inbound Adapter;信达 Push 源码已删除。</span>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<strong>统一模型</strong>
|
||||
@@ -308,7 +309,7 @@ flowchart TB
|
||||
codec["ingest-codec-common<br/>BCD / CRC / BCC / bit 工具"]
|
||||
session["session-core<br/>设备会话、命令分发接口、会话存储"]
|
||||
identity["vehicle-identity<br/>跨协议身份解析、MySQL 绑定表"]
|
||||
obs["observability<br/>metrics / tracing / health"]
|
||||
obs["observability<br/>metrics / health"]
|
||||
api["ingest-api<br/>ProtocolId / RawFrame / VehicleEvent / 注解 SPI"]
|
||||
registry["HandlerRegistry<br/>按协议、命令、infoType 路由"]
|
||||
dispatcher["Dispatcher<br/>RawFrame 到 Handler 到 EventBus"]
|
||||
@@ -316,28 +317,34 @@ flowchart TB
|
||||
end
|
||||
|
||||
subgraph sink["输出与明细存储 modules/sinks"]
|
||||
mq["sink-mq<br/>VehicleEvent 到 Protobuf Envelope 到 Kafka"]
|
||||
archive["sink-archive<br/>RawArchive 到本地/S3/OSS 冷存"]
|
||||
kafka["sink-kafka<br/>VehicleEvent 到 Protobuf Envelope 到 Kafka"]
|
||||
archive["sink-archive<br/>RawArchive 到本地文件系统冷存"]
|
||||
tdengineStore["tdengine-history-store<br/>TDengine raw_frames + locations"]
|
||||
fileStore["event-file-store<br/>Parquet/DuckDB 兼容可选"]
|
||||
end
|
||||
|
||||
subgraph services["消费服务 modules/services"]
|
||||
history["event-history-service<br/>Kafka event/raw 到 TDengine 历史查询"]
|
||||
state["vehicle-state-service<br/>Kafka 全字段事件到 Redis 热状态"]
|
||||
state["vehicle-state-service<br/>可选 Redis 热状态<br/>optional-latest-state"]
|
||||
stat["vehicle-stat-service<br/>Kafka 全字段事件到指标表"]
|
||||
end
|
||||
|
||||
subgraph apps["应用入口 modules/apps"]
|
||||
gateway["command-gateway<br/>HTTP 到设备命令入口"]
|
||||
bootall["bootstrap-all<br/>聚合所有 AutoConfiguration 一体启动"]
|
||||
gbApp["gb32960-ingest-app<br/>GB32960 TCP 接入"]
|
||||
jtApp["jt808-ingest-app<br/>JT808 TCP 接入"]
|
||||
yutongApp["yutong-mqtt-app<br/>宇通 MQTT 接入"]
|
||||
historyApp["vehicle-history-app<br/>历史查询与 TDengine 写入"]
|
||||
analyticsApp["vehicle-analytics-app<br/>808 日里程指标消费"]
|
||||
gateway["command-gateway<br/>可选 HTTP 设备命令入口"]
|
||||
end
|
||||
|
||||
vehicle --> gb
|
||||
vehicle --> jt808
|
||||
vehicle --> gbApp
|
||||
vehicle --> jtApp
|
||||
vehicle --> jt1078
|
||||
vehicle --> jsatl12
|
||||
mqttSource --> mqtt
|
||||
mqttSource --> yutongApp
|
||||
gbApp --> gb
|
||||
jtApp --> jt808
|
||||
yutongApp --> mqtt
|
||||
mqtt --> mqttProfile
|
||||
mqttProfile --> api
|
||||
commandClient --> gateway
|
||||
@@ -355,13 +362,14 @@ flowchart TB
|
||||
api --> registry
|
||||
registry --> dispatcher
|
||||
dispatcher --> bus
|
||||
bus --> mq
|
||||
bus --> kafka
|
||||
bus --> archive
|
||||
mq --> history
|
||||
mq --> state
|
||||
mq --> stat
|
||||
kafka --> historyApp
|
||||
kafka --> analyticsApp
|
||||
historyApp --> history
|
||||
analyticsApp --> stat
|
||||
kafka -.可选热状态.-> state
|
||||
history --> tdengineStore
|
||||
history -.兼容可选.-> fileStore
|
||||
|
||||
gateway --> session
|
||||
session --> jt808
|
||||
@@ -371,15 +379,7 @@ flowchart TB
|
||||
|
||||
obs -.监控.-> dispatcher
|
||||
obs -.监控.-> bus
|
||||
obs -.监控.-> mq
|
||||
bootall -.装配.-> gb
|
||||
bootall -.装配.-> jt808
|
||||
bootall -.装配.-> mqtt
|
||||
bootall -.装配.-> mq
|
||||
bootall -.装配.-> archive
|
||||
bootall -.装配.-> history
|
||||
bootall -.装配.-> state
|
||||
bootall -.装配.-> stat
|
||||
obs -.监控.-> kafka
|
||||
|
||||
classDef entry fill:#eff6ff,stroke:#2563eb,color:#1e3a8a;
|
||||
classDef core fill:#ecfdf3,stroke:#0f766e,color:#134e48;
|
||||
@@ -387,11 +387,11 @@ flowchart TB
|
||||
classDef command fill:#f5f3ff,stroke:#6d28d9,color:#4c1d95;
|
||||
classDef support fill:#f8fafc,stroke:#64748b,color:#334155;
|
||||
|
||||
class gb,jt808,jt1078,jsatl12,mqtt,mqttProfile entry;
|
||||
class gb,jt808,jt1078,jsatl12,mqtt,mqttProfile,gbApp,jtApp,yutongApp,historyApp,analyticsApp entry;
|
||||
class api,registry,dispatcher,bus,identity core;
|
||||
class mq,archive,tdengineStore,fileStore sink;
|
||||
class kafka,archive,tdengineStore sink;
|
||||
class gateway,session command;
|
||||
class codec,obs,bootall support;
|
||||
class codec,obs support;
|
||||
</pre>
|
||||
</div>
|
||||
<div class="legend">
|
||||
@@ -414,7 +414,7 @@ sequenceDiagram
|
||||
participant Dispatcher as Dispatcher
|
||||
participant Handler as Protocol Handler<br/>Mapper
|
||||
participant EventBus as DisruptorEventBus
|
||||
participant MQ as sink-mq<br/>Kafka Envelope
|
||||
participant Kafka as sink-kafka<br/>Kafka Envelope
|
||||
participant Archive as sink-archive<br/>ArchiveStore
|
||||
participant History as vehicle-history-app
|
||||
participant TDengine as tdengine-history-store<br/>raw_frames + locations
|
||||
@@ -430,12 +430,12 @@ sequenceDiagram
|
||||
Handler->>Handler: 协议字段映射为内部字段
|
||||
Dispatcher->>Handler: 绑定 rawArchiveKey / rawArchiveUri
|
||||
Handler->>EventBus: VehicleEvent(Realtime / Location / Alarm / Login ...)
|
||||
EventBus->>MQ: 投递规整后的业务事件
|
||||
MQ->>MQ: EnvelopeMapper 转 Protobuf
|
||||
MQ->>History: Kafka event/raw topic,key=vin,单车有序
|
||||
EventBus->>Kafka: 投递规整后的业务事件
|
||||
Kafka->>Kafka: EnvelopeMapper 转 Protobuf
|
||||
Kafka->>History: event/raw topic,key=vin,单车有序
|
||||
History->>TDengine: 写 raw_frames、locations 和 raw parsed JSON
|
||||
TDengine->>Consumer: 分页历史查询、原始帧追溯、位置查询
|
||||
MQ->>Consumer: Redis 热状态、指标统计、告警工单等下游消费
|
||||
Kafka->>Consumer: Redis 热状态、指标统计、告警工单等下游消费
|
||||
</pre>
|
||||
</div>
|
||||
<div class="callout">
|
||||
@@ -444,7 +444,7 @@ sequenceDiagram
|
||||
</div>
|
||||
<div class="callout">
|
||||
<strong>原始追溯:</strong>
|
||||
Dispatcher 为所有带 rawBytes 的 RawFrame 生成统一的 <code>rawArchiveKey</code>,业务事件 metadata 使用 <code>rawArchiveUri=archive://...</code> 指向同一份原始 bytes。归档文件的真实本地/S3/OSS 地址只属于 ArchiveStore,查询、导出和 Kafka Envelope 使用逻辑 URI 解耦存储实现。
|
||||
Dispatcher 为所有带 rawBytes 的 RawFrame 生成统一的 <code>rawArchiveKey</code>,业务事件 metadata 使用 <code>rawArchiveUri=archive://...</code> 指向同一份原始 bytes。归档文件的真实本地路径只属于 ArchiveStore,查询、导出和 Kafka Envelope 使用逻辑 URI 解耦存储实现。
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -521,7 +521,7 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/session-core</td>
|
||||
<td>设备会话、命令下发抽象、SessionStore、CommandDispatcher 默认实现。SessionStore 支持 <code>memory</code> 和 <code>redis</code> 两种后端;生产默认使用 Redis 保存 <code>sessionId</code>、VIN 和 phone 三索引及 TTL,内存实现作为本地开发/无 Redis 降级,协议模块只依赖 SPI,不感知存储后端。</td>
|
||||
<td>设备会话、命令下发抽象、SessionStore、CommandDispatcher 默认实现。SessionStore 仅保留 Redis 后端;协议进程本地只保存真实 Netty Channel,Redis 保存 <code>sessionId</code>、VIN 和 phone 三索引及 TTL,协议模块只依赖 SPI,不感知存储细节。</td>
|
||||
<td>设备连接、命令请求。</td>
|
||||
<td>会话状态、命令分发结果、Redis 会话索引。</td>
|
||||
<td><span class="tag">在线状态</span><span class="tag">下行控制</span><span class="tag green">多实例会话索引</span></td>
|
||||
@@ -535,7 +535,7 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/observability</td>
|
||||
<td>监控、指标、追踪、健康检查等横切能力。</td>
|
||||
<td>监控、指标、健康检查等横切能力。</td>
|
||||
<td>核心管线和 Sink 运行状态。</td>
|
||||
<td>Micrometer 指标、健康信息。</td>
|
||||
<td><span class="tag green">可用性</span><span class="tag">延迟监控</span></td>
|
||||
@@ -563,7 +563,7 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/protocols/protocol-jsatl12</td>
|
||||
<td>苏标主动安全报警附件接入。依赖 ArchiveStore 和 JT808 decoder;生产默认端口按旧接收服务对齐为 <code>7612</code>;附件 DataPacket 按旧服务真实布局 <code>01cd + 50B文件名 + offset + length + data</code> 分帧解析,不再把文件名前 4 字节误判为帧长;附件数据写 ArchiveStore 后通过 Dispatcher 产出 MediaMeta 引用事件,事件 metadata 保留 fileName、fileOffset、declaredChunkSizeBytes、archiveKey 和 archiveRef,便于附件明细查询、导出和补传诊断;内置附件状态机处理 T1210 文件清单、DataPacket 分块区间和 T1212 上传完成消息,数据分块会按 T1210 文件清单中的文件名继承手机号并通过共享身份解析器得到内部 VIN,入口 RawFrame、ArchivedChunk 和 MediaMeta 均保留 phone、vin、identityResolved、identitySource,只有找不到文件归属时才标记 UNKNOWN;按文件名合并无身份数据分块并返回 0x9212 完成/补传应答,补传应答包含缺失 offset/length 区间;附件归档失败会产出 JSATL12 Passthrough 错误事件并保留原始分块;JT_MESSAGE 信令复用 JT808 decoder 后进入 Dispatcher,桥接 RawFrame 使用共享身份解析后的内部 VIN并保留 phone、identityResolved、identitySource,坏 JT 信令入口 RawFrame 同样标记 UNKNOWN 身份;超长/畸形附件帧、未闭合超长 JT 信令和坏 JT 信令都以 JSATL12 Passthrough 兜底,附件帧异常同时标记 frameError 与统一 parseError/parseErrorMessage,保留原始字节、peer、长度和错误元数据,帧长和 worker 线程支持部署配置。</td>
|
||||
<td>苏标主动安全报警附件接入(仅 <code>optional-attachments</code> profile 显式构建,不属于默认生产面)。依赖 ArchiveStore 和 JT808 decoder;端口按旧接收服务对齐为 <code>7612</code>;附件 DataPacket 按旧服务真实布局 <code>01cd + 50B文件名 + offset + length + data</code> 分帧解析,不再把文件名前 4 字节误判为帧长;附件数据写 ArchiveStore 后通过 Dispatcher 产出 MediaMeta 引用事件,事件 metadata 保留 fileName、fileOffset、declaredChunkSizeBytes、archiveKey 和 archiveRef,便于附件明细查询、导出和补传诊断;内置附件状态机处理 T1210 文件清单、DataPacket 分块区间和 T1212 上传完成消息,数据分块会按 T1210 文件清单中的文件名继承手机号并通过共享身份解析器得到内部 VIN,入口 RawFrame、ArchivedChunk 和 MediaMeta 均保留 phone、vin、identityResolved、identitySource,只有找不到文件归属时才标记 UNKNOWN;按文件名合并无身份数据分块并返回 0x9212 完成/补传应答,补传应答包含缺失 offset/length 区间;附件归档失败会产出 JSATL12 Passthrough 错误事件并保留原始分块;JT_MESSAGE 信令复用 JT808 decoder 后进入 Dispatcher,桥接 RawFrame 使用共享身份解析后的内部 VIN并保留 phone、identityResolved、identitySource,坏 JT 信令入口 RawFrame 同样标记 UNKNOWN 身份;超长/畸形附件帧、未闭合超长 JT 信令和坏 JT 信令都以 JSATL12 Passthrough 兜底,附件帧异常同时标记 frameError 与统一 parseError/parseErrorMessage,保留原始字节、peer、长度和错误元数据,帧长和 worker 线程支持部署配置。</td>
|
||||
<td>主动安全附件流、JT808 信令帧。</td>
|
||||
<td>归档文件、MediaMeta、JT808 统一事件、坏帧/坏信令/归档失败 Passthrough。</td>
|
||||
<td><span class="tag amber">附件归档</span><span class="tag">报警证据</span><span class="tag">信令复用</span><span class="tag">错误隔离</span></td>
|
||||
@@ -576,7 +576,7 @@ flowchart LR
|
||||
<td><span class="tag">车企平台接入</span><span class="tag green">profile 扩展</span><span class="tag green">身份映射</span><span class="tag green">双向 TLS</span><span class="tag green">错误隔离</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/sink-mq</td>
|
||||
<td>modules/sinks/sink-kafka</td>
|
||||
<td>Kafka Sink。把 VehicleEvent 转成 Protobuf Envelope,按 VIN 分区投递;业务事件携带全字段 TelemetrySnapshot 和 rawArchiveUri,显式 RawArchive Envelope 会填充 archive:// 逻辑 URI 与 size,默认 Kafka Sink 仍不发送原始 bytes 本体;同时提供 KafkaEnvelopeConsumerFactory、KafkaEnvelopeConsumerRunner、KafkaEnvelopeConsumerWorker 和 KafkaEnvelopeDeadLetterSink,统一服务侧 Kafka 消费启动、独立 group 绑定、死信发布和 topic 到 EnvelopeConsumerProcessor 的分发。</td>
|
||||
<td>VehicleEvent。</td>
|
||||
<td>Kafka Protobuf 消息、消费侧 DLQ 记录。</td>
|
||||
@@ -584,23 +584,16 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/sink-archive</td>
|
||||
<td>原始报文冷存,当前有本地文件系统实现,后续可替换 S3/OSS;写入键与业务事件 metadata 中的 <code>rawArchiveKey</code> 保持一致。</td>
|
||||
<td>原始报文冷存,当前为本地文件系统实现;写入键与业务事件 metadata 中的 <code>rawArchiveKey</code> 保持一致。</td>
|
||||
<td>RawArchive 事件、附件流。</td>
|
||||
<td>可回放原始文件、<code>archive://...</code> 逻辑引用。</td>
|
||||
<td><span class="tag amber">问题排查</span><span class="tag amber">合规留痕</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/event-file-store</td>
|
||||
<td>兼容文件型明细库,使用 Parquet 存储解析后的事件明细和 RawArchive 可查询索引,使用 DuckDB 按日期范围正序/倒序查询;默认生产部署不启用,仅用于旧低层记录接口或本地调查场景。</td>
|
||||
<td>VehicleEvent 或 EventFileRecord。</td>
|
||||
<td>protocol/date 分区的 Parquet 文件和兼容查询结果。</td>
|
||||
<td><span class="tag">兼容查询</span><span class="tag amber">默认关闭</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/tdengine-history-store</td>
|
||||
<td>生产历史库边界,负责 TDengine schema、raw_frames/location/telemetry field 行映射、批量写入和分页查询语句;raw_frames 保留完整 payloadJson.parsed,位置表保持轻量字段并通过 rawUri 关联原始帧。</td>
|
||||
<td>生产历史库边界,负责 TDengine schema、raw_frames/location 行映射、批量写入和分页查询语句;raw_frames 保留完整 payloadJson.parsed,位置表保持轻量字段并通过 rawUri 关联原始帧。</td>
|
||||
<td>Kafka Protobuf Envelope、Raw Envelope。</td>
|
||||
<td>TDengine <code>raw_frames</code>、位置表、可选 telemetry field 表。</td>
|
||||
<td>TDengine <code>raw_frames</code> 和位置表。</td>
|
||||
<td><span class="tag green">生产历史</span><span class="tag green">分页查询</span><span class="tag amber">raw 追溯</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -612,7 +605,7 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/services/vehicle-state-service</td>
|
||||
<td>消费 Kafka 全字段事件,更新车辆最新状态、位置、安全和最后事件到 Redis;生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf 和缺快照 Envelope,并把失败记录交给死信出口。</td>
|
||||
<td>消费 Kafka 全字段事件,更新车辆最新状态、位置、安全和最后事件到 Redis;仅通过 <code>optional-latest-state</code> profile 显式构建,不属于默认生产 reactor;生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf 和缺快照 Envelope,并把失败记录交给死信出口。</td>
|
||||
<td>Kafka Protobuf Envelope。</td>
|
||||
<td><code>vehicle:state:{vin}</code>、<code>vehicle:location:{vin}</code>、<code>vehicle:safety:{vin}</code>。</td>
|
||||
<td><span class="tag green">毫秒级热查询</span><span class="tag red">氢泄露安全</span></td>
|
||||
@@ -626,17 +619,45 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/command-gateway</td>
|
||||
<td>HTTP 到设备下行命令入口。支持会话查询、808 位置查询、参数查询/设置、终端控制、平台通用应答、终端属性查询 0x8107、区域删除 0x8601、人工报警确认 0x8203,以及 1078 音视频属性查询、实时预览/控制、历史回放/控制、资源列表查询、文件上传/控制;通过 CommandDispatcher 复用 protocol-jt808 的在线通道、流水号和同步应答等待能力。</td>
|
||||
<td>可选 HTTP 到设备下行命令入口。支持会话查询、808 位置查询、参数查询/设置、终端控制、平台通用应答、终端属性查询 0x8107、区域删除 0x8601、人工报警确认 0x8203,以及 1078 音视频属性查询、实时预览/控制、历史回放/控制、资源列表查询、文件上传/控制;通过 CommandDispatcher 复用 protocol-jt808 的在线通道、流水号和同步应答等待能力。</td>
|
||||
<td>业务系统命令请求。</td>
|
||||
<td>CommandDispatcher 调用、设备应答摘要。</td>
|
||||
<td><span class="tag">远程控制</span><span class="tag">参数设置</span><span class="tag">属性查询</span><span class="tag">区域删除</span><span class="tag">报警确认</span><span class="tag">视频控制</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/bootstrap-all</td>
|
||||
<td>一体化启动模块,聚合各模块 AutoConfiguration 和运行配置。</td>
|
||||
<td>application.yml、Spring Boot 启动。</td>
|
||||
<td>完整接入服务实例。</td>
|
||||
<td><span class="tag green">部署</span><span class="tag green">开关配置</span></td>
|
||||
<td>modules/apps/gb32960-ingest-app</td>
|
||||
<td>生产 GB32960 TCP 接入应用,只负责接收、鉴权、解析、冷存引用和 Kafka 投递。</td>
|
||||
<td>GB/T 32960 TCP 32960。</td>
|
||||
<td><code>vehicle.raw.gb32960.v1</code>、<code>vehicle.event.gb32960.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">32960</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/jt808-ingest-app</td>
|
||||
<td>生产 JT808 TCP 接入应用,只负责 808 注册/鉴权/位置等上行解析、冷存引用和 Kafka 投递。</td>
|
||||
<td>JT/T 808 TCP 808。</td>
|
||||
<td><code>vehicle.raw.jt808.v1</code>、<code>vehicle.event.jt808.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">808</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/yutong-mqtt-app</td>
|
||||
<td>生产宇通 MQTT 接入应用,按 endpoint profile 解析 MQTT 报文并投递 Kafka。</td>
|
||||
<td>宇通 MQTT topic。</td>
|
||||
<td><code>vehicle.raw.mqtt-yutong.v1</code>、<code>vehicle.event.mqtt-yutong.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">MQTT</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/vehicle-history-app</td>
|
||||
<td>生产历史应用,消费 32960、808、宇通 MQTT 的 raw/event Kafka Envelope,写 TDengine 并提供历史查询。</td>
|
||||
<td>Kafka Protobuf Envelope。</td>
|
||||
<td>TDengine <code>raw_frames</code>、位置表和历史查询 API。</td>
|
||||
<td><span class="tag green">历史查询</span><span class="tag amber">raw JSON</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/vehicle-analytics-app</td>
|
||||
<td>生产指标应用,当前只消费 JT808 事件并用 GPS 总里程首末差值写每日里程指标。</td>
|
||||
<td><code>vehicle.event.jt808.v1</code>。</td>
|
||||
<td>MySQL <code>vehicle_stat_metric</code>。</td>
|
||||
<td><span class="tag green">每日里程</span><span class="tag">指标表</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -691,7 +712,7 @@ flowchart LR
|
||||
<tr>
|
||||
<td>明细展示和导出</td>
|
||||
<td><code>vehicle-history-app</code> + <code>tdengine-history-store</code></td>
|
||||
<td>生产查询通过 TDengine raw_frames、位置表和 rawUri 关联完成;Parquet/DuckDB 仅作为兼容文件型明细库,不进入默认热路径。</td>
|
||||
<td>生产查询通过 TDengine raw_frames、位置表和 rawUri 关联完成;不再维护文件型事件索引旁路。</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
227
docs/operations/current-ecs-deployment.md
Normal file
227
docs/operations/current-ecs-deployment.md
Normal file
@@ -0,0 +1,227 @@
|
||||
# 当前 ECS Go 原生部署说明
|
||||
|
||||
更新时间:2026-07-02 01:30 CST
|
||||
|
||||
本文档记录当前 `lingniu-vehicle-ingest` 的生产运行面。当前 goal 的接入链路已经切到 Go 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。密钥只维护在 ECS 环境文件或受控凭据中,不写入 Git。
|
||||
|
||||
## 部署范围
|
||||
|
||||
| systemd 服务 | 二进制 | 说明 | 对外端口 |
|
||||
| --- | --- | --- | --- |
|
||||
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP、JT808 TCP、宇通 MQTT 接入,解析后写 Kafka RAW 和统一事件 | TCP `32960`、TCP `808` |
|
||||
| `lingniu-go-history-writer.service` | `history-writer` | 消费 Kafka RAW,写 TDengine RAW、位置点、里程点核心表 | 无 HTTP |
|
||||
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 Kafka RAW,按总里程差值法写 MySQL 每日指标 | 无 HTTP |
|
||||
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费统一事件写 Redis,并提供实时、历史、统计查询 API | HTTP `20210` |
|
||||
|
||||
信达 Push 已废弃,不参与当前 Go 生产链路。旧 Java/Docker 服务不应占用 `808`、`32960`、`20210`。
|
||||
|
||||
## 应用 ECS
|
||||
|
||||
| 项 | 值 |
|
||||
| --- | --- |
|
||||
| 公网 IP | `115.29.187.205` |
|
||||
| 登录用户 | `root` |
|
||||
| 部署目录 | `/opt/lingniu-go-native` |
|
||||
| 当前 release | `/opt/lingniu-go-native/current` |
|
||||
| 环境文件目录 | `/opt/lingniu-go-native/env` |
|
||||
| systemd unit 目录 | `/etc/systemd/system` |
|
||||
| 对外服务 | `32960`、`808`、`20210` |
|
||||
|
||||
运行目录结构:
|
||||
|
||||
```text
|
||||
/opt/lingniu-go-native/
|
||||
current -> /opt/lingniu-go-native/releases/<git-short-sha>
|
||||
releases/<git-short-sha>/
|
||||
gateway
|
||||
history-writer
|
||||
stat-writer
|
||||
realtime-api
|
||||
env/
|
||||
gateway.env
|
||||
history-writer.env
|
||||
stat-writer.env
|
||||
realtime-api.env
|
||||
spool/gateway/
|
||||
```
|
||||
|
||||
## 中间件
|
||||
|
||||
| 组件 | 内网地址 | 公网地址 | 用途 |
|
||||
| --- | --- | --- | --- |
|
||||
| Kafka | `172.17.111.56:9092` | `114.55.58.251:9092` | RAW 和统一事件消息总线 |
|
||||
| TDengine | `172.17.111.57:6041` | `115.29.185.82:6041` | RAW、位置、里程点时序热存储 |
|
||||
| MySQL RDS | `rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306` | `rm-bp179zbv481rnw3e2no.mysql.rds.aliyuncs.com:3306` | 身份映射、JT808 注册、每日指标 |
|
||||
| Redis RDS | `r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379` | 无 | 准实时车辆状态缓存 |
|
||||
|
||||
生产应用之间访问中间件优先使用内网地址。RDS 白名单需要允许应用 ECS 私网访问。
|
||||
|
||||
## Kafka Topic
|
||||
|
||||
| Topic | 生产者 | 消费者 | 内容 |
|
||||
| --- | --- | --- | --- |
|
||||
| `vehicle.raw.go.gb32960.v1` | `gateway` | `history-writer`、`stat-writer` | GB32960 完整 RAW 记录 |
|
||||
| `vehicle.raw.go.jt808.v1` | `gateway` | `history-writer`、`stat-writer` | JT808 完整 RAW 记录 |
|
||||
| `vehicle.raw.go.yutong-mqtt.v1` | `gateway` | `history-writer` | 宇通 MQTT 完整 RAW 记录 |
|
||||
| `vehicle.event.go.unified.v1` | `gateway` | `realtime-api` | 三类协议统一事件,用于 Redis 实时状态 |
|
||||
|
||||
`gateway` 配置了 Kafka retry;生产 env 中启用 `KAFKA_SPOOL_DIR` 时,本地 spool 可在 Kafka 短暂不可用后回放。
|
||||
|
||||
Kafka 消费组:
|
||||
|
||||
| Consumer Group | Topic | 当前验证 |
|
||||
| --- | --- | --- |
|
||||
| `go-history-writer` | 三个 `vehicle.raw.go.*` topic | 2026-07-02 验证 lag 为 `0` |
|
||||
| `go-stat-writer` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1` | 2026-07-02 验证 lag 为 `0` |
|
||||
| `go-realtime-api` | `vehicle.event.go.unified.v1` | 2026-07-02 验证 lag 为 `0` |
|
||||
|
||||
可重复 smoke:
|
||||
|
||||
```bash
|
||||
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
|
||||
```
|
||||
|
||||
该脚本通过 SSH 登录 Kafka ECS,检查 Go 生产 topic 是否存在,并校验 `go-history-writer`、`go-stat-writer`、`go-realtime-api` 的消费组 lag。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
## TDengine 数据层
|
||||
|
||||
默认数据库:`lingniu_vehicle_ts`
|
||||
|
||||
| 表 | 类型 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `raw_frames` | stable | 完整 RAW 帧,包含 `raw_hex`、`parsed_json`、`fields_json`、`source_endpoint` 和协议/车辆 tags |
|
||||
| `vehicle_locations` | stable | 最小化位置历史,保留 `frame_id` 回链 RAW |
|
||||
| `vehicle_mileage_points` | stable | 最小化总里程点,供历史查询和统计复核 |
|
||||
|
||||
设计原则:
|
||||
|
||||
- 完整结构化 payload 只落在 `raw_frames.parsed_json` / `fields_json`。
|
||||
- `vehicle_locations` 和 `vehicle_mileage_points` 只保留核心查询字段,不重复保存完整 JSON。
|
||||
- `telemetry_fields` 不由当前服务维护,字段配置解析由独立子服务处理。
|
||||
- API 传入东八区时间时,服务会转换为 TDengine 当前存储使用的 UTC 字面量再查询。
|
||||
|
||||
## MySQL 数据层
|
||||
|
||||
默认业务库:`lingniu_vehicle_data`
|
||||
|
||||
| 表 | 维护方 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `vehicle_identity_binding` | 人工/外部主数据 | VIN 映射主表,用 `phone`、`device_id`、`plate` 降级定位 VIN |
|
||||
| `jt808_registration` | `gateway` 自动写入 | 仅 JT808 注册和鉴权记录,以 `phone` 作为主键,记录设备、车牌、厂家、鉴权、来源端点 |
|
||||
| `vehicle_daily_metric` | `stat-writer` 自动写入 | 每日指标表,保存 `daily_mileage_km` 和 `daily_total_mileage_km` |
|
||||
|
||||
每日里程算法:
|
||||
|
||||
```text
|
||||
daily_mileage_km = 当日最大 total_mileage_km - 当日最小 total_mileage_km
|
||||
daily_total_mileage_km = 当日最大 total_mileage_km
|
||||
```
|
||||
|
||||
统计按 `Asia/Shanghai` 自然日归档,当前支持 GB32960、JT808、宇通 MQTT 中能解析出 `total_mileage_km` 的数据。JT808 的总里程来自位置附加信息 `0x01`,单位按协议转换为 km;宇通 MQTT 的 `TOTAL_MILEAGE` 原始单位为米,入库前转换为 km。
|
||||
|
||||
## Redis 实时层
|
||||
|
||||
`realtime-api` 消费 `vehicle.event.go.unified.v1`,按 VIN/vehicle key 维护准实时快照。主要能力:
|
||||
|
||||
| API | 说明 |
|
||||
| --- | --- |
|
||||
| `/api/realtime/vehicles/{vin}` | 查询车辆合并实时快照 |
|
||||
| `/api/realtime/vehicles/{vin}/online` | 查询车辆是否在线 |
|
||||
| `/api/realtime/vehicles/{vin}/protocols/{protocol}` | 查询指定协议的实时快照 |
|
||||
|
||||
Redis 只作为实时缓存;历史和统计以 TDengine/MySQL 为准。
|
||||
|
||||
## API 入口
|
||||
|
||||
公网基础地址:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210
|
||||
```
|
||||
|
||||
常用查询:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=YUTONG_MQTT&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/locations?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/mileage-points?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/stats/daily-metrics?dateFrom=2026-07-02&dateTo=2026-07-02&limit=20'
|
||||
```
|
||||
|
||||
分页参数统一使用 `limit` 和 `offset`。
|
||||
|
||||
部署后推荐直接运行 Go 原生生产 smoke:
|
||||
|
||||
```bash
|
||||
python3 tools/go_prod_acceptance.py --date 2026-07-02
|
||||
```
|
||||
|
||||
该聚合脚本会串行执行 systemd/端口检查、Kafka topic/consumer lag 检查、HTTP 数据链路检查,任一子检查失败时整体退出码为非 0。需要 SSH 可登录应用 ECS 和 Kafka ECS。
|
||||
|
||||
也可以单独运行子检查:
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
|
||||
```
|
||||
|
||||
`go_systemd_prod_smoke.py` 通过 SSH 检查四个 Go systemd 服务是否 `active` 且 `enabled`,并确认 `808`、`32960` 由 `gateway` 监听、`20210` 由 `realtime-api` 监听,同时要求 `/opt/lingniu-go-native/current` 指向有效 release、四个生产二进制均可执行、`/opt/lingniu-go-native/spool/gateway` 没有待回放文件,避免旧 Java/Docker 进程占用生产端口、部署目录错乱或 Kafka spool 静默堆积。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
`go_native_prod_smoke.py` 通过 `20210` HTTP API 验证 GB32960、JT808、宇通 MQTT 的 RAW 查询及结构化 `parsed_json`,三类协议的位置和里程点查询及 `frame_id` RAW 回链,GB32960/JT808 的 `daily_mileage_km`、`daily_total_mileage_km` 统计公式,以及三类协议的 Redis realtime snapshot、online、protocol 查询。实时 snapshot/protocol 查询会校验 `updated_at_ms` 新鲜度;默认检查今天东八区数据时,还会要求三类 RAW 最新样本不超过 15 分钟,避免旧数据误判为接收正常;任一检查不达标时退出码为非 0。
|
||||
|
||||
## 部署命令
|
||||
|
||||
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS,切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收:
|
||||
|
||||
```bash
|
||||
python3 tools/go_native_deploy.py --date 2026-07-02
|
||||
```
|
||||
|
||||
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout` 和 `--spool-poll-interval` 调整。
|
||||
|
||||
脚本执行的发布动作等价于:
|
||||
|
||||
```bash
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/history-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/stat-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/realtime-api
|
||||
scp release.tar.gz root@115.29.187.205:/tmp/
|
||||
ln -sfn /opt/lingniu-go-native/releases/<git-short-sha> /opt/lingniu-go-native/current
|
||||
systemctl daemon-reload
|
||||
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
```
|
||||
|
||||
只更新查询 API 时,可以仅替换 `/opt/lingniu-go-native/current/realtime-api` 并重启:
|
||||
|
||||
```bash
|
||||
systemctl restart lingniu-go-realtime-api.service
|
||||
```
|
||||
|
||||
## 运行检查
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-history-writer --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-stat-writer --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-realtime-api --since '5 minutes ago' --no-pager
|
||||
```
|
||||
|
||||
2026-07-02 01:30 CST 已验证:
|
||||
|
||||
- 四个 Go systemd 服务均为 `active` 且 `enabled`。
|
||||
- `/opt/lingniu-go-native/current` 指向有效 release,四个生产二进制均存在且可执行。
|
||||
- `gateway` 正在监听 `32960` 和 `808`。
|
||||
- `realtime-api` 正在监听 `20210`。
|
||||
- gateway Kafka spool 当前无待回放文件。
|
||||
- GB32960、JT808、YUTONG_MQTT 的东八区 RAW 查询均可命中生产数据,且最新 RAW 样本包含结构化 `parsed_json`。
|
||||
- GB32960、JT808、YUTONG_MQTT 的最新 RAW 样本均在 15 分钟内。
|
||||
- GB32960、JT808、YUTONG_MQTT 的 `vehicle_locations`、`vehicle_mileage_points` 查询均可命中生产数据,且样本包含 `frame_id` 回链 RAW。
|
||||
- `vehicle_daily_metric` 可查到 `daily_mileage_km` 和 `daily_total_mileage_km`,且公式满足 `daily_mileage_km = latest_total_mileage_km - first_total_mileage_km`、`daily_total_mileage_km = latest_total_mileage_km`。
|
||||
- Redis realtime 可查到 GB32960、JT808、YUTONG_MQTT 的在线状态和协议快照,且 snapshot/protocol 快照最近刷新。
|
||||
@@ -1,28 +1,33 @@
|
||||
# GB32960 Split Service Runbook
|
||||
|
||||
This runbook covers the split GB32960 ingest, history, and analytics runtimes. Kafka is the durability boundary between the protocol service and downstream business services.
|
||||
This runbook covers the default production runtimes: GB32960 ingest, JT808 ingest, Yutong MQTT ingest, history, and analytics. Kafka is the durability boundary between protocol ingest services and downstream business services.
|
||||
|
||||
## Service Roles
|
||||
|
||||
| Service | Module | Role | Local ports |
|
||||
| Service | Module | Role | Production ports |
|
||||
| --- | --- | --- | --- |
|
||||
| GB32960 ingest | `:gb32960-ingest-app` | Accept GB32960 TCP connections, decode/authenticate frames, produce raw and normalized records to Kafka, and send protocol ACKs after required Kafka production succeeds. | TCP `32960`, HTTP `20100` |
|
||||
| JT808 ingest | `:jt808-ingest-app` | Accept JT/T 808 TCP connections on the production receive port, parse raw frames, archive frame bytes, and produce raw/event envelopes to Kafka. | TCP `808`, HTTP `20482` |
|
||||
| Vehicle history | `:vehicle-history-app` | Consume GB32960/JT808 raw/event Kafka records, store compact history rows plus TDengine `raw_frames` `parsedJson`/`metadataJson`/`rawUri`, and expose TDengine-backed history APIs. | HTTP `20200` |
|
||||
| Vehicle analytics | `:vehicle-analytics-app` | Consume JT808 event Kafka records, calculate daily mileage from reported GPS total mileage, and write metrics to `vehicle_stat_metric`. | HTTP `20300` |
|
||||
| JT808 ingest | `:jt808-ingest-app` | Accept JT/T 808 TCP connections on the production receive port, parse raw frames, archive frame bytes, and produce raw/event envelopes to Kafka. | TCP `808`, HTTP `20400` |
|
||||
| Yutong MQTT ingest | `:yutong-mqtt-app` | Connect to Yutong MQTT, parse subscribed messages, archive payloads, and produce raw/event envelopes to Kafka. | HTTP `20500` |
|
||||
| Vehicle history | `:vehicle-history-app` | Consume GB32960/JT808/Yutong MQTT raw/event Kafka records, store compact history rows plus TDengine `raw_frames` `parsedJson`/`metadataJson`/`rawUri`, and expose TDengine-backed history APIs. | HTTP `20200` |
|
||||
| Vehicle analytics | `:vehicle-analytics-app` | Consume JT808 event Kafka records, calculate daily mileage from reported GPS total mileage, and write metrics to `vehicle_stat_metric`. | HTTP `20310` |
|
||||
|
||||
## Kafka Contract
|
||||
|
||||
| Topic | Producer | Consumers | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `vehicle.raw.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Raw frame records. History writes the parsed JSON, metadata JSON, and raw URI to TDengine `raw_frames`. |
|
||||
| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` | Normalized vehicle events keyed by VIN when available. |
|
||||
| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Normalized vehicle events keyed by VIN when available. |
|
||||
| `vehicle.dlq.gb32960.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed production or consumer processing. |
|
||||
| `vehicle.raw.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app` | JT808 raw frame records for TDengine `raw_frames` and raw-frame troubleshooting APIs. |
|
||||
| `vehicle.event.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app` | JT808 parsed location/session/alarm events keyed by phone or mapped VIN. |
|
||||
| `vehicle.event.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` | JT808 parsed location/session/alarm events keyed by phone or mapped VIN; analytics uses location total mileage for daily mileage metrics. |
|
||||
| `vehicle.dlq.jt808.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed JT808 production or consumer processing. |
|
||||
| `vehicle.raw.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT raw payload records for TDengine `raw_frames` and troubleshooting APIs. |
|
||||
| `vehicle.event.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT parsed telemetry events keyed by mapped VIN or device identity. |
|
||||
| `vehicle.dlq.mqtt-yutong.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed Yutong MQTT production or consumer processing. |
|
||||
| `vehicle.dlq.history.v1` | History app consumer error paths | Operators/replay tooling | Dead-letter records for failed history storage processing. |
|
||||
|
||||
Default local consumer groups:
|
||||
Default consumer groups:
|
||||
|
||||
- History: `vehicle-history`
|
||||
- Analytics statistics: `vehicle-stat`
|
||||
@@ -35,13 +40,13 @@ History and analytics failures do not block GB32960 ACKs. They are downstream Ka
|
||||
|
||||
Authorization failures are rejected before the Kafka durability boundary and can be ACKed/rejected immediately according to protocol handling.
|
||||
|
||||
## Local Prerequisites
|
||||
## Build Prerequisites
|
||||
|
||||
- Java 25 and Maven available on PATH.
|
||||
- Local Kafka reachable at `127.0.0.1:9092`, plus Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer`.
|
||||
- Production GB32960/JT808 session state uses Redis by default. For local
|
||||
one-process smoke tests without Redis, set `SESSION_STORE=memory` explicitly.
|
||||
- No repository-local Kafka bootstrap script or compose file was found during Task 8 verification on 2026-06-23.
|
||||
- Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer` available on the ECS/operator host.
|
||||
- Session state requires Redis. GB32960/JT808 ingest services keep live Netty
|
||||
channels in-process, but session indexes and TTL metadata use Redis only.
|
||||
- Redis is configured through Portainer environment variables such as `REDIS_HOST` and `REDIS_DATABASE=50`; there is no memory session-store fallback.
|
||||
|
||||
If your shell needs an explicit JDK:
|
||||
|
||||
@@ -55,99 +60,82 @@ export PATH="$JAVA_HOME/bin:/opt/homebrew/bin:$PATH"
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
Expected result: `BUILD SUCCESS` and these runnable jars:
|
||||
|
||||
- `modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar`
|
||||
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
|
||||
- `modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar`
|
||||
- `modules/apps/vehicle-history-app/target/vehicle-history-app.jar`
|
||||
- `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar`
|
||||
|
||||
## Create Topics
|
||||
|
||||
Run these only after Kafka is reachable at `127.0.0.1:9092`:
|
||||
Run these only on the ECS/operator host after Kafka is reachable. The default bootstrap address is the ECS private Kafka address used by the production stack:
|
||||
|
||||
```bash
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.jt808.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.jt808.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.jt808.v1 --partitions 3 --replication-factor 1
|
||||
KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:-172.17.111.56:9092}"
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.jt808.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.jt808.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.jt808.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.mqtt-yutong.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.mqtt-yutong.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.mqtt-yutong.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.history.v1 --partitions 3 --replication-factor 1
|
||||
```
|
||||
|
||||
Optional sanity check:
|
||||
|
||||
```bash
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808)\.v1'
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808|mqtt-yutong|history)\.v1'
|
||||
```
|
||||
|
||||
## Start Split Services Locally
|
||||
## Production Runtime
|
||||
|
||||
Run each command in a separate terminal from the repository root.
|
||||
Production services run on ECS through Portainer/Docker. Do not start GB32960, JT808, Yutong MQTT, history, or analytics services on local developer machines; local machines are for code build, unit tests, and source inspection only.
|
||||
|
||||
GB32960 ingest:
|
||||
Use `deploy/portainer/docker-compose.yml` as the production runtime manifest. The default stack is:
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=127.0.0.1:9092 \
|
||||
GB32960_PORT=32960 \
|
||||
HTTP_PORT=20100 \
|
||||
SESSION_STORE=memory \
|
||||
java --sun-misc-unsafe-memory-access=allow \
|
||||
-jar modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar
|
||||
```
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
- `vehicle-history-app`
|
||||
- `vehicle-analytics-app`
|
||||
|
||||
JT808 ingest:
|
||||
`vehicle-history-app` intentionally has no raw archive volume. The history hot path is Kafka to TDengine: raw envelopes are stored in TDengine `raw_frames` with `parsedJson`, `metadataJson`, and `rawUri`. Ingest services may still write original `.bin` files as cold backup, but history API queries must not depend on a shared local archive mount.
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=127.0.0.1:9092 \
|
||||
JT808_PORT=808 \
|
||||
HTTP_PORT=20400 \
|
||||
SESSION_STORE=memory \
|
||||
java --sun-misc-unsafe-memory-access=allow \
|
||||
-jar modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar
|
||||
```
|
||||
Keep `TDENGINE_HISTORY_ENABLED=true` in the `vehicle-history-app` environment so history reads and writes use TDengine in production.
|
||||
|
||||
Vehicle history:
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=127.0.0.1:9092 \
|
||||
HTTP_PORT=20200 \
|
||||
TDENGINE_HISTORY_ENABLED=true \
|
||||
TDENGINE_HISTORY_DATABASE=vehicle_ts \
|
||||
TDENGINE_JDBC_URL='jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts' \
|
||||
TDENGINE_USERNAME=root \
|
||||
TDENGINE_PASSWORD='<tdengine-password>' \
|
||||
TDENGINE_TELEMETRY_FIELDS_ENABLED=false \
|
||||
java --sun-misc-unsafe-memory-access=allow \
|
||||
-jar modules/apps/vehicle-history-app/target/vehicle-history-app.jar
|
||||
```
|
||||
|
||||
`SINK_ARCHIVE_PATH` is intentionally omitted from the history JVM. The history hot path is Kafka to TDengine: raw envelopes are stored in TDengine `raw_frames` with `parsedJson`, `metadataJson`, and `rawUri`. Ingest services may still write original `.bin` files as cold backup, but history API queries must not depend on a shared local archive mount.
|
||||
|
||||
Vehicle analytics:
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=127.0.0.1:9092 \
|
||||
HTTP_PORT=20300 \
|
||||
java --sun-misc-unsafe-memory-access=allow \
|
||||
-jar modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar
|
||||
```
|
||||
|
||||
`vehicle-analytics-app` is intentionally a metric runtime only. Latest-state APIs belong to `vehicle-state-service` and should be deployed as a separate consumer if that surface is needed.
|
||||
`vehicle-analytics-app` is intentionally a metric runtime only. Latest-state APIs belong to `vehicle-state-service`, which stays outside the default reactor and should be built with `-Poptional-latest-state` and deployed as a separate consumer if that surface is needed.
|
||||
|
||||
## Health Verification
|
||||
|
||||
Run these from the ECS host or inside the matching container:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20300/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
Expected: each endpoint returns `{"status":"UP"}` or an equivalent Spring Boot health JSON with top-level status `UP`.
|
||||
Expected: each aggregate, liveness, and readiness endpoint returns `{"status":"UP"}` or an equivalent Spring Boot health JSON with top-level status `UP`.
|
||||
|
||||
## End-to-End Verification
|
||||
|
||||
@@ -173,8 +161,10 @@ Expected: the ACK file is non-empty and the decoded bytes begin with the GB32960
|
||||
Verify only what was actually run in your environment:
|
||||
|
||||
```bash
|
||||
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
```
|
||||
|
||||
Use TDengine CLI or a JDBC client to verify history writes:
|
||||
@@ -185,6 +175,8 @@ SELECT COUNT(*) FROM raw_frames WHERE protocol = 'GB32960';
|
||||
SELECT COUNT(*) FROM vehicle_locations WHERE protocol = 'GB32960';
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM jt808_locations WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'MQTT_YUTONG';
|
||||
SELECT COUNT(*) FROM vehicle_locations WHERE protocol = 'MQTT_YUTONG';
|
||||
```
|
||||
|
||||
Useful HTTP query checks after events are consumed. For `realtime_001.hex`, the fixture VIN is `LTEST000000000001`; replace `<date-from>`, `<date-to>`, and `<stat-date>` with values that cover the consumed record's event time:
|
||||
@@ -193,7 +185,7 @@ Useful HTTP query checks after events are consumed. For `realtime_001.hex`, the
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/locations?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/raw-frames?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/dictionary'
|
||||
curl -sS 'http://127.0.0.1:20300/api/vehicle-stat/LTEST000000000001/daily?date=<stat-date>'
|
||||
curl -sS 'http://127.0.0.1:20310/api/vehicle-stat/LTEST000000000001/daily?date=<stat-date>'
|
||||
```
|
||||
|
||||
Expected E2E result when Kafka and all services are running:
|
||||
@@ -202,24 +194,23 @@ Expected E2E result when Kafka and all services are running:
|
||||
- `vehicle.raw.gb32960.v1` receives a raw frame envelope.
|
||||
- `vehicle.event.gb32960.v1` receives one or more normalized records.
|
||||
- TDengine `raw_frames` receives GB32960/JT808 RAW rows with parsed JSON and metadata.
|
||||
- TDengine location tables receive compact GB32960/JT808 location rows.
|
||||
- TDengine `raw_frames` receives Yutong MQTT RAW rows with parsed JSON and metadata when MQTT payloads are consumed.
|
||||
- TDengine location tables receive compact GB32960/JT808/Yutong MQTT location rows when applicable telemetry is present.
|
||||
- JT808 analytics writes daily mileage rows to MySQL `vehicle_stat_metric` after consuming applicable `vehicle.event.jt808.v1` records.
|
||||
|
||||
Do not expect `vehicle-history-app` to create raw `.bin` archive files from Kafka raw records in the current implementation. `RawArchiveEventSink` can write archive files only when it receives `VehicleEvent.RawArchive.rawBytes()` inside the same JVM; the Kafka envelope carries only `RawArchiveRef` metadata.
|
||||
|
||||
Do not mark any of these as verified unless the matching command was run and the output was inspected.
|
||||
|
||||
## Local launchctl Deployment
|
||||
## Runtime Location Policy
|
||||
|
||||
Use the templates under `deploy/local/launchctl/` for the local production-verification machine. They codify the current three-service deployment and keep the runtime knobs in one place:
|
||||
Production services run on ECS through Portainer/Docker.
|
||||
|
||||
- GB32960 ingest: `com.lingniu.gb32960`, TCP `32960`, HTTP `20100`
|
||||
- JT808 ingest: `com.lingniu.jt808`, TCP `808`, HTTP `20482`
|
||||
- History: `com.lingniu.vehicle-history`, HTTP `20200`
|
||||
- Do not create local service runners for GB32960, JT808, Yutong MQTT, history, or analytics.
|
||||
- Do not run the service jars directly on developer machines for production verification.
|
||||
- Keep local work limited to source edits, builds, and automated tests.
|
||||
|
||||
The launchctl history template does not inject `SINK_ARCHIVE_PATH`. Use TDengine `raw_frames` as the authoritative hot-query store; keep any ingest-side raw archive path as an optional cold backup for operator inspection.
|
||||
|
||||
For Portainer deployments, mirror this same shape: no history archive volume, history consumes raw/event Kafka topics and writes TDengine history rows.
|
||||
For Portainer deployments, keep history without an archive volume: history consumes raw/event Kafka topics and writes TDengine history rows.
|
||||
|
||||
## Rollback Guidance
|
||||
|
||||
@@ -231,37 +222,37 @@ If the split deployment is unhealthy:
|
||||
4. If Kafka itself is unavailable, stop `gb32960-ingest-app` or route GB32960 traffic to a previously verified release that preserves the Kafka durability boundary.
|
||||
5. After rollback, compare Kafka consumer group lag and DLQ contents before resuming the split services.
|
||||
|
||||
Rollback commands depend on the deployment environment. For local testing, stop the three Java processes and restart the previous verified app jars or images.
|
||||
Rollback commands depend on the deployment environment. For the default production stack, roll back the Portainer stack image version and let Docker restart the affected containers.
|
||||
|
||||
## Task 8 Verification Notes
|
||||
|
||||
Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`:
|
||||
|
||||
- Package build ran with `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true` and ended with `BUILD SUCCESS`.
|
||||
- Package build should use the current active-app command shown in the Build section.
|
||||
- Repository-local Kafka setup inspection found no Kafka script, no Docker Compose file, and no compose YAML within the searched repository paths.
|
||||
- `nc -z -w 2 127.0.0.1 9092` exited `1`, so no local Kafka broker was reachable at `127.0.0.1:9092`.
|
||||
- `kafka-topics`, `kafka-topics.sh`, `docker`, and `docker-compose` were not found on PATH.
|
||||
- Kafka topic creation, service startup, health checks, Kafka record checks, archive checks, TDengine checks, stat output checks, and ACK observation were not run because the local Kafka prerequisite was absent.
|
||||
|
||||
## Latest Local Verification
|
||||
## Latest Build Verification
|
||||
|
||||
Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`:
|
||||
|
||||
- Targeted module tests: `mvn -pl :sink-mq,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test` ended with `BUILD SUCCESS`; Maven reported 55 protocol/sink/core tests, 30 event-history tests, 14 vehicle-state tests, and 19 vehicle-stat tests with 0 failures/errors/skips.
|
||||
- Split app tests: `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app test` ended with `BUILD SUCCESS`; Maven reported 6 app composition/default tests with 0 failures/errors/skips.
|
||||
- Package verification: `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true` ended with `BUILD SUCCESS` and repackaged all three app jars.
|
||||
- Split app startup: not run in this verification pass because local Kafka was still absent.
|
||||
- Kafka raw records: not verified; no local Kafka broker or Kafka CLI tools were available.
|
||||
- Kafka event records: not verified; no local Kafka broker or Kafka CLI tools were available.
|
||||
- Targeted module tests: `mvn -pl :sink-kafka,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test` ended with `BUILD SUCCESS`; Maven reported 55 protocol/sink/core tests, 30 event-history tests, 14 vehicle-state tests, and 19 vehicle-stat tests with 0 failures/errors/skips.
|
||||
- Split app tests should cover the current active apps listed in the Service Roles section.
|
||||
- Package verification should use the active-app package command from the Build section.
|
||||
- Split app startup: not run in this verification pass because an ECS/Portainer runtime was outside that build-only check.
|
||||
- Kafka raw records: not verified in that build-only check.
|
||||
- Kafka event records: not verified in that build-only check.
|
||||
- TDengine history rows: not verified; downstream services were not started without Kafka.
|
||||
- Analytics output: not verified; downstream services were not started without Kafka.
|
||||
- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above in a Kafka-backed local or staging environment.
|
||||
- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above on ECS or staging.
|
||||
|
||||
## Remote Kafka Live Verification
|
||||
|
||||
Observed on 2026-06-23 with Kafka `114.55.58.251:9092`:
|
||||
|
||||
- Replaced the local `bootstrap-all` listener on TCP `32960` with `gb32960-ingest-app` on TCP `32960` and HTTP `20100`.
|
||||
- `gb32960-ingest-app` handled TCP `32960` and HTTP `20100`.
|
||||
- Ingest app health check `curl -sS http://127.0.0.1:20100/actuator/health` returned `{"status":"UP"}`.
|
||||
- Kafka AdminClient confirmed topics `vehicle.raw.gb32960.v1` and `vehicle.event.gb32960.v1` existed, and created `vehicle.dlq.gb32960.v1`.
|
||||
- The app accepted the live Hyundai platform connection from `8.134.95.166`, authenticated platform login, and received real GB32960 realtime/report/login/logout frames.
|
||||
@@ -271,15 +262,12 @@ Observed on 2026-06-23 with Kafka `114.55.58.251:9092`:
|
||||
|
||||
This verifies the live path `GB32960 TCP 32960 -> gb32960-ingest-app -> Kafka raw/event topics` against the remote Kafka broker. Downstream `vehicle-history-app` and `vehicle-analytics-app` were not started in this pass.
|
||||
|
||||
## 2026-06-29 Local Production-Path Verification
|
||||
## 2026-06-29 Superseded Local Verification Record
|
||||
|
||||
Observed on 2026-06-29 in worktree `.worktrees/vehicle-ingest-redesign-phase1`:
|
||||
Observed on 2026-06-29 in worktree `.worktrees/vehicle-ingest-redesign-phase1`. This record is retained only as historical evidence of parser/storage behavior; do not repeat it as a deployment procedure because production services now run on ECS through Portainer/Docker.
|
||||
|
||||
- `jt808-ingest-app` was running through launchctl on TCP `808` and HTTP `20482`; `curl -sS http://127.0.0.1:20482/actuator/health` returned top-level status `UP`.
|
||||
- `gb32960-ingest-app` was running through launchctl on TCP `32960` and HTTP `20100`; `curl -sS http://127.0.0.1:20100/actuator/health` returned top-level status `UP`.
|
||||
- `vehicle-history-app` was running through launchctl on HTTP `20200`; `curl -sS http://127.0.0.1:20200/actuator/health` returned top-level status `UP`.
|
||||
- External JT808 traffic on TCP `808` was parsed and written through Kafka into TDengine. Direct TDengine checks showed more than `6000` JT808 `raw_frames`, more than `1900` JT808 `vehicle_locations`, and more than `15000` JT808 `telemetry_fields`.
|
||||
- External JT808 traffic on TCP `808` was parsed and written through Kafka into TDengine. Direct TDengine checks showed more than `6000` JT808 `raw_frames` and more than `1900` JT808 `vehicle_locations`; current production history does not maintain a `telemetry_fields` table.
|
||||
- A GB32960 synthetic realtime frame sent to TCP `32960` received a binary ACK beginning with `2323`. The same VIN `LTEST202606290001` was queryable from TDengine-backed history APIs with speed, mileage, longitude, and latitude fields.
|
||||
- History raw and event Kafka consumption used separate bindings so raw-frame writes remained current while location/field event ingestion continued.
|
||||
|
||||
This verifies the local path `GB32960/JT808 TCP -> protocol app -> Kafka raw/event topics -> vehicle-history-app -> TDengine raw_frames/location tables -> Swagger/API query`.
|
||||
The verified behavior was `GB32960/JT808 TCP -> protocol app -> Kafka raw/event topics -> vehicle-history-app -> TDengine raw_frames/location tables -> Swagger/API query`; the deployment method from that historical run is superseded.
|
||||
|
||||
@@ -0,0 +1,888 @@
|
||||
# Go Vehicle Gateway 旁路验证记录
|
||||
|
||||
验证时间:2026-07-01 22:00-22:10 CST
|
||||
|
||||
## 部署版本
|
||||
|
||||
- Git commit:`60ca42e`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-60ca42e-20260701220622`
|
||||
- 部署方式:ECS `115.29.187.205` 上 Docker Compose 旁路部署
|
||||
- Compose 目录:`/opt/lingniu-go`
|
||||
|
||||
## 旁路端口和 topic
|
||||
|
||||
为了不影响现有 Java 生产服务,本次没有占用生产端口和生产 topic。
|
||||
|
||||
| 项 | 值 |
|
||||
|---|---|
|
||||
| GB32960 旁路 TCP | `115.29.187.205:23296 -> container :32960` |
|
||||
| JT808 旁路 TCP | `115.29.187.205:18080 -> container :808` |
|
||||
| Realtime API | `115.29.187.205:20210` |
|
||||
| GB32960 RAW topic | `vehicle.raw.go.gb32960.v1` |
|
||||
| JT808 RAW topic | `vehicle.raw.go.jt808.v1` |
|
||||
| Yutong MQTT RAW topic | `vehicle.raw.go.yutong-mqtt.v1` |
|
||||
| Unified topic | `vehicle.event.go.unified.v1` |
|
||||
|
||||
## 容器状态
|
||||
|
||||
ECS 上四个 Go 容器均已启动:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway Up
|
||||
go-history-writer Up
|
||||
go-stat-writer Up
|
||||
go-realtime-api Up
|
||||
```
|
||||
|
||||
## Kafka 验证
|
||||
|
||||
向 `18080` 注入 JT808 样例帧后,`vehicle.raw.go.jt808.v1` 和
|
||||
`vehicle.event.go.unified.v1` 均消费到统一 JSON envelope。
|
||||
|
||||
样例解析结果包含:
|
||||
|
||||
- `protocol=JT808`
|
||||
- `message_id=0x0200`
|
||||
- `phone=013307795425`
|
||||
- `longitude=121.069881`
|
||||
- `latitude=30.590151`
|
||||
- `speed_kmh=23`
|
||||
- `total_mileage_km=10241.2`
|
||||
- 表 27 附加项 `0x01` 原始值 `0001900C`
|
||||
|
||||
## TDengine 验证
|
||||
|
||||
目标库:`lingniu_vehicle_ts`
|
||||
|
||||
```text
|
||||
raw_frames count = 3
|
||||
vehicle_locations count = 2
|
||||
vehicle_mileage_points count = 2
|
||||
```
|
||||
|
||||
最新 RAW 标识:
|
||||
|
||||
```text
|
||||
GB32960 | LNBSCB3D4R1234567 | LNBSCB3D4R1234567 |
|
||||
JT808 | JT808:013307795425 | | 013307795425
|
||||
JT808 | JT808:013307795425 | | 013307795425
|
||||
```
|
||||
|
||||
说明:JT808 样例 phone 当前没有在 `vehicle_identity_binding` 中解析到 VIN,因此
|
||||
TDengine tag 使用 `vehicle_key=JT808:013307795425`。
|
||||
|
||||
## Redis 验证
|
||||
|
||||
向 `23296` 注入带 VIN 的 GB32960 合成帧后,Realtime API 可查询:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LNBSCB3D4R1234567
|
||||
```
|
||||
|
||||
返回字段包含:
|
||||
|
||||
- `vin=LNBSCB3D4R1234567`
|
||||
- `protocols=["GB32960"]`
|
||||
- `online=true`
|
||||
- `speed_kmh=30`
|
||||
- `total_mileage_km=10000`
|
||||
- `soc_percent=85`
|
||||
- `longitude=121`
|
||||
- `latitude=30.56`
|
||||
|
||||
## MySQL 统计验证
|
||||
|
||||
向 `23296` 注入带 VIN 的 GB32960 合成帧后,`vehicle_daily_metric` 写入:
|
||||
|
||||
```text
|
||||
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_mileage_km | 0.000 | 10000.000 | 10000.000 | 1
|
||||
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_total_mileage_km | 10000.000 | 10000.000 | 10000.000 | 1
|
||||
```
|
||||
|
||||
说明:本次 GB32960 合成帧的设备时间为测试值,因此统计日期为 `2020-07-01`。
|
||||
|
||||
## 已修复问题
|
||||
|
||||
- TDengine WebSocket 普通 `INSERT` 使用参数占位符时,字符串未按预期加引号,导致语法错误。已在 `60ca42e` 中改为显式 SQL literal 转义。
|
||||
- MySQL Go DSN 中 `charset=utf8mb4%2Cutf8` 会被 driver 传给 MySQL,导致语法错误;ECS 旁路 env 已改为 `charset=utf8mb4`。
|
||||
- TDengine ECS 当前可用 root 密码与早先记录不一致,本次旁路 env 已按实际可认证值修正。
|
||||
|
||||
## 未完成
|
||||
|
||||
- 未切换真实生产端口 `32960/808`。
|
||||
- 未将真实外部 32960/808 流量转发到 Go 旁路端口。
|
||||
- 未开启宇通 MQTT 正式订阅。
|
||||
- JT808 样例未解析到 VIN,需维护 `vehicle_identity_binding` 后再验证 808 每日统计。
|
||||
|
||||
## 2026-07-01 22:22 真实归档帧复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`5f4f4fd`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5f4f4fd-20260701222128`
|
||||
|
||||
本轮补充验证内容:
|
||||
|
||||
- JT808 支持 2019 版 versioned header,真实帧 `0x0200` 可解析出 `phone=14894135060`、`longitude=117.178187`、`latitude=30.570155`、`speed_kmh=33.4`、`total_mileage_km=7868.9`。
|
||||
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 可解析 `0x01` 整车、`0x02` 驱动电机、`0x03` 燃料电池、`0x04` 发动机、`0x05` 位置、`0x06` 极值、`0x07` 报警、`0x08` 电压、`0x09` 温度;后续 `0x30` 广东燃料电池扩展暂以 unknown raw 保留。
|
||||
- GB32960 真实帧字段包含 `fuel_cell_hydrogen_consumption_kg_per_100km=1.8`、`longitude=120.800326`、`latitude=31.634907`、`total_mileage_km=53490.9`。
|
||||
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回上述经纬度和氢耗字段。
|
||||
- TDengine `vehicle_locations` 和 `vehicle_mileage_points` 已写入真实 GB32960 位置和里程点:
|
||||
|
||||
```text
|
||||
2020-07-01 08:09:22 | GB32960 | LB9A32A21R0LS1707 | lon=120.800326 | lat=31.634907 | mileage=53490.9
|
||||
```
|
||||
|
||||
后续仍需补齐:
|
||||
|
||||
- JT808 真实 phone `14894135060` 仍未解析到 VIN,需维护身份绑定后再验证 VIN 维度实时和统计。
|
||||
|
||||
## 2026-07-01 22:28 广东燃料电池扩展复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`a15de91`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-a15de91-20260701222743`
|
||||
|
||||
本轮补充验证内容:
|
||||
|
||||
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 的 vendor 尾部已从 `unknown_unit` 补为结构化块:
|
||||
`0x30 gd_fc_stack`、`0x31 gd_fc_auxiliary`、`0x32 gd_fc_dcdc`、`0x33 gd_fc_air_conditioner`、
|
||||
`0x34 gd_fc_vehicle_info`、`0x80 gd_fc_demo_extension`、`0x83 gd_fc_vendor_tlv`。
|
||||
- Kafka `vehicle.raw.go.gb32960.v1` 最新消息包含上述 block,并输出关键扁平字段:
|
||||
`gd_fc_stack_hydrogen_inlet_pressure_kpa=97`、`gd_fc_stack_cell_count=108`、
|
||||
`gd_fc_vehicle_hydrogen_mass_kg=4.3`、`gd_fc_demo_stack_temp_c=65`。
|
||||
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回 vendor 扁平字段:
|
||||
|
||||
```text
|
||||
gd_fc_dcdc_controller_temp_c=47
|
||||
gd_fc_demo_stack_temp_c=65
|
||||
gd_fc_stack_cell_count=108
|
||||
gd_fc_stack_frame_cell_count=108
|
||||
gd_fc_stack_hydrogen_inlet_pressure_kpa=97
|
||||
gd_fc_stack_water_outlet_temp_c=65
|
||||
gd_fc_vehicle_hydrogen_mass_kg=4.3
|
||||
```
|
||||
|
||||
- TDengine `raw_frames` 中 GB32960 行数增加到 `4`,最新 RAW 标识:
|
||||
|
||||
```text
|
||||
2026-07-01 14:28:24.869 | 172.20.0.1:47082 | GB32960 | LB9A32A21R0LS1707 | OK
|
||||
```
|
||||
|
||||
后续仍需补齐:
|
||||
|
||||
- JT808 真实 phone `14894135060` 到 VIN 的绑定数据维护与统计复验。
|
||||
- Go 旁路尚未切换到生产 `32960/808` 端口;仍运行在 `23296/18080`。
|
||||
|
||||
## 2026-07-01 22:43 JT808 前导零 phone 身份解析复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`6b55694`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-6b55694-20260701224300`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- JT808 2011 老版包头的终端手机号按 BCD[6] 解析会保留前导 `0`,例如包头为 `013079963379`。
|
||||
- `vehicle_identity_binding.phone` 中实际维护的是去前导零后的手机号 `13079963379`。
|
||||
- 身份解析候选键已调整为先查原始 phone,再查去前导零 phone,之后再按 `device_id`、`plate` 降级。
|
||||
|
||||
真实帧复验样本:
|
||||
|
||||
- 归档帧:`/opt/lingniu/vehicle-ingest/gb32960/archive/2026/07/01/JT808/unknown/1782903942845000.bin`
|
||||
- 消息:`0x0200` 位置信息汇报
|
||||
- 包头 phone:`013079963379`
|
||||
- 绑定表 phone:`13079963379`
|
||||
- 解析 VIN:`LKLG7C4E3NA774736`
|
||||
|
||||
Kafka `vehicle.raw.go.jt808.v1` 最新消息确认:
|
||||
|
||||
```text
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
parsed.header.phone_trim_zero=13079963379
|
||||
parsed.identity.resolved=true
|
||||
parsed.identity.source=phone
|
||||
parsed.identity.value=13079963379
|
||||
```
|
||||
|
||||
Realtime API 确认:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
speed_kmh=0
|
||||
total_mileage_km=0
|
||||
device_time=2026-07-01T19:05:40+08:00
|
||||
```
|
||||
|
||||
TDengine 确认:
|
||||
|
||||
```text
|
||||
raw_frames:
|
||||
2026-07-01 14:49:10.913 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | message_id=512
|
||||
|
||||
vehicle_locations:
|
||||
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | lon=119.557997 | lat=29.049092 | speed=0 | mileage=0
|
||||
|
||||
vehicle_mileage_points:
|
||||
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | mileage=0
|
||||
```
|
||||
|
||||
MySQL `vehicle_daily_metric` 确认:
|
||||
|
||||
```text
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
说明:该真实位置帧携带的表 27 附加项 `0x01` 总里程值为 `0`,因此本次只验证身份解析、链路落地和差值统计写入;要验证非零日里程,需要回放或等待同一 VIN 的非零总里程连续位置点。
|
||||
|
||||
## 2026-07-01 22:55 Kafka 发布重试部署复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`7e7066d`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7e7066d-20260701225503`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- gateway 的 Kafka 发布增加同步重试封装。
|
||||
- 默认配置:
|
||||
- `KAFKA_PUBLISH_ATTEMPTS=3`
|
||||
- `KAFKA_PUBLISH_BACKOFF_MS=100`
|
||||
- `KAFKA_PUBLISH_TIMEOUT_MS=3000`
|
||||
- RAW 仍先于 unified event 写入;RAW 写失败不会继续写 unified。
|
||||
- 单元测试覆盖首次失败后成功、重试耗尽返回错误、上下文取消时停止重试。
|
||||
|
||||
部署后容器状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway | go-7e7066d-20260701225503 | Up
|
||||
go-history-writer | go-7e7066d-20260701225503 | Up
|
||||
go-stat-writer | go-7e7066d-20260701225503 | Up
|
||||
go-realtime-api | go-7e7066d-20260701225503 | Up
|
||||
```
|
||||
|
||||
真实 808 帧回放复验:
|
||||
|
||||
```text
|
||||
source_endpoint=172.20.0.1:51966
|
||||
protocol=JT808
|
||||
message_id=0x0200
|
||||
phone=013079963379
|
||||
vin=LKLG7C4E3NA774736
|
||||
parsed.identity.value=13079963379
|
||||
```
|
||||
|
||||
Kafka `vehicle.raw.go.jt808.v1` 确认新消息写入,partition 6 offset 从 `1` 增加到 `2`。最新消息包含:
|
||||
|
||||
```text
|
||||
received_at_ms=1782917751903
|
||||
source_endpoint=172.20.0.1:51966
|
||||
vin=LKLG7C4E3NA774736
|
||||
fields.longitude=119.557997
|
||||
fields.latitude=29.049092
|
||||
fields.total_mileage_km=0
|
||||
```
|
||||
|
||||
TDengine `raw_frames` 确认新 RAW 落地:
|
||||
|
||||
```text
|
||||
2026-07-01 14:55:51.903 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | source=172.20.0.1:51966
|
||||
```
|
||||
|
||||
Realtime API 确认 Redis 已刷新:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
|
||||
received_at_ms=1782917751903
|
||||
source_endpoint=172.20.0.1:51966
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=0
|
||||
```
|
||||
|
||||
说明:当前重试层只能覆盖 Kafka 短暂抖动;如果 Kafka 长时间不可用,仍需后续增加本地磁盘 spool/WAL 和恢复补发。
|
||||
|
||||
## 2026-07-01 23:02 切换 Go 到生产 32960/808 端口
|
||||
|
||||
本轮操作:
|
||||
|
||||
- 停止 ECS 上原有 Java Docker 容器:
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
- `vehicle-history-app`
|
||||
- `vehicle-analytics-app`
|
||||
- `vehicle-state-app`
|
||||
- `telemetry-field-parser-app`
|
||||
- 保留 Go sidecar 和 Portainer agent。
|
||||
- 修改 `/opt/lingniu-go/.env`:
|
||||
- `GO_GB32960_TCP_PORT=32960`
|
||||
- `GO_JT808_TCP_PORT=808`
|
||||
- Go gateway 使用镜像:
|
||||
- `crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-243de5c-20260701230028`
|
||||
- 开启 Kafka 本地 spool:
|
||||
- host:`/opt/lingniu-go/spool/gateway`
|
||||
- container:`/data/spool/gateway`
|
||||
|
||||
端口验证:
|
||||
|
||||
```text
|
||||
0.0.0.0:32960 -> go-vehicle-gateway:32960
|
||||
0.0.0.0:808 -> go-vehicle-gateway:808
|
||||
18080/23296 -> 已释放
|
||||
```
|
||||
|
||||
启动日志确认:
|
||||
|
||||
```text
|
||||
kafka durable spool enabled dir=/data/spool/gateway replay_interval_ms=1000
|
||||
tcp listener started protocol=GB32960 addr=[::]:32960
|
||||
tcp listener started protocol=JT808 addr=[::]:808
|
||||
```
|
||||
|
||||
真实连接确认:
|
||||
|
||||
```text
|
||||
GB32960 external remote: 8.134.95.166:50084
|
||||
JT808 external remote: 115.231.168.135:13801
|
||||
```
|
||||
|
||||
生产 `808` 回放复验:
|
||||
|
||||
```text
|
||||
source_endpoint=172.20.0.1:54680
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=0
|
||||
```
|
||||
|
||||
TDengine 最新 RAW 复验显示生产 808 已有真实外部数据持续落地:
|
||||
|
||||
```text
|
||||
2026-07-01 15:03:02.960 | JT808 | LKLG7C4E1NA774802 | 013079963291 | 222.66.200.68:28394
|
||||
2026-07-01 15:03:02.114 | JT808 | | 14894135583 | 122.152.221.156:33871
|
||||
2026-07-01 15:03:01.352 | JT808 | | 013307795519 | 115.231.168.135:13801
|
||||
2026-07-01 15:03:01.330 | JT808 | LKLG7C4E3NA774736 | 013079963379 | 222.66.200.68:28393
|
||||
```
|
||||
|
||||
Kafka spool 目录确认:
|
||||
|
||||
```text
|
||||
/opt/lingniu-go/spool/gateway file count = 0
|
||||
```
|
||||
|
||||
说明:生产 808 中仍有部分 phone 未解析出 VIN,后续需要继续维护 `vehicle_identity_binding` 的 phone/device_id/plate 到 VIN 映射,或者增强 808 注册/鉴权帧中的 identity 回写。
|
||||
|
||||
## 2026-07-01 23:09 808 里程统计防 0 污染复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`d347525`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-d347525-20260701230845`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- `stat-writer` 不再把 `total_mileage_km <= 0` 的采样写入 MySQL 每日指标。
|
||||
- MySQL upsert 对历史 `first_total_mileage_km=0` / `latest_total_mileage_km=0` 做纠偏;后续首次有效总里程会替换掉历史 0。
|
||||
- `realtime-api` 合并实时快照时,非正 `total_mileage_km` 不再覆盖已有正值;速度、位置、状态等其他字段仍按事件时间更新。
|
||||
|
||||
测试验证:
|
||||
|
||||
```text
|
||||
go test ./...
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
生产回放验证:
|
||||
|
||||
- 第一条 808 帧:`device_time=2026-07-01T23:09:35+08:00`,`total_mileage_km=12345.6`
|
||||
- 第二条 808 帧:`device_time=2026-07-01T23:09:36+08:00`,`total_mileage_km=0`
|
||||
- 两条帧使用同一 VIN:`LKLG7C4E3NA774736`
|
||||
|
||||
Redis merged 快照确认:第二条 0 里程帧没有覆盖已有有效总里程。
|
||||
|
||||
```text
|
||||
event_time_ms=1782918576000
|
||||
source_endpoint=172.20.0.1:40946
|
||||
speed_kmh=0
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=12345.6
|
||||
field_times_ms.total_mileage_km=1782918575000
|
||||
```
|
||||
|
||||
MySQL `vehicle_daily_metric` 确认:0 里程采样未进入统计,sample_count 只因有效总里程采样增加一次。
|
||||
|
||||
```text
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=13 | first=12345.600 | latest=12345.600
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 12345.600 | sample_count=13 | first=12345.600 | latest=12345.600
|
||||
```
|
||||
|
||||
说明:生产 TDengine 中已经存在大量 808 非零 `total_mileage_km` 采样,但许多 phone 尚未映射到 VIN,因此 MySQL 按 VIN 的统计只会覆盖已解析 VIN 的车辆。下一步应继续补齐 `vehicle_identity_binding`,让更多 808 车辆进入统计。
|
||||
|
||||
## 2026-07-01 23:23 808 注册身份自动维护复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`7b44f97`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7b44f97-20260701232200`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- 808 `0x0100` 注册帧解析 `province`、`city`、`manufacturer`、`device_type`、`device_id`、`plate_color`、`plate`。
|
||||
- 808 `0x0102` 鉴权帧解析 `auth_token`。
|
||||
- Go gateway 在 identity resolve 后写入 `jt808_registration`,普通上行也会刷新 `latest_seen_at`。
|
||||
- 注册帧通过车牌解析到 VIN 后,会尝试把 phone/device/plate 反写到 `vehicle_identity_binding`。
|
||||
- 注册帧车牌支持 GBK 解码,避免 `Incorrect string value` 写入错误。
|
||||
|
||||
验证命令:
|
||||
|
||||
```text
|
||||
go test ./...
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
ECS 运行状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway | go-7b44f97-20260701232200 | 0.0.0.0:808->808, 0.0.0.0:32960->32960
|
||||
go-history-writer | go-7b44f97-20260701232200 | Up
|
||||
go-stat-writer | go-7b44f97-20260701232200 | Up
|
||||
go-realtime-api | go-7b44f97-20260701232200 | 0.0.0.0:20210->20210
|
||||
```
|
||||
|
||||
生产日志确认:
|
||||
|
||||
- 新版本启动后真实 808、32960 连接持续进入。
|
||||
- GBK 车牌注册帧不再出现 `Incorrect string value`。
|
||||
|
||||
MySQL `jt808_registration` 确认:
|
||||
|
||||
```text
|
||||
registration_rows=214
|
||||
distinct_phone=214
|
||||
vin_rows=74
|
||||
plate_rows=74
|
||||
device_rows=72
|
||||
|
||||
013079963379 | device_id=9963379 | plate=沪A06788F | vin=LKLG7C4E3NA774736 | latest_registered_at=2026-07-01 23:23:01
|
||||
013079963321 | device_id=9963321 | plate=沪A59613F | vin=LKLG7C4EXNA774796 | latest_registered_at=2026-07-01 23:23:04
|
||||
```
|
||||
|
||||
说明:`jt808_registration.phone` 保存 808 包头原始 phone;`vehicle_identity_binding.phone` 中已有部分去前导零 phone。Go resolver 会同时查询原始 phone 和去前导零 phone。
|
||||
|
||||
## 2026-07-01 23:28 生产链路与指标清理复验
|
||||
|
||||
本轮复验目标:
|
||||
|
||||
- 确认 Go gateway 的 Kafka durable spool 是重启窗口残留,而不是持续发布失败。
|
||||
- 确认 stat-writer / history-writer / realtime-api 的 Kafka consumer group 无 lag。
|
||||
- 确认真实 808 车辆 registration、TDengine RAW/里程点、Redis 实时查询均在持续更新。
|
||||
- 清理之前生产合成回放产生的 JT808 当日指标测试数据。
|
||||
|
||||
Kafka topic / consumer group:
|
||||
|
||||
```text
|
||||
vehicle.raw.go.jt808.v1 有持续 offset
|
||||
vehicle.raw.go.gb32960.v1 有持续 offset
|
||||
vehicle.event.go.unified.v1 有持续 offset
|
||||
|
||||
go-stat-writer LAG=0
|
||||
go-history-writer LAG=0
|
||||
go-realtime-api LAG=0
|
||||
```
|
||||
|
||||
TDengine:
|
||||
|
||||
```text
|
||||
raw_frames:
|
||||
JT808 2922
|
||||
GB32960 27
|
||||
|
||||
vehicle_mileage_points:
|
||||
JT808 2050
|
||||
GB32960 2
|
||||
```
|
||||
|
||||
Redis 实时查询样例:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
vin=LKLG7C4E3NA774736
|
||||
source_endpoint=222.66.200.68:29646
|
||||
plate=沪A06788F
|
||||
longitude=119.556866
|
||||
latitude=29.047911
|
||||
total_mileage_km=12345.6
|
||||
```
|
||||
|
||||
说明:该 Redis merged 快照保留了此前有效 `total_mileage_km=12345.6`,后续真实 808 位置帧上报的 `total_mileage_km=0` 不会覆盖该字段。TDengine 中该 VIN 后续真实里程点的 `total_mileage_km` 均为 0,因此 stat-writer 按“只统计正总里程”的规则不再写 JT808 每日指标。
|
||||
|
||||
MySQL 指标清理:
|
||||
|
||||
```text
|
||||
删除前:
|
||||
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_mileage_km | first=0 | latest=0 | sample_count=7
|
||||
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_total_mileage_km | first=0 | latest=0 | sample_count=7
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
|
||||
|
||||
删除后等待真实流量窗口:
|
||||
JT808 2026-07-01 metric rows = 0
|
||||
```
|
||||
|
||||
清理原因:
|
||||
|
||||
- `LKLG7C4E3NA774736` 的 12345.6 来自此前生产连通性合成回放,不是真实平台上报。
|
||||
- `LKLG7C4E2NA774775` 的 0 指标来自旧逻辑污染,当前 stat-writer 已跳过非正总里程。
|
||||
|
||||
Kafka spool:
|
||||
|
||||
```text
|
||||
23:24 spool_count=904
|
||||
23:25 spool_count=862
|
||||
23:28 spool_count=722
|
||||
new_spool_1min=0
|
||||
```
|
||||
|
||||
说明:spool 正在回放下降,且最近 1 分钟没有新增文件;生产 gateway 日志未见 `Incorrect string value`、Kafka publish error 或 replay error。
|
||||
|
||||
## 2026-07-01 23:34 每日指标查询 API 复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`56f4811`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-56f4811-20260701233311`
|
||||
|
||||
新增接口:
|
||||
|
||||
```text
|
||||
GET /api/stats/daily-metrics
|
||||
```
|
||||
|
||||
查询参数:
|
||||
|
||||
```text
|
||||
vin 可选
|
||||
protocol 可选,GB32960 / JT808 / YUTONG_MQTT
|
||||
metricKey 可选,daily_mileage_km / daily_total_mileage_km
|
||||
dateFrom 可选,YYYY-MM-DD
|
||||
dateTo 可选,YYYY-MM-DD
|
||||
limit 可选,默认 50,最大 1000
|
||||
offset 可选,默认 0
|
||||
```
|
||||
|
||||
生产可访问样例:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01&limit=10
|
||||
```
|
||||
|
||||
返回确认:
|
||||
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"vin": "LB9A32A21R0LS1707",
|
||||
"stat_date": "2020-07-01",
|
||||
"protocol": "GB32960",
|
||||
"metric_key": "daily_mileage_km",
|
||||
"metric_value": 0,
|
||||
"metric_unit": "km",
|
||||
"first_total_mileage_km": 53490.9,
|
||||
"latest_total_mileage_km": 53490.9,
|
||||
"sample_count": 3,
|
||||
"calculation_method": "TOTAL_MILEAGE_DIFF",
|
||||
"created_at": "2026-07-01 22:14:13",
|
||||
"updated_at": "2026-07-01 22:28:25"
|
||||
},
|
||||
{
|
||||
"vin": "LB9A32A21R0LS1707",
|
||||
"stat_date": "2020-07-01",
|
||||
"protocol": "GB32960",
|
||||
"metric_key": "daily_total_mileage_km",
|
||||
"metric_value": 53490.9,
|
||||
"metric_unit": "km",
|
||||
"first_total_mileage_km": 53490.9,
|
||||
"latest_total_mileage_km": 53490.9,
|
||||
"sample_count": 3,
|
||||
"calculation_method": "TOTAL_MILEAGE_DIFF",
|
||||
"created_at": "2026-07-01 22:14:13",
|
||||
"updated_at": "2026-07-01 22:28:25"
|
||||
}
|
||||
],
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
"total": 2
|
||||
}
|
||||
```
|
||||
|
||||
JT808 真实生产查询:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/stats/daily-metrics?protocol=JT808&dateFrom=2026-07-01&dateTo=2026-07-01&limit=10
|
||||
```
|
||||
|
||||
返回 `total=0`。原因是此前合成回放产生的测试指标已清理,当前真实 808 已绑定 VIN 的车辆仍上报 `total_mileage_km=0`,stat-writer 按规则不写非正总里程指标。
|
||||
|
||||
实时接口回归:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
```
|
||||
|
||||
返回 `online=true`,说明新增 stats API 没有影响 Redis 实时查询。
|
||||
|
||||
发布后 spool 观察:
|
||||
|
||||
```text
|
||||
发布后立即:spool_count=1101,new_spool_1min=102
|
||||
等待 60 秒:spool_count=1033,new_spool_1min=0
|
||||
```
|
||||
|
||||
说明:本次发布重启窗口产生的 spool 正在回放下降,且最近 1 分钟没有新增;gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。
|
||||
|
||||
## 2026-07-01 23:50 生产端口接管与 RAW 查询 API 复验
|
||||
|
||||
部署版本:
|
||||
|
||||
- Git commit:`5937132`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5937132-20260701234220`
|
||||
|
||||
生产端口状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway 0.0.0.0:808->808/tcp
|
||||
go-vehicle-gateway 0.0.0.0:32960->32960/tcp
|
||||
```
|
||||
|
||||
旧 Java 容器状态:
|
||||
|
||||
```text
|
||||
gb32960-ingest-app restart=no status=exited
|
||||
jt808-ingest-app restart=no status=exited
|
||||
yutong-mqtt-app restart=no status=exited
|
||||
vehicle-history-app restart=no status=exited
|
||||
vehicle-analytics-app restart=no status=exited
|
||||
vehicle-state-app restart=no status=exited
|
||||
telemetry-field-parser-app restart=no status=exited
|
||||
```
|
||||
|
||||
生产连接确认:
|
||||
|
||||
- JT808 端口 `808` 已收到外部连接:`222.66.200.68`、`115.231.168.135`。
|
||||
- GB32960 端口 `32960` 已收到外部连接:`8.134.95.166`、`117.160.0.65`。
|
||||
- ECS 当前没有 `8089` 监听。
|
||||
|
||||
RAW 查询 API:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames
|
||||
```
|
||||
|
||||
生产可访问样例:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
|
||||
http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
|
||||
```
|
||||
|
||||
接口复验结果:
|
||||
|
||||
```text
|
||||
JT808_RAW:
|
||||
total=1
|
||||
protocol=JT808
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
message_id=512
|
||||
message_id_hex=0x0200
|
||||
parse_status=OK
|
||||
raw_hex_len=94
|
||||
parsed_json_len=651
|
||||
fields_json_len=186
|
||||
|
||||
GB32960_RAW:
|
||||
total=1
|
||||
protocol=GB32960
|
||||
vin=LB9A32A21R0LS1707
|
||||
message_id=2
|
||||
message_id_hex=0x0002
|
||||
parse_status=OK
|
||||
raw_hex_len=1548
|
||||
parsed_json_len=3698
|
||||
fields_json_len=563
|
||||
```
|
||||
|
||||
同轮回归验证:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
online=true
|
||||
protocols=["JT808"]
|
||||
|
||||
GET /api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2026-07-01&limit=1
|
||||
total=1
|
||||
protocol=GB32960
|
||||
metric_key=daily_mileage_km
|
||||
calculation_method=TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- RAW 查询已返回 `raw_hex`、`parsed_json`、`fields_json`,可以用于查看完整解析结果。
|
||||
- 日统计和实时查询在生产端口接管后仍可用。
|
||||
|
||||
## 2026-07-02 00:00 原生 systemd 部署与宇通 MQTT 复验
|
||||
|
||||
根据本次 goal 的新要求,Go 接入链路后续不再使用 Docker 部署。本轮已中断 Docker 镜像构建路径,改为 ECS 原生 Linux 二进制 + systemd。
|
||||
|
||||
部署版本:
|
||||
|
||||
- Git commit:`594ab2d`
|
||||
- 部署目录:`/opt/lingniu-go-native`
|
||||
- 当前 release:`/opt/lingniu-go-native/releases/594ab2d`
|
||||
- 当前指针:`/opt/lingniu-go-native/current -> /opt/lingniu-go-native/releases/594ab2d`
|
||||
|
||||
systemd 服务:
|
||||
|
||||
```text
|
||||
lingniu-go-gateway.service active
|
||||
lingniu-go-history-writer.service active
|
||||
lingniu-go-stat-writer.service active
|
||||
lingniu-go-realtime-api.service active
|
||||
```
|
||||
|
||||
端口确认:
|
||||
|
||||
```text
|
||||
[::]:32960 gateway
|
||||
[::]:808 gateway
|
||||
[::]:20210 realtime-api
|
||||
```
|
||||
|
||||
Go Docker 容器已停止:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway Exited
|
||||
go-history-writer Exited
|
||||
go-stat-writer Exited
|
||||
go-realtime-api Exited
|
||||
```
|
||||
|
||||
宇通 MQTT:
|
||||
|
||||
```text
|
||||
yutong mqtt client started
|
||||
mqtt subscribed broker=ssl://cpxlm.axxc.cn:38883 topic=/ytforward/shln/+ qos=1
|
||||
```
|
||||
|
||||
说明:MQTT broker 存在短连接 `EOF` 后自动重连现象,但订阅后已收到真实数据并落 RAW。
|
||||
|
||||
RAW 查询复验:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=3
|
||||
|
||||
total=3
|
||||
protocol=YUTONG_MQTT
|
||||
vin=LMRKH9AC1R1004131
|
||||
vehicle_key=LMRKH9AC1R1004131
|
||||
parse_status=OK
|
||||
parsed_json_len=1104
|
||||
fields_json_len=116
|
||||
```
|
||||
|
||||
同轮回归:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
|
||||
total=1
|
||||
parse_status=OK
|
||||
|
||||
GET /api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
|
||||
total=1
|
||||
parse_status=OK
|
||||
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
online=true
|
||||
protocols=["JT808"]
|
||||
```
|
||||
|
||||
本轮落地文件:
|
||||
|
||||
- `deploy/systemd/README.md`
|
||||
- `deploy/systemd/lingniu-go-gateway.service`
|
||||
- `deploy/systemd/lingniu-go-history-writer.service`
|
||||
- `deploy/systemd/lingniu-go-stat-writer.service`
|
||||
- `deploy/systemd/lingniu-go-realtime-api.service`
|
||||
|
||||
## 2026-07-02 00:04 宇通 MQTT EOF 诊断
|
||||
|
||||
现象:
|
||||
|
||||
Go 原生部署后,`lingniu-go-gateway` 日志中出现多次:
|
||||
|
||||
```text
|
||||
mqtt connection lost error=EOF
|
||||
mqtt connection lost error=write: broken pipe
|
||||
mqtt subscribed topic=/ytforward/shln/+ qos=1
|
||||
```
|
||||
|
||||
10 分钟窗口统计:
|
||||
|
||||
```text
|
||||
mqtt subscribed = 33
|
||||
mqtt connection lost = 32
|
||||
```
|
||||
|
||||
对照旧 Java `yutong-mqtt-app` 历史日志,停止前也存在同类行为:
|
||||
|
||||
```text
|
||||
mqtt endpoint [yutong] received topic=/ytforward/shln/1 bytes=574
|
||||
mqtt endpoint [yutong] connection lost
|
||||
Caused by: java.io.EOFException
|
||||
mqtt endpoint [yutong] re-subscribed topic=/ytforward/shln/+ qos=1
|
||||
mqtt endpoint [yutong] received topic=/ytforward/shln/3 bytes=586
|
||||
```
|
||||
|
||||
判断:
|
||||
|
||||
- EOF/短连接不是 Go TLS 支持或 systemd 原生部署新引入的问题。
|
||||
- 旧 Java 和新 Go 都表现为:订阅、收到数据、broker 断开、自动重连。
|
||||
- 当前链路可持续收到宇通 MQTT 数据并写入 TDengine RAW。
|
||||
|
||||
复验:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=1
|
||||
|
||||
total=1
|
||||
protocol=YUTONG_MQTT
|
||||
vin=LMRKH9ACXR1004094
|
||||
parse_status=OK
|
||||
parsed_json_len=1080
|
||||
fields_json_len=115
|
||||
```
|
||||
|
||||
运维建议:
|
||||
|
||||
- 单条 EOF 不作为不可用判断。
|
||||
- 以 `YUTONG_MQTT` RAW 入库增长、最近成功 `mqtt subscribed` 时间、服务 `systemctl is-active` 作为可用性判断。
|
||||
- 如果 EOF 频率继续升高且 RAW 不再增长,再联系宇通侧确认 broker 长连接策略、同一 clientId 并发限制或网络出口策略。
|
||||
158
docs/operations/go-vehicle-gateway-verification.md
Normal file
158
docs/operations/go-vehicle-gateway-verification.md
Normal file
@@ -0,0 +1,158 @@
|
||||
# Go Vehicle Gateway 验证手册
|
||||
|
||||
本文档用于验证 Go 重构运行面:
|
||||
|
||||
- `go-vehicle-gateway`:GB32960 TCP、JT808 TCP、宇通 MQTT 接入。
|
||||
- `go-history-writer`:Kafka RAW 写 TDengine。
|
||||
- `go-stat-writer`:Kafka RAW 写 MySQL 每日指标。
|
||||
- `go-realtime-api`:Kafka unified event 写 Redis,并提供实时查询 API。
|
||||
|
||||
## 本地构建验证
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./...
|
||||
go build ./cmd/gateway
|
||||
go build ./cmd/history-writer
|
||||
go build ./cmd/stat-writer
|
||||
go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
## 生产原生部署验证
|
||||
|
||||
```bash
|
||||
python3 tools/go_prod_acceptance.py --date 2026-07-02
|
||||
```
|
||||
|
||||
该命令会聚合执行:
|
||||
|
||||
- `go_systemd_prod_smoke.py`:应用 ECS systemd 服务 active/enabled、端口归属和 gateway spool
|
||||
- `go_kafka_prod_smoke.py`:Kafka topic 和消费组 lag
|
||||
- `go_native_prod_smoke.py`:HTTP API、RAW `parsed_json`、历史核心表 `frame_id` 回链、每日指标公式、Redis 实时新鲜度、TDengine/MySQL/Redis 数据链路
|
||||
|
||||
如需单独检查 systemd:
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
```
|
||||
|
||||
脚本通过 SSH 检查:
|
||||
|
||||
- `lingniu-go-gateway.service`
|
||||
- `lingniu-go-history-writer.service`
|
||||
- `lingniu-go-stat-writer.service`
|
||||
- `lingniu-go-realtime-api.service`
|
||||
- 上述服务均为 `active` 且 `enabled`
|
||||
- `808`、`32960` 由 `gateway` 监听
|
||||
- `20210` 由 `realtime-api` 监听
|
||||
- `/opt/lingniu-go-native/spool/gateway` 无待回放文件
|
||||
|
||||
## 生产环境变量
|
||||
|
||||
当前 goal 的生产面使用 ECS 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。以下环境变量维护在 `/opt/lingniu-go-native/env/*.env`:
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=172.17.111.56:9092
|
||||
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
|
||||
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
|
||||
IDENTITY_MYSQL_DSN=${MYSQL_DSN}
|
||||
VEHICLE_IDENTITY_TABLE=vehicle_identity_binding
|
||||
REDIS_ADDR=r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379
|
||||
REDIS_PASSWORD=<password>
|
||||
REDIS_DB=50
|
||||
```
|
||||
|
||||
宇通 MQTT 开启时再配置:
|
||||
|
||||
```bash
|
||||
YUTONG_MQTT_ENABLED=true
|
||||
YUTONG_MQTT_URI=<mqtt-uri>
|
||||
YUTONG_MQTT_TOPICS=/ytforward/shln/+
|
||||
YUTONG_MQTT_CLIENT_ID=lingniu-go-yutong-mqtt
|
||||
YUTONG_MQTT_USERNAME=<username>
|
||||
YUTONG_MQTT_PASSWORD=<password>
|
||||
```
|
||||
|
||||
## ECS 进程检查
|
||||
|
||||
```bash
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
```
|
||||
|
||||
## Kafka 验证
|
||||
|
||||
优先使用 smoke 脚本做可重复检查:
|
||||
|
||||
```bash
|
||||
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
|
||||
```
|
||||
|
||||
脚本会检查 `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1`、`vehicle.event.go.unified.v1` 是否存在,并汇总 `go-history-writer`、`go-stat-writer`、`go-realtime-api` 的消费 lag。运行环境需要 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
手工核对命令:
|
||||
|
||||
```bash
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.jt808.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.gb32960.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.yutong-mqtt.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.event.go.unified.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-history-writer
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-stat-writer
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-realtime-api
|
||||
```
|
||||
|
||||
## TDengine 验证
|
||||
|
||||
```sql
|
||||
USE lingniu_vehicle_ts;
|
||||
SHOW STABLES;
|
||||
SELECT COUNT(*) FROM raw_frames;
|
||||
SELECT COUNT(*) FROM vehicle_locations;
|
||||
SELECT COUNT(*) FROM vehicle_mileage_points;
|
||||
```
|
||||
|
||||
## MySQL 验证
|
||||
|
||||
```sql
|
||||
SELECT protocol, metric_key, COUNT(*)
|
||||
FROM vehicle_daily_metric
|
||||
GROUP BY protocol, metric_key;
|
||||
|
||||
SELECT *
|
||||
FROM vehicle_daily_metric
|
||||
WHERE metric_key IN ('daily_mileage_km', 'daily_total_mileage_km')
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
## Redis 实时查询
|
||||
|
||||
```bash
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>'
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/online'
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/protocols/JT808'
|
||||
```
|
||||
|
||||
## 验收证据
|
||||
|
||||
正式切换前至少记录:
|
||||
|
||||
- 一个真实 GB32960 VIN 的 RAW、location、mileage point、Redis snapshot。
|
||||
- 一个真实 JT808 VIN/phone 的 RAW、location、mileage point、daily metric、Redis snapshot。
|
||||
- 一个真实宇通 MQTT VIN 或 device_id 的 RAW、Redis snapshot。
|
||||
- `go-history-writer` 和 `go-stat-writer` 重启后 Kafka offset 能继续推进。
|
||||
@@ -4,18 +4,20 @@
|
||||
|
||||
The analytics app can calculate daily mileage from JT808 telemetry only. It consumes `vehicle.event.jt808.v1` and saves the current daily result into the common `vehicle-stat` metric repository.
|
||||
|
||||
Only supported message backbone: Kafka.
|
||||
|
||||
## Storage
|
||||
|
||||
There is no separate JT808 daily-mileage table and no Redis mileage state. The derived value is stored as one `daily_mileage_km` metric row in the common JDBC/MySQL table `vehicle_stat_metric`; the first and latest GPS total mileage values are kept on that same row as calculation source columns.
|
||||
Runtime state: none outside `vehicle_stat_metric`. There is no separate JT808 daily-mileage table. The derived value is stored as one `daily_mileage_km` metric row in the common JDBC/MySQL table `vehicle_stat_metric`; the local-day minimum and maximum GPS total mileage values are kept on that same row as calculation source columns.
|
||||
|
||||
The JT808 daily-mileage value is calculated from the GPS total mileage reported in location additional information:
|
||||
|
||||
```text
|
||||
daily_mileage_km = last_total_mileage_km - first_total_mileage_km
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
calculation_method = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
The first valid JT808 location point for a vehicle and local day stores the day start total on the metric row and writes `daily_mileage_km=0.0`. Later ordered points update the latest total on the same row and rewrite the derived `daily_mileage_km`. If a later total mileage is lower than the stored latest total, the previous latest total remains.
|
||||
The first valid JT808 location point for a vehicle and local day stores both calculation endpoints on the metric row and writes `daily_mileage_km=0.0`. Later ordered or replayed points update the lower endpoint when the reported GPS total mileage is smaller, update the upper endpoint when it is larger, and rewrite the derived `daily_mileage_km` on the same row.
|
||||
|
||||
## Runtime Settings
|
||||
|
||||
@@ -29,4 +31,4 @@ MYSQL_USERNAME=<user>
|
||||
MYSQL_PASSWORD=<password>
|
||||
```
|
||||
|
||||
Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `daily_mileage_km` metric row, so no separate mileage state store is required.
|
||||
Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `daily_mileage_km` metric row.
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# Vehicle Ingest TDengine 链路验收手册
|
||||
|
||||
这份手册用于验收当前 32960/JT808 接入重构链路:
|
||||
这份手册用于验收当前 GB32960/JT808/Yutong MQTT 接入重构链路:
|
||||
|
||||
1. 协议 App 接收 TCP 报文。
|
||||
2. 协议 App 归档原始帧元数据,并把事件信封发布到 Kafka。
|
||||
3. `vehicle-history-app` 消费 Kafka。
|
||||
4. history 写入 raw、location 到 TDengine;按需开启 telemetry field 逐字段宽表。
|
||||
4. history 写入 raw、location 到 TDengine;不写 telemetry_fields。
|
||||
5. Swagger/API 通过 TDengine 做历史分页查询。
|
||||
|
||||
每一步都必须执行命令并检查输出后,才能标记为已验证。
|
||||
@@ -24,102 +24,72 @@ export TDENGINE_USERNAME=root
|
||||
export TDENGINE_PASSWORD='<tdengine-password>'
|
||||
export TDENGINE_MIN_IDLE=0
|
||||
export TDENGINE_MAX_POOL_SIZE=32
|
||||
export EVENT_FILE_STORE_ENABLED=false
|
||||
export TDENGINE_TELEMETRY_FIELDS_ENABLED=false
|
||||
```
|
||||
|
||||
`TDENGINE_MIN_IDLE=0` 用于减少冷启动时 TDengine 连接重试日志。TDengine 地址稳定后,再按生产并发调大连接池。
|
||||
|
||||
高吞吐生产验收默认关闭 `EVENT_FILE_STORE_ENABLED` 和 `TDENGINE_TELEMETRY_FIELDS_ENABLED`,验证 TDengine `raw_frames`、位置表和分页查询闭环。RAW 帧的 `parsedJson`、`metadataJson`、`rawUri` 直接进入 TDengine,GB32960 snapshot/fields 接口优先基于 `raw_frames` 中的结构化 JSON 返回结果;接入服务写出的原始 `.bin` 只作为冷备复核材料。需要兼容旧 `/api/event-history/records` 低层记录接口或逐字段趋势宽表时,再显式开启对应开关,并单独压测写入放大。
|
||||
高吞吐生产验收只验证 TDengine `raw_frames`、位置表和分页查询闭环。RAW 帧的 `parsedJson`、`metadataJson`、`rawUri` 直接进入 TDengine,GB32960 snapshot/fields 接口基于 `raw_frames` 与当前解析器即时返回结构化结果;接入服务写出的原始 `.bin` 只作为冷备复核材料。逐字段趋势宽表由独立字段解析服务消费 Kafka RAW/事件后维护,不由 history-app 写入。
|
||||
|
||||
## 构建
|
||||
|
||||
```bash
|
||||
mvn -pl modules/apps/jt808-ingest-app,modules/apps/gb32960-ingest-app,modules/apps/vehicle-history-app -am package -DskipTests
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
预期产物:
|
||||
|
||||
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
|
||||
- `modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar`
|
||||
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
|
||||
- `modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar`
|
||||
- `modules/apps/vehicle-history-app/target/vehicle-history-app.jar`
|
||||
- `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar`
|
||||
|
||||
## 启动 JT808 接入服务,监听 808 端口
|
||||
## ECS 服务状态验证
|
||||
|
||||
生产服务只在 ECS 上运行,通过 Portainer/Docker 管理。不要在本机用 `java -jar`、plist 或其它守护进程方式启动 GB32960、JT808、Yutong MQTT、history、analytics。
|
||||
|
||||
```bash
|
||||
HTTP_PORT=20482 \
|
||||
JT808_PORT=808 \
|
||||
KAFKA_CONSUMER_ENABLED=false \
|
||||
java -jar modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar
|
||||
```
|
||||
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' \
|
||||
| egrep 'gb32960|jt808|yutong|vehicle-history|vehicle-analytics|NAMES'
|
||||
|
||||
验证:
|
||||
ss -lntp | egrep ':(808|32960|20100|20200|20310|20400|20500)\b'
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20482/actuator/health
|
||||
lsof -nP -iTCP:808 -sTCP:LISTEN
|
||||
```
|
||||
|
||||
预期:健康检查为 `UP`,并且 Java 进程监听 TCP `808`。
|
||||
|
||||
## 启动 GB32960 接入服务
|
||||
|
||||
```bash
|
||||
HTTP_PORT=20100 \
|
||||
GB32960_PORT=32960 \
|
||||
KAFKA_CONSUMER_ENABLED=false \
|
||||
java -jar modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar
|
||||
```
|
||||
|
||||
验证:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
lsof -nP -iTCP:32960 -sTCP:LISTEN
|
||||
```
|
||||
|
||||
## 启动 TDengine 版历史服务
|
||||
|
||||
```bash
|
||||
HTTP_PORT=20200 \
|
||||
KAFKA_CONSUMER_ENABLED=true \
|
||||
EVENT_FILE_STORE_ENABLED=false \
|
||||
EVENT_FILE_STORE_PATH=./target/tdengine-verification/event-store \
|
||||
TDENGINE_TELEMETRY_FIELDS_ENABLED=false \
|
||||
java -jar modules/apps/vehicle-history-app/target/vehicle-history-app.jar
|
||||
```
|
||||
|
||||
验证:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/v3/api-docs \
|
||||
| grep -E '/api/event-history/jt808/locations|/api/event-history/gb32960/snapshots|/api/event-history/telemetry/fields'
|
||||
| grep -E '/api/event-history/locations|/api/event-history/raw-frames'
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
预期:健康检查为 `UP`,OpenAPI 中包含 JT808 TDengine 分页查询接口和 GB32960 snapshot 接口。`/api/event-history/telemetry/fields` 会存在,但只有在 `TDENGINE_TELEMETRY_FIELDS_ENABLED=true` 后才会持续写入新字段数据。
|
||||
预期:五个容器均为运行状态;健康检查为 `UP`;JT808/GB32960 TCP 端口在 ECS 上监听;OpenAPI 中包含通用位置分页查询和 RAW 帧查询接口。history-app 不暴露 `/api/event-history/telemetry/fields`,也不持续写入逐字段宽表。JT808 位置帧中有 GPS 总里程时,`vehicle-analytics-app` 按差值法写入 MySQL `vehicle_stat_metric` 的 `daily_mileage_km` 指标。
|
||||
|
||||
## 本机 launchctl 部署
|
||||
## 运行位置约束
|
||||
|
||||
当前本机生产验证建议使用仓库模板生成 plist:
|
||||
生产身份绑定固定使用 MySQL,不再提供 file/memory/sqlite 运行时切换入口。生产服务固定运行在 ECS/Portainer,不提供本机守护进程模板。
|
||||
|
||||
```bash
|
||||
deploy/local/launchctl/
|
||||
```
|
||||
|
||||
关键约束:
|
||||
|
||||
- `jt808-ingest-app` 监听 TCP `808`,HTTP `20482`。
|
||||
- `jt808-ingest-app` 监听 TCP `808`,HTTP `20400`。
|
||||
- `gb32960-ingest-app` 监听 TCP `32960`,HTTP `20100`。
|
||||
- `yutong-mqtt-app` 监听 HTTP `20500`。
|
||||
- `vehicle-history-app` 监听 HTTP `20200`。
|
||||
- history 热查询只依赖 Kafka 和 TDengine `raw_frames`;本地模板不会给 history 注入 `SINK_ARCHIVE_PATH`。
|
||||
- GB32960/JT808 接入服务可以继续使用 `SINK_ARCHIVE_PATH` 保存原始 `.bin` 冷备,但这不是 history API 的实时查询前置条件。
|
||||
- 替换 jar 前先 `launchctl bootout`,再复制 jar 和 `launchctl bootstrap`,避免 KeepAlive 在 jar 拷贝中途重启。
|
||||
- `vehicle-analytics-app` 监听 HTTP `20310`。
|
||||
- history 热查询只依赖 Kafka 和 TDengine `raw_frames`;history 不需要共享 archive volume。
|
||||
- GB32960/JT808/Yutong MQTT 接入服务可以继续使用 `SINK_ARCHIVE_PATH` 保存原始冷备,但这不是 history API 的实时查询前置条件。
|
||||
|
||||
JT808 注册身份绑定:
|
||||
|
||||
- 生产默认使用 `VEHICLE_IDENTITY_STORE=mysql`,不再使用 file/memory 作为运行时身份绑定仓储。
|
||||
- 生产需要把 0x0100 注册信息维护到 MySQL 时,设置 `VEHICLE_IDENTITY_STORE=mysql`。
|
||||
- 生产需要把 0x0100 注册信息维护到 MySQL 时,只需要配置 `VEHICLE_IDENTITY_MYSQL_*` 连接参数。
|
||||
- 需要提供 `VEHICLE_IDENTITY_MYSQL_JDBC_URL`、`VEHICLE_IDENTITY_MYSQL_USERNAME`、`VEHICLE_IDENTITY_MYSQL_PASSWORD`。
|
||||
- 服务启动时自动创建 `vehicle_identity_binding` 和 `vehicle_identity_binding_registration` 两张表。`vehicle_identity_binding` 只维护 `plate`、`vin` 两列;`vehicle_identity_binding_registration` 按 `protocol + phone` 保存 JT808 0x0100 注册字段。
|
||||
- 注册帧无真实 VIN 时也会写入 registration 表,`vin` 默认为 `unknown`。外部系统识别车辆后只需要维护车牌和 VIN,例如:`INSERT INTO vehicle_identity_binding (plate, vin) VALUES ('沪A61559F', 'LNVIN000000000001') ON DUPLICATE KEY UPDATE vin = VALUES(vin);`
|
||||
@@ -129,16 +99,30 @@ JT808 注册身份绑定:
|
||||
健康检查:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20482/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
Swagger:
|
||||
API / Swagger:
|
||||
|
||||
- JT808 ingest: `http://127.0.0.1:20482/swagger-ui/index.html`
|
||||
- GB32960 ingest: `http://127.0.0.1:20100/swagger-ui/index.html`
|
||||
- JT808 ingest health: `http://127.0.0.1:20400/actuator/health`
|
||||
- GB32960 ingest health: `http://127.0.0.1:20100/actuator/health`
|
||||
- Yutong MQTT ingest health: `http://127.0.0.1:20500/actuator/health`
|
||||
- History query: `http://127.0.0.1:20200/swagger-ui/index.html`
|
||||
- Analytics metrics: `http://127.0.0.1:20310/swagger-ui/index.html`
|
||||
|
||||
## Kafka Topic 验证
|
||||
|
||||
@@ -148,6 +132,8 @@ Swagger:
|
||||
- `vehicle.raw.gb32960.v1`
|
||||
- `vehicle.event.jt808.v1`
|
||||
- `vehicle.raw.jt808.v1`
|
||||
- `vehicle.event.mqtt-yutong.v1`
|
||||
- `vehicle.raw.mqtt-yutong.v1`
|
||||
|
||||
使用 Kafka CLI 或管理工具检查 topic 是否存在,并确认 history consumer group 在测试流量后没有持续堆积。默认 `KAFKA_GROUP_HISTORY=vehicle-history` 时,实际 group 会拆成:
|
||||
|
||||
@@ -155,10 +141,12 @@ Swagger:
|
||||
- `vehicle-history-gb32960-raw`
|
||||
- `vehicle-history-jt808-event`
|
||||
- `vehicle-history-jt808-raw`
|
||||
- `vehicle-history-yutong-mqtt-event`
|
||||
- `vehicle-history-yutong-mqtt-raw`
|
||||
|
||||
## JT808 实时转发验收
|
||||
|
||||
如果外部平台已经把 JT808 报文转发到本机 TCP `808`,至少观察 60 秒日志和 Kafka 数据。
|
||||
如果外部平台已经把 JT808 报文转发到 ECS TCP `808`,至少观察 60 秒日志和 Kafka 数据。
|
||||
|
||||
必须拿到以下证据:
|
||||
|
||||
@@ -186,22 +174,6 @@ curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin=
|
||||
|
||||
预期:snapshot 返回 `sourceFrames.rawArchiveUri` 和解析后的 `blocks`;字段投影返回所选字段。新增协议字段后,先通过 replay 或专用解析任务补写 `raw_frames.parsedJson`,再基于历史 RAW 结构化结果查询。
|
||||
|
||||
## Telemetry Field 查询验收
|
||||
|
||||
查询 GB32960 字段历史。该能力需要 `TDENGINE_TELEMETRY_FIELDS_ENABLED=true` 并确认压测后无持续 lag:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/telemetry/fields?protocol=GB32960&vin=<vin>&fieldKey=<field-key>&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
```
|
||||
|
||||
查询 JT808 字段历史。没有 VIN 映射时,优先使用 `phone`。该能力同样需要 `TDENGINE_TELEMETRY_FIELDS_ENABLED=true`:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/telemetry/fields?protocol=JT808&phone=<phone>&fieldKey=location.speed_kmh&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
```
|
||||
|
||||
预期:响应包含 `items`,可能包含 `nextCursor`。下一页使用 `nextCursor` 里的 `cursorTs` 和 `cursorId` 查询。
|
||||
|
||||
## TDengine 直接检查
|
||||
|
||||
使用 TDengine CLI 或 JDBC 客户端检查超级表和子表是否创建:
|
||||
@@ -212,14 +184,12 @@ SHOW STABLES;
|
||||
SELECT COUNT(*) FROM raw_frames;
|
||||
SELECT COUNT(*) FROM vehicle_locations;
|
||||
SELECT COUNT(*) FROM jt808_locations;
|
||||
-- 仅在 TDENGINE_TELEMETRY_FIELDS_ENABLED=true 时检查:
|
||||
SELECT COUNT(*) FROM telemetry_fields;
|
||||
```
|
||||
|
||||
预期:
|
||||
|
||||
- `raw_frames`、`vehicle_locations`、`jt808_locations` 存在;有对应协议实时流量后,计数持续增长。
|
||||
- `telemetry_fields` 表结构会创建;默认高吞吐模式不会持续增长,除非显式开启 `TDENGINE_TELEMETRY_FIELDS_ENABLED=true`。
|
||||
- 字段趋势宽表不属于 history-app 验收范围,由独立字段解析服务负责。
|
||||
- 有实时流量后,计数持续增加。
|
||||
|
||||
## 失败语义
|
||||
@@ -228,15 +198,17 @@ SELECT COUNT(*) FROM telemetry_fields;
|
||||
- TDengine 未启用时,TDengine 专属接口不应该出现在 OpenAPI 中。
|
||||
- 路径不存在时,API 层应该返回 HTTP `404`,而不是泛化的存储失败。
|
||||
|
||||
## 2026-06-29 本机实时链路验证结果
|
||||
## 2026-06-29 Superseded Local Live Verification Record
|
||||
|
||||
运行服务:
|
||||
该记录仅保留为解析和存储行为的历史证据;生产运行位置已经收敛到 ECS/Portainer。
|
||||
|
||||
- `com.lingniu.jt808.808`: `jt808-ingest-app`,TCP `808`,HTTP `20482`
|
||||
- `com.lingniu.gb32960.32960`: `gb32960-ingest-app`,TCP `32960`,HTTP `20100`
|
||||
- `com.lingniu.vehicle-history.jt808`: `vehicle-history-app`,HTTP `20200`
|
||||
当时参与验证的服务:
|
||||
|
||||
健康检查已确认三个服务均返回 `{"status":"UP"}`。
|
||||
- `jt808-ingest-app`:TCP `808`,HTTP `20400`
|
||||
- `gb32960-ingest-app`:TCP `32960`,HTTP `20100`
|
||||
- `vehicle-history-app`:HTTP `20200`
|
||||
|
||||
当次健康检查均返回 `{"status":"UP"}`。
|
||||
|
||||
可重复运行 live 验收工具:
|
||||
|
||||
@@ -256,16 +228,16 @@ python3 tools/vehicle_ingest_live_verify.py \
|
||||
|
||||
JT808 真实转发链路已验证:
|
||||
|
||||
- 本机 TCP `808` 有外部平台 `222.66.200.68` 持续发送 JT808 报文。
|
||||
- ECS TCP `808` 有外部平台 `222.66.200.68` 持续发送 JT808 报文。
|
||||
- `vehicle.raw.jt808.v1` 和 `vehicle.event.jt808.v1` 被 history 消费。
|
||||
- `raw_frames` 中 `protocol='JT808'` 的真实行数已超过 `36000`。
|
||||
- 外部平台真实 0x0100 注册帧已超过 `347` 条,真实 0x0200 位置帧已超过 `12400` 条。
|
||||
- `jt808_locations` 中位置行数已超过 `8783`。
|
||||
- 在旧的字段宽表开启模式下,`telemetry_fields` 中 `protocol='JT808'` 的字段行数曾超过 `15000`;当前高吞吐默认不要求该表持续增长。
|
||||
- 当前高吞吐默认链路不维护 `telemetry_fields`;JT808 明细验证以 `raw_frames`、`vehicle_locations` 和 API 分页结果为准。
|
||||
- 分页 API 已用终端号 `13079963301` 验证第一页和 nextCursor 第二页,样例位置包含 `longitude=118.913846`、`latitude=31.927309`、`statusFlag=3` 和 `archive://2026/06/29/JT808/...` 原始帧引用。
|
||||
- 注册帧样例:终端号 `13079963320`、deviceId `9963320`、deviceType `SEG-9888G`、plate `沪A61559F`、maker `70112`、province `31`、city `113`、plateColor `2`。生产 VIN 反写依赖 MySQL identity store,后续 raw/event 的 VIN 解析也从 MySQL 绑定读取。
|
||||
|
||||
## JT808 本机 smoke / 轻压测工具
|
||||
## JT808 ECS smoke / 轻压测工具
|
||||
|
||||
仓库提供 `tools/jt808_e2e_smoke.py`,用于重复验证 TCP `808` 到 TDengine history 的闭环:
|
||||
|
||||
@@ -295,8 +267,6 @@ python3 tools/jt808_e2e_smoke.py \
|
||||
USE vehicle_ts;
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM jt808_locations WHERE protocol = 'JT808';
|
||||
-- 仅在 TDENGINE_TELEMETRY_FIELDS_ENABLED=true 时复查:
|
||||
SELECT COUNT(*) FROM telemetry_fields WHERE protocol = 'JT808';
|
||||
SELECT event_time, vehicle_key, phone, message_id, raw_uri
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'JT808'
|
||||
@@ -325,7 +295,7 @@ GB32960 链路已用正式平台登录和合成实时帧验证:
|
||||
- 当前正式平台尚未发送车辆实时上报 `0x02`;非测试 VIN 的 `message_id=2` 计数为 `0`,因此不能把平台登录误判为车辆数据在线。
|
||||
- TCP `32960` 返回 GB32960 `2323` ACK。
|
||||
- `raw_frames` 中测试 VIN `LTEST202606290001` 已有 `2` 条 raw 记录。
|
||||
- 在旧的字段宽表开启模式下,`telemetry_fields` 中测试 VIN `LTEST202606290001` 曾有 `34` 条字段记录;当前默认通过 GB32960 snapshot/fields 即时解析 raw,不要求该表增长。
|
||||
- 当前默认通过 GB32960 snapshot/fields 即时解析 `raw_frames` 中的 RAW JSON,不维护字段宽表增长。
|
||||
- latest raw 包含 `archive://2026/06/29/GB32960/LTEST202606290001/...`。
|
||||
- snapshots API 可查到 `speedKmh=62.4`、`totalMileageKm=223456.7`、`longitude=116.397128`、`latitude=39.916527`。
|
||||
|
||||
@@ -335,9 +305,6 @@ GB32960 链路已用正式平台登录和合成实时帧验证:
|
||||
USE vehicle_ts;
|
||||
SELECT COUNT(*) FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001';
|
||||
-- 仅在 TDENGINE_TELEMETRY_FIELDS_ENABLED=true 时复查:
|
||||
SELECT COUNT(*) FROM telemetry_fields
|
||||
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001';
|
||||
SELECT event_time, vin, raw_uri, frame_id
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001'
|
||||
@@ -377,7 +344,7 @@ curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin=
|
||||
|
||||
## 当前注意事项
|
||||
|
||||
- 当前本机使用 TDengine WebSocket JDBC,例如 `jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts`。
|
||||
- 当前生产链路使用 TDengine WebSocket JDBC,例如 `jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts`。
|
||||
- `KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest` 适合生产接入新流量;如果要回放历史 Kafka 数据,需要切换 consumer group 或重置 offset。
|
||||
- `vehicle-history-app` raw 和 event 使用不同 consumer binding;raw consumer 必须开启,否则 `raw_frames` 不会持续增长。
|
||||
- JT808 设备没有 VIN 映射时,`vehicle_key` 使用 `jt808:<phone>`;启用 MySQL identity store 后,registration 表中已反写 VIN 的 phone/deviceId/plate 会用于后续 raw/event 的 VIN 解析。
|
||||
|
||||
@@ -543,7 +543,7 @@ EOF
|
||||
Run: `mvn test -q`
|
||||
Expected: BUILD SUCCESS。重点关注:
|
||||
- `protocol-gb32960` 模块全绿(Golden、FullBlocks、Isolation、GuangdongFcEndToEnd、Decoder、Mapper、所有 parser 单测、profile)
|
||||
- 其它模块(`ingest-core`、`sink-mq` 等)不受影响(无改动应自动通过)
|
||||
- 其它模块(`ingest-core`、`sink-kafka` 等)不受影响(无改动应自动通过)
|
||||
|
||||
若有红,停下来诊断。
|
||||
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
> **Superseded:** This 2026-06-23 DuckDB hot-store plan is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine stores hot history, `sink-archive`
|
||||
> owns raw bytes, and `event-file-store` / DuckDB are no longer part of the
|
||||
> current build surface.
|
||||
|
||||
# GB32960 History DuckDB Hot Store Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
> **Superseded:** This 2026-06-23 service-split plan is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: GB32960, JT808, and Yutong MQTT are the
|
||||
> active ingest apps, history is backed by TDengine, and Xinda/event-file-store
|
||||
> paths have been removed from the current build surface.
|
||||
|
||||
# GB32960 Service Split Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
@@ -13,12 +20,14 @@
|
||||
## Current-State Notes
|
||||
|
||||
- Existing all-in-one entrypoint: `modules/apps/bootstrap-all/src/main/java/com/lingniu/ingest/bootstrap/IngestApplication.java`.
|
||||
- Existing all-in-one module: `modules/apps/bootstrap-all/pom.xml` imports protocols, sinks, history, state, stat, command gateway, MQTT, Xinda, and JT modules.
|
||||
- Existing all-in-one module at the time of this plan imported protocols,
|
||||
sinks, history, state, stat, command gateway, MQTT, and JT modules. Xinda was
|
||||
later removed from source and build configuration.
|
||||
- Existing GB32960 handler: `modules/protocols/protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/inbound/Gb32960ChannelHandler.java`.
|
||||
- Existing handler currently writes report ACK before `dispatcher.dispatch(rf)`.
|
||||
- Existing `DisruptorEventBus` publishes to sinks asynchronously and does not wait for sink completion.
|
||||
- Existing `KafkaEventSink.accepts()` rejects `VehicleEvent.RawArchive`.
|
||||
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.mq.topics`.
|
||||
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.kafka.topics`.
|
||||
- Existing history consumer logic: `modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java`.
|
||||
- Existing analytics consumers: `VehicleStateEnvelopeIngestor` and `VehicleStatEnvelopeIngestor`.
|
||||
- There is an unrelated working-tree change in `modules/apps/bootstrap-all/src/main/resources/application.yml` for GB32960 diagnostics capacity. Do not revert it. Do not include it in service-split commits unless the user explicitly asks.
|
||||
@@ -48,11 +57,11 @@ Create app modules:
|
||||
Modify shared build and Kafka contract files:
|
||||
|
||||
- `pom.xml`
|
||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
|
||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
|
||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
|
||||
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
|
||||
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java`
|
||||
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java`
|
||||
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java`
|
||||
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java`
|
||||
- `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java`
|
||||
- `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java`
|
||||
|
||||
Modify ACK/Kafka durability boundary files:
|
||||
|
||||
@@ -157,7 +166,7 @@ Create `modules/apps/gb32960-ingest-app/pom.xml`:
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -222,7 +231,7 @@ Create `modules/apps/vehicle-history-app/pom.xml`:
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -303,7 +312,7 @@ Create `modules/apps/vehicle-analytics-app/pom.xml`:
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -466,16 +475,19 @@ lingniu:
|
||||
rate-limit:
|
||||
per-vin-qps: 50
|
||||
session:
|
||||
store: ${SESSION_STORE:memory}
|
||||
store: ${SESSION_STORE:redis}
|
||||
ttl: ${SESSION_TTL:30m}
|
||||
identity:
|
||||
store: ${VEHICLE_IDENTITY_STORE:file}
|
||||
file:
|
||||
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
|
||||
store: ${VEHICLE_IDENTITY_STORE:mysql}
|
||||
mysql:
|
||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
|
||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||
sink:
|
||||
mq:
|
||||
kafka:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
compression-type: zstd
|
||||
linger-ms: 20
|
||||
@@ -568,9 +580,8 @@ lingniu:
|
||||
gb32960:
|
||||
enabled: false
|
||||
sink:
|
||||
mq:
|
||||
kafka:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
topics:
|
||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
@@ -666,9 +677,8 @@ lingniu:
|
||||
gb32960:
|
||||
enabled: false
|
||||
sink:
|
||||
mq:
|
||||
kafka:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
topics:
|
||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
@@ -754,7 +764,7 @@ import com.lingniu.ingest.eventfilestore.EventFileStore;
|
||||
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
@@ -770,7 +780,7 @@ class Gb32960IngestAppCompositionTest {
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.gb32960.enabled=true",
|
||||
"lingniu.ingest.gb32960.port=0",
|
||||
"lingniu.ingest.sink.mq.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=false",
|
||||
"lingniu.ingest.sink.archive.enabled=false",
|
||||
"lingniu.ingest.event-file-store.enabled=false",
|
||||
"lingniu.ingest.event-history.enabled=false",
|
||||
@@ -835,7 +845,7 @@ class VehicleHistoryAppCompositionTest {
|
||||
"lingniu.ingest.event-history.enabled=true",
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=false",
|
||||
"lingniu.ingest.sink.mq.enabled=false")
|
||||
"lingniu.ingest.sink.kafka.enabled=false")
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration.class,
|
||||
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration.class,
|
||||
@@ -896,7 +906,7 @@ class VehicleAnalyticsAppCompositionTest {
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=true",
|
||||
"lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"),
|
||||
"lingniu.ingest.sink.mq.enabled=false")
|
||||
"lingniu.ingest.sink.kafka.enabled=false")
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration.class,
|
||||
com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration.class))
|
||||
@@ -932,22 +942,22 @@ git commit -m "test: cover split service composition"
|
||||
### Task 4: Version Kafka Topics for GB32960 Raw/Event/DLQ
|
||||
|
||||
**Files:**
|
||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
|
||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
|
||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
|
||||
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
|
||||
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java`
|
||||
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java`
|
||||
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java`
|
||||
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java`
|
||||
- Create or modify: `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java`
|
||||
- Create or modify: `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java`
|
||||
|
||||
- [ ] **Step 1: Write topic routing tests**
|
||||
|
||||
Create `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java` if it does not exist. Add tests proving GB32960 raw and normalized events can route to versioned topics. Use the existing `VehicleEvent` constructors from `EventFileStoreSinkTest` and `EnvelopeMapperTelemetrySnapshotTest` as examples.
|
||||
Create `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java` if it does not exist. Add tests proving GB32960 raw and normalized events can route to versioned topics. Use the existing `VehicleEvent` constructors from `EventFileStoreSinkTest` and `EnvelopeMapperTelemetrySnapshotTest` as examples.
|
||||
|
||||
Expected test names:
|
||||
|
||||
```java
|
||||
@Test
|
||||
void rawArchiveRoutesToVersionedGb32960RawTopic() {
|
||||
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
|
||||
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
|
||||
topics.setRawArchive("vehicle.raw.gb32960.v1");
|
||||
TopicRouter router = new TopicRouter(topics);
|
||||
|
||||
@@ -970,7 +980,7 @@ Also add a normalized realtime event test:
|
||||
```java
|
||||
@Test
|
||||
void realtimeRoutesToVersionedGb32960EventTopic() {
|
||||
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
|
||||
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
|
||||
topics.setRealtime("vehicle.event.gb32960.v1");
|
||||
TopicRouter router = new TopicRouter(topics);
|
||||
|
||||
@@ -995,14 +1005,14 @@ Adjust constructor arguments to match the current sealed `VehicleEvent` definiti
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :sink-mq -Dtest=TopicRouterTest test
|
||||
mvn -pl :sink-kafka -Dtest=TopicRouterTest test
|
||||
```
|
||||
|
||||
Expected before implementation: compile failure if constructors are wrong or assertion failure if defaults are old. Fix constructors first; keep routing assertions.
|
||||
|
||||
- [ ] **Step 3: Update `SinkMqProperties.Topics` defaults**
|
||||
- [ ] **Step 3: Update `KafkaSinkProperties.Topics` defaults**
|
||||
|
||||
In `SinkMqProperties.Topics`, change defaults:
|
||||
In `KafkaSinkProperties.Topics`, change defaults:
|
||||
|
||||
```java
|
||||
private String realtime = "vehicle.event.gb32960.v1";
|
||||
@@ -1027,12 +1037,12 @@ public boolean accepts(VehicleEvent event) {
|
||||
|
||||
Update the JavaDoc above it to say Kafka carries both normalized events and raw archive records in split-service mode.
|
||||
|
||||
- [ ] **Step 5: Run sink-mq tests**
|
||||
- [ ] **Step 5: Run sink-kafka tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :sink-mq test
|
||||
mvn -pl :sink-kafka test
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
@@ -1040,11 +1050,11 @@ Expected: PASS.
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java \
|
||||
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java \
|
||||
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java \
|
||||
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java \
|
||||
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java
|
||||
git add modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java \
|
||||
modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java \
|
||||
modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java \
|
||||
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java \
|
||||
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java
|
||||
git commit -m "feat: route gb32960 kafka topics"
|
||||
```
|
||||
|
||||
@@ -1353,8 +1363,8 @@ void gb32960IngestDefaultsOnlyEnableProtocolAndKafkaProducer() {
|
||||
assertThat(props.getProperty("spring.application.name")).isEqualTo("gb32960-ingest-app");
|
||||
assertThat(props.getProperty("lingniu.ingest.gb32960.enabled")).isEqualTo("true");
|
||||
assertThat(props.getProperty("lingniu.ingest.gb32960.port")).isEqualTo("${GB32960_PORT:32960}");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.kafka.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
|
||||
@@ -1378,7 +1388,7 @@ void historyDefaultsEnableStorageAndHistoryConsumerOnly() {
|
||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("true");
|
||||
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
|
||||
.isEqualTo("${KAFKA_GROUP_HISTORY:vehicle-history}");
|
||||
}
|
||||
```
|
||||
@@ -1398,7 +1408,7 @@ void analyticsDefaultsEnableStatConsumerAndDisableProtocolAndHistoryStorage() {
|
||||
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("${VEHICLE_STAT_ENABLED:true}");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
|
||||
.isEqualTo("${KAFKA_GROUP_STAT:vehicle-stat}");
|
||||
}
|
||||
```
|
||||
@@ -1566,7 +1576,7 @@ git commit -m "docs: add gb32960 split service runbook"
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :sink-mq,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test
|
||||
mvn -pl :sink-kafka,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test
|
||||
```
|
||||
|
||||
Expected: BUILD SUCCESS.
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
> **Superseded:** This 2026-06-29 phase-1 implementation plan is historical
|
||||
> context. Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: raw bytes are owned by `sink-archive`,
|
||||
> history reads come from TDengine, and the former `raw-archive-store`
|
||||
> prototype has been removed from the current build surface.
|
||||
|
||||
# Vehicle Ingest Redesign Phase 1 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
@@ -16,7 +23,7 @@ This plan implements only the first slice of the redesign:
|
||||
|
||||
- `modules/core/ingest-facts`: protocol-neutral fact records and helpers.
|
||||
- `modules/sinks/raw-archive-store`: raw archive write/read contract and local URI-safe implementation skeleton.
|
||||
- `modules/sinks/sink-mq`: protobuf messages for raw frame facts and decoded facts.
|
||||
- `modules/sinks/sink-kafka`: protobuf messages for raw frame facts and decoded facts.
|
||||
- Parent Maven wiring and focused tests.
|
||||
|
||||
It intentionally does not yet modify:
|
||||
@@ -53,9 +60,9 @@ Create:
|
||||
Modify:
|
||||
|
||||
- `pom.xml`: add modules and dependency management entries.
|
||||
- `modules/sinks/sink-mq/pom.xml`: add dependency on `ingest-facts`.
|
||||
- `modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto`: add `RawFrameFactPayload`, `DecodedFactPayload`, and related enums/messages.
|
||||
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/VehicleEnvelopeProtoCompatibilityTest.java`: verify new proto messages can round-trip.
|
||||
- `modules/sinks/sink-kafka/pom.xml`: add dependency on `ingest-facts`.
|
||||
- `modules/sinks/sink-kafka/src/main/proto/vehicle_envelope.proto`: add `RawFrameFactPayload`, `DecodedFactPayload`, and related enums/messages.
|
||||
- `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/VehicleEnvelopeProtoCompatibilityTest.java`: verify new proto messages can round-trip.
|
||||
|
||||
---
|
||||
|
||||
@@ -1260,21 +1267,21 @@ git commit -m "feat: add raw archive store contract"
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `modules/sinks/sink-mq/pom.xml`
|
||||
- Modify: `modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto`
|
||||
- Create: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/VehicleEnvelopeProtoCompatibilityTest.java`
|
||||
- Modify: `modules/sinks/sink-kafka/pom.xml`
|
||||
- Modify: `modules/sinks/sink-kafka/src/main/proto/vehicle_envelope.proto`
|
||||
- Create: `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/VehicleEnvelopeProtoCompatibilityTest.java`
|
||||
|
||||
- [ ] **Step 1: Write failing proto compatibility test**
|
||||
|
||||
Create `VehicleEnvelopeProtoCompatibilityTest.java`:
|
||||
|
||||
```java
|
||||
package com.lingniu.ingest.sink.mq;
|
||||
package com.lingniu.ingest.sink.kafka;
|
||||
|
||||
import com.lingniu.ingest.sink.mq.proto.DecodedFactPayload;
|
||||
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
|
||||
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
|
||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
||||
import com.lingniu.ingest.sink.kafka.proto.DecodedFactPayload;
|
||||
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
|
||||
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
|
||||
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -1341,14 +1348,14 @@ class VehicleEnvelopeProtoCompatibilityTest {
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl modules/sinks/sink-mq -Dtest=VehicleEnvelopeProtoCompatibilityTest test
|
||||
mvn -pl modules/sinks/sink-kafka -Dtest=VehicleEnvelopeProtoCompatibilityTest test
|
||||
```
|
||||
|
||||
Expected: FAIL because proto generated classes do not exist.
|
||||
|
||||
- [ ] **Step 3: Add `ingest-facts` dependency to sink-mq**
|
||||
- [ ] **Step 3: Add `ingest-facts` dependency to sink-kafka**
|
||||
|
||||
Add this dependency to `modules/sinks/sink-mq/pom.xml` after `ingest-api`:
|
||||
Add this dependency to `modules/sinks/sink-kafka/pom.xml` after `ingest-api`:
|
||||
|
||||
```xml
|
||||
<dependency>
|
||||
@@ -1412,17 +1419,17 @@ message DecodedFactPayload {
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl modules/sinks/sink-mq -Dtest=VehicleEnvelopeProtoCompatibilityTest test
|
||||
mvn -pl modules/sinks/sink-kafka -Dtest=VehicleEnvelopeProtoCompatibilityTest test
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 6: Run existing sink-mq tests**
|
||||
- [ ] **Step 6: Run existing sink-kafka tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl modules/sinks/sink-mq test
|
||||
mvn -pl modules/sinks/sink-kafka test
|
||||
```
|
||||
|
||||
Expected: PASS. Existing envelope tests must continue to pass because new proto fields are additive.
|
||||
@@ -1430,9 +1437,9 @@ Expected: PASS. Existing envelope tests must continue to pass because new proto
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add modules/sinks/sink-mq/pom.xml \
|
||||
modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto \
|
||||
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/VehicleEnvelopeProtoCompatibilityTest.java
|
||||
git add modules/sinks/sink-kafka/pom.xml \
|
||||
modules/sinks/sink-kafka/src/main/proto/vehicle_envelope.proto \
|
||||
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/VehicleEnvelopeProtoCompatibilityTest.java
|
||||
git commit -m "feat: add fact payloads to kafka envelope"
|
||||
```
|
||||
|
||||
@@ -1449,7 +1456,7 @@ git commit -m "feat: add fact payloads to kafka envelope"
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl modules/core/ingest-facts,modules/sinks/raw-archive-store,modules/sinks/sink-mq -am test
|
||||
mvn -pl modules/core/ingest-facts,modules/sinks/raw-archive-store,modules/sinks/sink-kafka -am test
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
@@ -1459,7 +1466,7 @@ Expected: PASS.
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl modules/sinks/sink-mq -am -DskipTests package
|
||||
mvn -pl modules/sinks/sink-kafka -am -DskipTests package
|
||||
```
|
||||
|
||||
Expected: PASS. Generated protobuf Java sources include `RawFrameFactPayload`, `DecodedFactPayload`, and `ParseStatusProto`.
|
||||
@@ -1473,7 +1480,7 @@ rg -n "RawFrameFact|DecodedFact|RawArchiveWriter|vehicle.raw-frame.v1|vehicle.de
|
||||
modules/apps modules/protocols modules/services modules/core modules/sinks
|
||||
```
|
||||
|
||||
Expected: Matches only in the new modules, sink-mq proto/test, and planned references. No existing handler behavior should be changed in Phase 1.
|
||||
Expected: Matches only in the new modules, sink-kafka proto/test, and planned references. No existing handler behavior should be changed in Phase 1.
|
||||
|
||||
- [ ] **Step 4: Commit final verification note if any docs changed**
|
||||
|
||||
|
||||
@@ -2,14 +2,15 @@
|
||||
|
||||
## Current Goal
|
||||
|
||||
Consume JT808 Kafka location events, keep rolling daily state, and write the derived daily mileage into the common vehicle-stat metric repository.
|
||||
Consume JT808 Kafka location events and write the derived daily mileage into the common vehicle-stat metric repository.
|
||||
|
||||
## Current Design
|
||||
|
||||
- Only supported message backbone: Kafka.
|
||||
- Source topic: `vehicle.event.jt808.v1`
|
||||
- Runtime app: `vehicle-analytics-app`
|
||||
- Runtime state: Redis `Jt808MileageStateStore`
|
||||
- Metric output: `VehicleStatRepository.saveDailyStat(...)`
|
||||
- Runtime state: none outside `vehicle_stat_metric`
|
||||
- Metric output: `VehicleStatRepository.recordDailyMileageSample(...)`
|
||||
- Production metric storage: JDBC/MySQL `vehicle_stat_metric`
|
||||
- Date boundary: `Asia/Shanghai`
|
||||
- Calculation method: `JT808_TOTAL_MILEAGE_DIFF`
|
||||
@@ -17,10 +18,17 @@ Consume JT808 Kafka location events, keep rolling daily state, and write the der
|
||||
JT808 daily mileage is calculated only from the GPS total mileage reported by JT808 location additional information:
|
||||
|
||||
```text
|
||||
daily_mileage_km = last_total_mileage_km - first_total_mileage_km
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
```
|
||||
|
||||
The stream saves a metric only when at least two ordered location points for the same vehicle and local day contain valid `total_mileage_km`. Negative differences are treated as invalid and are not written.
|
||||
The first valid local-day sample writes `daily_mileage_km=0.0`. Later ordered or replayed samples update the same metric row with:
|
||||
|
||||
```text
|
||||
metric_value = max_total_mileage_km - min_total_mileage_km
|
||||
metric_key = daily_mileage_km
|
||||
```
|
||||
|
||||
The local-day minimum and maximum GPS total mileage values stay on that same metric row as calculation source columns so restarts recover from MySQL without Redis or another state table.
|
||||
|
||||
## Runtime Properties
|
||||
|
||||
@@ -28,16 +36,11 @@ The stream saves a metric only when at least two ordered location points for the
|
||||
KAFKA_TOPIC_JT808_EVENT=vehicle.event.jt808.v1
|
||||
VEHICLE_STAT_ENABLED=true
|
||||
VEHICLE_STAT_JT808_MILEAGE_ENABLED=true
|
||||
VEHICLE_STAT_REPOSITORY_TYPE=jdbc
|
||||
MYSQL_JDBC_URL=<jdbc-url>
|
||||
MYSQL_USERNAME=<user>
|
||||
MYSQL_PASSWORD=<password>
|
||||
REDIS_HOST=<host>
|
||||
REDIS_PORT=6379
|
||||
REDIS_DATABASE=50
|
||||
REDIS_PASSWORD=<password>
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
Do not create or write a protocol-specific JT808 daily-mileage table. Do not add GPS segment distance, speed integral, or jump-filter state back into this path unless the mileage definition changes again.
|
||||
Do not create or write a protocol-specific JT808 daily-mileage table. Do not add another message backbone, distance accumulation, integral calculation, Redis state, or memory state back into this path unless the mileage definition changes again.
|
||||
|
||||
720
docs/superpowers/plans/2026-07-01-go-ingest-redesign-phase1.md
Normal file
720
docs/superpowers/plans/2026-07-01-go-ingest-redesign-phase1.md
Normal file
@@ -0,0 +1,720 @@
|
||||
# Go Vehicle Ingest Redesign Phase 1 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build the first production-capable Go runtime for GB32960, JT808, and Yutong MQTT ingestion with unified Kafka, TDengine, MySQL statistics, and Redis realtime state boundaries.
|
||||
|
||||
**Architecture:** Create a new single Go module at `go/vehicle-gateway` and migrate only the useful pieces from the existing `go/ingest-edge` and `go/vehicle-state` prototypes. The gateway produces unified envelopes to Kafka; independent consumers write TDengine history, MySQL daily metrics, and Redis realtime state.
|
||||
|
||||
**Tech Stack:** Go 1.26+, Kafka (`segmentio/kafka-go`), Redis (`redis/go-redis`), MySQL (`go-sql-driver/mysql`), TDengine official Go connector, MQTT (`eclipse/paho.mqtt.golang`), standard library TCP.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- Create: `go/vehicle-gateway/go.mod`
|
||||
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/*`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/*`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/yutongmqtt/*`
|
||||
- Create: `go/vehicle-gateway/internal/gateway/*`
|
||||
- Create: `go/vehicle-gateway/internal/identity/*`
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/*`
|
||||
- Create: `go/vehicle-gateway/internal/history/*`
|
||||
- Create: `go/vehicle-gateway/internal/stats/*`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/*`
|
||||
- Create: `go/vehicle-gateway/internal/observability/*`
|
||||
- Modify: `README.md`
|
||||
- Modify: `docs/target-architecture.md`
|
||||
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||
|
||||
The existing `go/ingest-edge` and `go/vehicle-state` directories are migration sources only. After phase 1 is verified, remove or mark them superseded.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Create Single Go Module
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/go.mod`
|
||||
- Create: `go/vehicle-gateway/internal/observability/logger.go`
|
||||
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
|
||||
- [ ] **Step 1: Create module manifest**
|
||||
|
||||
Add `go/vehicle-gateway/go.mod`:
|
||||
|
||||
```go
|
||||
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/taosdata/driver-go/v3 v3.8.1
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add logger helper**
|
||||
|
||||
Add `go/vehicle-gateway/internal/observability/logger.go`:
|
||||
|
||||
```go
|
||||
package observability
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
func NewLogger(service string) *slog.Logger {
|
||||
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||
return slog.New(handler).With("service", service)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add temporary gateway entrypoint**
|
||||
|
||||
Add `go/vehicle-gateway/cmd/gateway/main.go`:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-gateway")
|
||||
logger.Info("vehicle gateway scaffold started")
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify scaffold builds**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go mod tidy
|
||||
go test ./...
|
||||
go build ./cmd/gateway
|
||||
```
|
||||
|
||||
Expected: all commands exit `0`.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Define Unified Envelope
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||
|
||||
- [ ] **Step 1: Write envelope tests**
|
||||
|
||||
Add `go/vehicle-gateway/internal/envelope/envelope_test.go`:
|
||||
|
||||
```go
|
||||
package envelope
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||
e := FrameEnvelope{
|
||||
Protocol: ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013307795425",
|
||||
Sequence: 1,
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7e02000000ff7e",
|
||||
}
|
||||
a := e.StableEventID()
|
||||
b := e.StableEventID()
|
||||
if a == "" || a != b {
|
||||
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests and confirm failure**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/envelope
|
||||
```
|
||||
|
||||
Expected: fail because `FrameEnvelope` is not defined.
|
||||
|
||||
- [ ] **Step 3: Implement envelope**
|
||||
|
||||
Add `go/vehicle-gateway/internal/envelope/envelope.go`:
|
||||
|
||||
```go
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Protocol string
|
||||
|
||||
const (
|
||||
ProtocolGB32960 Protocol = "GB32960"
|
||||
ProtocolJT808 Protocol = "JT808"
|
||||
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||
)
|
||||
|
||||
type ParseStatus string
|
||||
|
||||
const (
|
||||
ParseOK ParseStatus = "OK"
|
||||
ParsePartial ParseStatus = "PARTIAL"
|
||||
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||
)
|
||||
|
||||
type FrameEnvelope struct {
|
||||
EventID string `json:"event_id"`
|
||||
TraceID string `json:"trace_id"`
|
||||
Protocol Protocol `json:"protocol"`
|
||||
MessageID string `json:"message_id"`
|
||||
Sequence uint16 `json:"sequence"`
|
||||
VIN string `json:"vin,omitempty"`
|
||||
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
Parsed map[string]any `json:"parsed,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
ParseStatus ParseStatus `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) VehicleKey() string {
|
||||
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
return string(e.Protocol) + ":unknown"
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) StableEventID() string {
|
||||
if strings.TrimSpace(e.EventID) != "" {
|
||||
return e.EventID
|
||||
}
|
||||
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||
sum := sha256.Sum256([]byte(input))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||
if e.EventID == "" {
|
||||
e.EventID = e.StableEventID()
|
||||
}
|
||||
if e.ParseStatus == "" {
|
||||
e.ParseStatus = ParseOK
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify tests pass**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/envelope
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Implement JT808 Frame and 0200 Parser
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser_test.go`
|
||||
|
||||
- [ ] **Step 1: Add sample frame tests**
|
||||
|
||||
Use the production sample provided in the thread:
|
||||
|
||||
```text
|
||||
7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E
|
||||
```
|
||||
|
||||
Expected assertions:
|
||||
|
||||
- message id is `0x0200`
|
||||
- phone keeps normalized BCD string `013307795425`
|
||||
- sequence is `1`
|
||||
- `fields.total_mileage_km` is parsed from additional item `0x01`
|
||||
- latitude, longitude, speed, direction, alarm, status, and device time are present
|
||||
|
||||
- [ ] **Step 2: Implement parser**
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- strip `0x7e` start/end delimiters
|
||||
- unescape `0x7d 0x02 -> 0x7e`
|
||||
- unescape `0x7d 0x01 -> 0x7d`
|
||||
- verify XOR checksum
|
||||
- parse 2011/2013 common header
|
||||
- parse BCD phone from 6 bytes and keep both raw and normalized values in `parsed.header`
|
||||
- parse 0200 fixed body
|
||||
- parse additional item list as `id,length,value_hex`
|
||||
- parse additional `0x01` as `total_mileage_km = uint32 / 10`
|
||||
|
||||
- [ ] **Step 3: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/protocol/jt808
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Implement GB32960 Frame and Data Unit Parser
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser_test.go`
|
||||
|
||||
- [ ] **Step 1: Add parser tests**
|
||||
|
||||
Tests must cover:
|
||||
|
||||
- `##` frame boundary
|
||||
- command id
|
||||
- response flag
|
||||
- 17-byte VIN
|
||||
- encryption flag
|
||||
- payload length
|
||||
- BCC verification
|
||||
- realtime data command `0x02`
|
||||
- reissue data command `0x03`
|
||||
- data unit `0x01` vehicle status fields
|
||||
- data unit `0x05` position fields
|
||||
|
||||
- [ ] **Step 2: Implement parser**
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- parse header without allocating large temporary buffers
|
||||
- keep RAW as hex in envelope
|
||||
- write all recognized data units to `Parsed`
|
||||
- write core fields to `Fields`
|
||||
- unsupported data unit stays in `Parsed["unknown_units"]`
|
||||
|
||||
- [ ] **Step 3: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/protocol/gb32960
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Implement Kafka Sink
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink.go`
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink_test.go`
|
||||
|
||||
- [ ] **Step 1: Add sink interface**
|
||||
|
||||
Create a sink interface:
|
||||
|
||||
```go
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Sink interface {
|
||||
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||
Close() error
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Implement Kafka topic routing**
|
||||
|
||||
Rules:
|
||||
|
||||
- `GB32960 -> vehicle.raw.gb32960.v1`
|
||||
- `JT808 -> vehicle.raw.jt808.v1`
|
||||
- `YUTONG_MQTT -> vehicle.raw.yutong-mqtt.v1`
|
||||
- unified topic is `vehicle.event.unified.v1`
|
||||
- message key is `env.VehicleKey()`
|
||||
|
||||
- [ ] **Step 3: Verify routing with unit tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/eventbus
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Implement TDengine History Writer
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/history/schema.go`
|
||||
- Create: `go/vehicle-gateway/internal/history/writer.go`
|
||||
- Create: `go/vehicle-gateway/internal/history/writer_test.go`
|
||||
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||
|
||||
- [ ] **Step 1: Add schema bootstrap SQL**
|
||||
|
||||
Implement schema strings for:
|
||||
|
||||
- database `lingniu_vehicle_ts`
|
||||
- stable `raw_frames`
|
||||
- stable `vehicle_locations`
|
||||
- stable `vehicle_mileage_points`
|
||||
|
||||
- [ ] **Step 2: Implement writer**
|
||||
|
||||
Rules:
|
||||
|
||||
- `AppendRawFrame` always writes one raw row.
|
||||
- `AppendLocation` writes only when longitude and latitude exist.
|
||||
- `AppendMileagePoint` writes only when `total_mileage_km` exists.
|
||||
- child table name is deterministic hash of `protocol + vehicle_key`.
|
||||
- escape tag values.
|
||||
|
||||
- [ ] **Step 3: Use TDengine official driver**
|
||||
|
||||
Import WebSocket driver in command:
|
||||
|
||||
```go
|
||||
import _ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
```
|
||||
|
||||
Default driver name:
|
||||
|
||||
```text
|
||||
TDENGINE_DRIVER=taosWS
|
||||
```
|
||||
|
||||
Short-term compatibility:
|
||||
|
||||
```text
|
||||
TDENGINE_DRIVER=taosSql
|
||||
```
|
||||
|
||||
Only use `taosSql` when ECS TDengine WebSocket is not available.
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/history
|
||||
go build ./cmd/history-writer
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Implement MySQL Daily Metric Writer
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/stats/schema.go`
|
||||
- Create: `go/vehicle-gateway/internal/stats/daily_metric.go`
|
||||
- Create: `go/vehicle-gateway/internal/stats/daily_metric_test.go`
|
||||
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||
|
||||
- [ ] **Step 1: Add schema bootstrap**
|
||||
|
||||
Implement `vehicle_daily_metric` schema from the design spec.
|
||||
|
||||
- [ ] **Step 2: Add metric derivation tests**
|
||||
|
||||
Test cases:
|
||||
|
||||
- no `total_mileage_km` produces no metric
|
||||
- one sample produces:
|
||||
- `daily_mileage_km = 0`
|
||||
- `daily_total_mileage_km = sample`
|
||||
- later larger sample updates:
|
||||
- `latest_total_mileage_km`
|
||||
- `daily_mileage_km`
|
||||
- `daily_total_mileage_km`
|
||||
- out-of-order smaller sample updates:
|
||||
- `first_total_mileage_km`
|
||||
- `daily_mileage_km`
|
||||
|
||||
- [ ] **Step 3: Implement MySQL upsert**
|
||||
|
||||
Use one table and one idempotent upsert:
|
||||
|
||||
```sql
|
||||
INSERT INTO vehicle_daily_metric
|
||||
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
first_total_mileage_km = LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)),
|
||||
latest_total_mileage_km = GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)),
|
||||
metric_value = CASE
|
||||
WHEN metric_key = 'daily_mileage_km'
|
||||
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
- LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
WHEN metric_key = 'daily_total_mileage_km'
|
||||
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
ELSE VALUES(metric_value)
|
||||
END,
|
||||
sample_count = sample_count + 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/stats
|
||||
go build ./cmd/stat-writer
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Implement Redis Realtime API
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/realtime/repository.go`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/repository_test.go`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/http.go`
|
||||
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||
|
||||
- [ ] **Step 1: Add repository contract**
|
||||
|
||||
Methods:
|
||||
|
||||
- `Update(ctx, envelope.FrameEnvelope) error`
|
||||
- `GetMerged(ctx, vin string) (Snapshot, error)`
|
||||
- `GetProtocol(ctx, vin string, protocol envelope.Protocol) (Snapshot, error)`
|
||||
- `IsOnline(ctx, vin string) (OnlineStatus, error)`
|
||||
|
||||
- [ ] **Step 2: Implement merge logic**
|
||||
|
||||
Rules:
|
||||
|
||||
- update `vehicle:latest:{vin}:{protocol}`
|
||||
- update `vehicle:latest:{vin}` with newest fields
|
||||
- update `vehicle:online:{vin}`
|
||||
- update sorted set `vehicle:last_seen`
|
||||
|
||||
- [ ] **Step 3: Implement HTTP API**
|
||||
|
||||
Routes:
|
||||
|
||||
- `GET /api/realtime/vehicles/{vin}`
|
||||
- `GET /api/realtime/vehicles/{vin}/online`
|
||||
- `GET /api/realtime/vehicles/{vin}/protocols/{protocol}`
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/realtime
|
||||
go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Wire Gateway Runtime
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/gateway/tcp_server.go`
|
||||
- Create: `go/vehicle-gateway/internal/gateway/mqtt_client.go`
|
||||
- Modify: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
|
||||
- [ ] **Step 1: Implement TCP server**
|
||||
|
||||
Rules:
|
||||
|
||||
- one goroutine per accepted connection
|
||||
- bounded max connections
|
||||
- read timeout and idle timeout
|
||||
- protocol-specific frame extractor
|
||||
- structured peer endpoint
|
||||
- graceful shutdown on SIGTERM
|
||||
|
||||
- [ ] **Step 2: Implement MQTT client**
|
||||
|
||||
Rules:
|
||||
|
||||
- connect with official production config from environment
|
||||
- subscribe configured topic list
|
||||
- convert each message into envelope
|
||||
- publish raw and unified events
|
||||
- reconnect with backoff
|
||||
|
||||
- [ ] **Step 3: Verify local JSON mode**
|
||||
|
||||
Run without Kafka:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
GB32960_TCP_ADDR=:132960 JT808_TCP_ADDR=:18080 go run ./cmd/gateway
|
||||
```
|
||||
|
||||
Expected: service starts and logs configured listeners.
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Docker and ECS Deployment
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/Dockerfile`
|
||||
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||
- Modify: `docs/operations/current-ecs-deployment.md`
|
||||
|
||||
- [ ] **Step 1: Add multi-stage Dockerfile**
|
||||
|
||||
Build all commands:
|
||||
|
||||
- `gateway`
|
||||
- `history-writer`
|
||||
- `stat-writer`
|
||||
- `realtime-api`
|
||||
|
||||
- [ ] **Step 2: Add Portainer compose**
|
||||
|
||||
Services:
|
||||
|
||||
- `go-vehicle-gateway`
|
||||
- `go-history-writer`
|
||||
- `go-stat-writer`
|
||||
- `go-realtime-api`
|
||||
|
||||
Each service must include:
|
||||
|
||||
- restart policy
|
||||
- memory limit
|
||||
- Kafka env
|
||||
- MySQL/TDengine/Redis env as needed
|
||||
- logging options
|
||||
|
||||
- [ ] **Step 3: Verify Linux build**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
Expected: all commands exit `0`.
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Production Verification
|
||||
|
||||
**Files:**
|
||||
- Create: `docs/operations/go-vehicle-gateway-verification.md`
|
||||
|
||||
- [ ] **Step 1: Record test commands**
|
||||
|
||||
Document commands to verify:
|
||||
|
||||
- gateway process health
|
||||
- Kafka topic consumption
|
||||
- TDengine row counts
|
||||
- MySQL daily metric rows
|
||||
- Redis realtime lookup
|
||||
|
||||
- [ ] **Step 2: Validate real traffic**
|
||||
|
||||
Evidence required:
|
||||
|
||||
- one real 32960 VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||
- one real JT808 phone/VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||
- one real Yutong MQTT VIN with RAW and Redis snapshot
|
||||
|
||||
- [ ] **Step 3: Keep old Java services until evidence is captured**
|
||||
|
||||
Only disable Java equivalents after the evidence file contains successful command outputs and timestamps.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review Checklist
|
||||
|
||||
- The plan creates one new Go module instead of extending scattered prototypes.
|
||||
- The plan covers GB32960, JT808, and Yutong MQTT ingress.
|
||||
- The plan covers Kafka, TDengine, MySQL, and Redis.
|
||||
- The plan includes 32960 and 808 daily mileage and daily total mileage.
|
||||
- The plan includes local and ECS verification.
|
||||
- The plan does not restore Xinda Push.
|
||||
- The plan keeps Java services untouched until Go evidence exists.
|
||||
@@ -1,3 +1,9 @@
|
||||
> **Superseded:** This 2026-06-23 DuckDB-based design is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine is the default hot history store,
|
||||
> `event-file-store` and Xinda Push have been removed from the production codebase.
|
||||
|
||||
# GB32960 Production Readiness Design
|
||||
|
||||
## Goal
|
||||
@@ -337,23 +343,19 @@ Validation:
|
||||
|
||||
### Daily Mileage
|
||||
|
||||
Preferred:
|
||||
JT808 daily mileage uses the GPS total mileage reported in location additional
|
||||
information. The local-day minimum and maximum GPS total mileage values rewrite
|
||||
the same metric row, so ordered streaming and historical replay use one
|
||||
calculation path.
|
||||
|
||||
```text
|
||||
dailyMileage = today.lastTotalMileage - previousDay.lastTotalMileage
|
||||
dailyMileage = today.maxTotalMileage - today.minTotalMileage
|
||||
calculationMethod = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
Fallback:
|
||||
|
||||
```text
|
||||
dailyMileage = today.lastTotalMileage - today.firstTotalMileage
|
||||
```
|
||||
|
||||
Quality:
|
||||
|
||||
- `PREVIOUS_DAY_BASELINE` when previous day baseline exists;
|
||||
- `INTRADAY_DELTA` when only same-day baseline exists;
|
||||
- `INSUFFICIENT_DATA` when neither is reliable.
|
||||
The result is stored in `vehicle_stat_metric` with
|
||||
`metric_key=daily_mileage_km`. There is no JT808-specific daily mileage table,
|
||||
no Redis mileage state, and no previous-day baseline calculation.
|
||||
|
||||
### Daily Power Consumption
|
||||
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
> **Superseded:** This 2026-06-23 split design is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine is the default hot history store,
|
||||
> `sink-archive` owns raw bytes, and `event-file-store` is optional
|
||||
> compatibility only.
|
||||
|
||||
# GB32960 Service Split Design
|
||||
|
||||
Date: 2026-06-23
|
||||
@@ -53,7 +60,7 @@ Initial module dependencies:
|
||||
- `ingest-codec-common`
|
||||
- `session-core`
|
||||
- `protocol-gb32960`
|
||||
- `sink-mq`
|
||||
- `sink-kafka`
|
||||
- `observability`
|
||||
|
||||
### `vehicle-history-app`
|
||||
@@ -79,7 +86,7 @@ Non-responsibilities:
|
||||
Initial module dependencies:
|
||||
|
||||
- `ingest-api`
|
||||
- `sink-mq`
|
||||
- `sink-kafka`
|
||||
- `sink-archive`
|
||||
- `event-file-store`
|
||||
- `event-history-service`
|
||||
@@ -109,7 +116,7 @@ Non-responsibilities:
|
||||
Initial module dependencies:
|
||||
|
||||
- `ingest-api`
|
||||
- `sink-mq`
|
||||
- `sink-kafka`
|
||||
- `vehicle-state-service`
|
||||
- `vehicle-stat-service`
|
||||
- `observability`
|
||||
@@ -126,7 +133,7 @@ Use versioned topics:
|
||||
- `vehicle.event.gb32960.v1`
|
||||
- `vehicle.dlq.gb32960.v1`
|
||||
|
||||
The current topic names under `lingniu.ingest.sink.mq.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
|
||||
The current topic names under `lingniu.ingest.sink.kafka.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
|
||||
|
||||
### Keys
|
||||
|
||||
|
||||
@@ -433,7 +433,7 @@ Realtime APIs:
|
||||
Realtime state APIs:
|
||||
|
||||
- `GET /api/realtime/vehicles/{vehicleKey}`
|
||||
- Reads Redis or in-memory latest state, not TDengine.
|
||||
- Reads Redis latest state, not TDengine.
|
||||
|
||||
Export:
|
||||
|
||||
@@ -486,7 +486,7 @@ Read path:
|
||||
|
||||
- Most high-QPS queries require vehicle key and time range.
|
||||
- Unbounded cross-vehicle queries are admin-only and capped.
|
||||
- Realtime latest state is served from Redis/in-memory state.
|
||||
- Realtime latest state is served from Redis latest-state cache.
|
||||
- Raw detail replay reads one raw object and decodes on demand.
|
||||
- Common dictionary and metadata responses are cached in memory.
|
||||
|
||||
|
||||
498
docs/superpowers/specs/2026-07-01-go-ingest-redesign-design.md
Normal file
498
docs/superpowers/specs/2026-07-01-go-ingest-redesign-design.md
Normal file
@@ -0,0 +1,498 @@
|
||||
# Go 车辆接入重构设计规格
|
||||
|
||||
## 目标
|
||||
|
||||
使用 Go 重新设计车辆数据接入运行面,让系统回到第一性原则:
|
||||
|
||||
- 接入层只负责可靠收包、协议解析、应答、身份解析和投递。
|
||||
- Kafka 是唯一在线消息通道。
|
||||
- TDengine 保存高吞吐时序明细和完整 RAW 解析 JSON。
|
||||
- MySQL 保存低频配置、身份映射和每日统计窄表。
|
||||
- Redis 保存可重建的准实时状态。
|
||||
- 32960、JT808、宇通 MQTT 使用同一套数据层接口,避免每个协议各自落库。
|
||||
|
||||
第一阶段只做生产链路的骨架和核心数据闭环:接收、解析、Kafka、TDengine、MySQL 统计、Redis 实时查询、ECS 验证。历史查询 API 和业务侧复杂分析放在后续阶段。
|
||||
|
||||
## 参考原则
|
||||
|
||||
本设计只吸收开源项目的边界思想,不复制外部实现代码。
|
||||
|
||||
- GB32960 参考 `DarkInno/gb32960-go-sdk`:连接管理、Handler、Forwarder、鉴权接口分离。
|
||||
- JT808 参考 `cuteLittleDevil/go-jt808`:高并发 Go 原生 TCP、协议核心小而可扩展、事件/适配器机制。
|
||||
- JT808 参考 `fakeyanss/jt808-server-go`:`FramePayload -> PacketData -> JT808Msg -> Reply` 的分层处理、Gateway 模式、保活和版本兼容。
|
||||
- TDengine 参考官方数据模型:按数据采集点建立 supertable,静态维度做 tags,明细数据按子表写入;Go 连接优先使用 WebSocket 驱动。
|
||||
|
||||
## 非目标
|
||||
|
||||
- 第一阶段不继续扩展 Java 服务。
|
||||
- 第一阶段不做统一大查询 API。
|
||||
- 第一阶段不恢复信达 Push。
|
||||
- 第一阶段不把所有协议字段展开成一张超宽 TDengine 表。
|
||||
- Redis 不作为历史事实源。
|
||||
- MySQL 不保存高频原始明细。
|
||||
|
||||
## 目标目录
|
||||
|
||||
新的 Go 运行面放在 `go/vehicle-gateway`,不继续沿用实验性的多 module 分散结构。现有 `go/ingest-edge` 和 `go/vehicle-state` 可以作为迁移素材,最终归并或删除。
|
||||
|
||||
```text
|
||||
go/vehicle-gateway/
|
||||
cmd/
|
||||
gateway/ # 协议接入进程:32960 TCP、JT808 TCP、宇通 MQTT
|
||||
history-writer/ # Kafka -> TDengine RAW/location/mileage points
|
||||
stat-writer/ # Kafka -> MySQL 每日统计窄表
|
||||
realtime-api/ # Redis 准实时查询 API
|
||||
internal/
|
||||
config/ # 环境变量、配置文件、校验
|
||||
gateway/ # TCP/MQTT 生命周期、连接限制、优雅停机
|
||||
protocol/
|
||||
gb32960/ # 32960 编解码
|
||||
jt808/ # 808 编解码
|
||||
yutongmqtt/ # 宇通 MQTT payload 解析
|
||||
identity/ # VIN、phone、device_id、plate 绑定
|
||||
envelope/ # 统一事件模型
|
||||
eventbus/ # Kafka producer/consumer
|
||||
history/ # TDengine adapter
|
||||
stats/ # 每日里程聚合
|
||||
realtime/ # Redis adapter 与快照合并
|
||||
observability/ # 日志、metrics、健康检查
|
||||
```
|
||||
|
||||
## 协议接入设计
|
||||
|
||||
### GB32960
|
||||
|
||||
职责:
|
||||
|
||||
- 监听 TCP `32960`。
|
||||
- 支持车辆登入、实时信息上报、补发信息上报、车辆登出、平台登入/登出、心跳、校时。
|
||||
- 正确返回协议应答。
|
||||
- 完整解析 2016 数据单元:
|
||||
- 整车数据
|
||||
- 驱动电机数据
|
||||
- 燃料电池数据
|
||||
- 发动机数据
|
||||
- 车辆位置数据
|
||||
- 极值数据
|
||||
- 报警数据
|
||||
- 可充电储能装置电压数据
|
||||
- 可充电储能装置温度数据
|
||||
- RAW 中保存完整 parsed JSON。
|
||||
- 关键字段标准化为统一 field:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `soc_percent`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `vehicle_status`
|
||||
- `charge_status`
|
||||
|
||||
### JT808
|
||||
|
||||
职责:
|
||||
|
||||
- 监听 TCP `808`。
|
||||
- 支持 2011/2013 基础兼容,预留 2019 版本标记。
|
||||
- 分层处理:
|
||||
- Frame:`0x7e` 包边界、转义、校验。
|
||||
- Packet:消息头、手机号 BCD、流水号、分包。
|
||||
- Message:注册、鉴权、心跳、注销、位置、批量位置、未知上行。
|
||||
- Reply:注册应答、平台通用应答。
|
||||
- 终端手机号按协议头 BCD 解析,内部同时保留原始 BCD、规范化 phone、去前导零 phone。
|
||||
- 注册帧和鉴权帧写入 808 registration 表。
|
||||
- 如果设备直接上报位置帧,也要走 identity 解析,尝试用 phone、device_id、plate 找到 VIN。
|
||||
- 0200 附加信息必须解析表 27:
|
||||
- `0x01` GPS 总里程,DWORD,单位 0.1 km。
|
||||
- 其他已知附加项结构化放入 `parsed_json.additional`。
|
||||
- 标准化字段:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `altitude_m`
|
||||
- `direction_deg`
|
||||
- `alarm_flag`
|
||||
- `status_flag`
|
||||
|
||||
### 宇通 MQTT
|
||||
|
||||
职责:
|
||||
|
||||
- 使用正式 MQTT 配置订阅生产 topic。
|
||||
- 接收 payload 后解析为统一 envelope。
|
||||
- 按 payload 中的车辆标识解析 VIN、车牌、设备号。
|
||||
- 保存完整 payload 和 parsed JSON。
|
||||
- 标准化字段向 32960/808 对齐:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `soc_percent`
|
||||
- `vehicle_status`
|
||||
|
||||
## 统一 Envelope
|
||||
|
||||
所有协议接入后输出同一结构。
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "string",
|
||||
"trace_id": "string",
|
||||
"protocol": "GB32960|JT808|YUTONG_MQTT",
|
||||
"message_id": "string",
|
||||
"vin": "string",
|
||||
"vehicle_key": "string",
|
||||
"phone": "string",
|
||||
"device_id": "string",
|
||||
"plate": "string",
|
||||
"source_endpoint": "ip:port or mqtt://broker/topic",
|
||||
"event_time_ms": 0,
|
||||
"received_at_ms": 0,
|
||||
"raw_hex": "string",
|
||||
"raw_text": "string",
|
||||
"parsed": {},
|
||||
"fields": {
|
||||
"speed_kmh": 0,
|
||||
"total_mileage_km": 0,
|
||||
"longitude": 0,
|
||||
"latitude": 0
|
||||
},
|
||||
"parse_status": "OK|PARTIAL|BAD_FRAME",
|
||||
"parse_error": ""
|
||||
}
|
||||
```
|
||||
|
||||
规则:
|
||||
|
||||
- `event_id` 使用协议、设备标识、消息流水、事件时间、RAW hash 生成,保证幂等。
|
||||
- Kafka partition key 优先使用 VIN;没有 VIN 时使用 `protocol:phone/device_id`。
|
||||
- `parsed` 保存协议完整结构化字段。
|
||||
- `fields` 只保存跨协议核心字段,用于统计、位置和实时快照。
|
||||
|
||||
## Kafka Topic
|
||||
|
||||
生产只支持 Kafka。
|
||||
|
||||
| Topic | 内容 | Key |
|
||||
|---|---|---|
|
||||
| `vehicle.raw.gb32960.v1` | 32960 完整 RAW envelope | VIN 或 vehicle_key |
|
||||
| `vehicle.raw.jt808.v1` | 808 完整 RAW envelope | VIN 或 phone |
|
||||
| `vehicle.raw.yutong-mqtt.v1` | 宇通 MQTT 完整 RAW envelope | VIN 或 vehicle_key |
|
||||
| `vehicle.event.unified.v1` | 统一轻量事件,用于业务消费 | VIN 或 vehicle_key |
|
||||
|
||||
接入层必须先写 RAW topic,再写 unified event。后续消费者只依赖 Kafka,不直接依赖接入进程内存。
|
||||
|
||||
发布可靠性:
|
||||
|
||||
- Kafka 生产端必须开启同步写入,RAW 写成功后才允许写 unified event。
|
||||
- Kafka 写入必须支持可配置重试、单次写超时和短退避,默认值为 `KAFKA_PUBLISH_ATTEMPTS=3`、`KAFKA_PUBLISH_TIMEOUT_MS=3000`、`KAFKA_PUBLISH_BACKOFF_MS=100`。
|
||||
- gateway 支持本地磁盘 spool/WAL,配置 `KAFKA_SPOOL_DIR` 后启用;Kafka 长时间不可用时,发布失败的 envelope 先原子写入本地 JSON 文件。
|
||||
- spool 补发按文件名顺序执行,补发成功后删除文件;默认补发间隔由 `KAFKA_SPOOL_REPLAY_INTERVAL_MS=1000` 控制。
|
||||
- 如果 RAW 写 Kafka 失败并已落盘,同一个 event 的 unified event 不允许抢先写 Kafka,也必须进入 spool,确保恢复后按 RAW -> unified 顺序补发。
|
||||
- Kafka topic 不允许自动创建,topic 和分区数由部署脚本或运维初始化,避免生产拼写错误造成隐性分流。
|
||||
|
||||
## TDengine 数据库设计
|
||||
|
||||
如果现有 TDengine 库不符合目标,可以新建库并迁移。建议库名:
|
||||
|
||||
```sql
|
||||
CREATE DATABASE IF NOT EXISTS lingniu_vehicle_ts KEEP 7300 DURATION 10 BUFFER 256;
|
||||
```
|
||||
|
||||
### raw_frames
|
||||
|
||||
保存完整 RAW 和完整 parsed JSON。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
event_id NCHAR(64),
|
||||
message_id INT,
|
||||
event_time TIMESTAMP,
|
||||
received_at TIMESTAMP,
|
||||
raw_size_bytes INT,
|
||||
raw_hex BINARY(16374),
|
||||
raw_text BINARY(16374),
|
||||
parsed_json BINARY(16374),
|
||||
fields_json BINARY(4096),
|
||||
parse_status NCHAR(16),
|
||||
parse_error BINARY(1024),
|
||||
source_endpoint NCHAR(128)
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- 子表按 `protocol + vehicle_key hash` 创建。
|
||||
- RAW 查询优先按 tags 和时间范围过滤。
|
||||
- `parsed_json` 是完整协议字段;查询 API 可选择是否返回,避免默认大字段拖慢。
|
||||
|
||||
### vehicle_locations
|
||||
|
||||
只保存位置查询核心字段。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg INT,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
### vehicle_mileage_points
|
||||
|
||||
保存总里程采样点,用于回放重算和查询。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
total_mileage_km DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
## MySQL 数据库设计
|
||||
|
||||
MySQL 只保存配置、身份映射、808 注册鉴权和每日统计窄表。
|
||||
|
||||
### vehicle_identity_binding
|
||||
|
||||
用于把外部标识定位到 VIN。
|
||||
|
||||
```sql
|
||||
CREATE TABLE vehicle_identity_binding (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
plate VARCHAR(32) NULL,
|
||||
phone VARCHAR(32) NULL,
|
||||
device_id VARCHAR(64) NULL,
|
||||
remark VARCHAR(255) NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_vin (vin),
|
||||
KEY idx_plate (plate),
|
||||
KEY idx_phone (phone),
|
||||
KEY idx_device_id (device_id)
|
||||
);
|
||||
```
|
||||
|
||||
### jt808_registration
|
||||
|
||||
只记录 808 独有的注册、鉴权和在线身份信息。
|
||||
|
||||
```sql
|
||||
CREATE TABLE jt808_registration (
|
||||
phone VARCHAR(32) PRIMARY KEY,
|
||||
vin VARCHAR(32) NULL,
|
||||
plate VARCHAR(32) NULL,
|
||||
device_id VARCHAR(64) NULL,
|
||||
manufacturer_id VARCHAR(32) NULL,
|
||||
terminal_model VARCHAR(64) NULL,
|
||||
terminal_id VARCHAR(64) NULL,
|
||||
auth_code VARCHAR(128) NULL,
|
||||
source_endpoint VARCHAR(128) NULL,
|
||||
first_registered_at DATETIME NULL,
|
||||
latest_registered_at DATETIME NULL,
|
||||
latest_authed_at DATETIME NULL,
|
||||
latest_seen_at DATETIME NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
KEY idx_vin (vin),
|
||||
KEY idx_plate (plate),
|
||||
KEY idx_device_id (device_id)
|
||||
);
|
||||
```
|
||||
|
||||
### vehicle_daily_metric
|
||||
|
||||
32960 和 808 的每日里程、每日总里程都进入同一张窄表。
|
||||
|
||||
```sql
|
||||
CREATE TABLE vehicle_daily_metric (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,3) NOT NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
|
||||
KEY idx_stat_date (stat_date),
|
||||
KEY idx_protocol_metric (protocol, metric_key)
|
||||
);
|
||||
```
|
||||
|
||||
指标:
|
||||
|
||||
- `daily_mileage_km = max(total_mileage_km) - min(total_mileage_km)`
|
||||
- `daily_total_mileage_km = max(total_mileage_km)`
|
||||
|
||||
统计消费者可幂等重放。每个样本 upsert 时只维护当天 `min/max total_mileage_km`,不依赖进程内状态。
|
||||
|
||||
## Redis 实时数据设计
|
||||
|
||||
Redis 保存可从 Kafka 重建的数据。
|
||||
|
||||
| Key | 内容 |
|
||||
|---|---|
|
||||
| `vehicle:online:{vin}` | 在线状态、最后活跃时间、协议列表 |
|
||||
| `vehicle:latest:{vin}` | 合并后的 VIN 最新快照 |
|
||||
| `vehicle:latest:{vin}:{protocol}` | 单协议最新快照 |
|
||||
| `vehicle:last_seen` | VIN 到最后接收时间的 sorted set |
|
||||
|
||||
合并规则:
|
||||
|
||||
- 位置以最新 event_time 为准。
|
||||
- 总里程取最新上报值,不做递增假设。
|
||||
- 32960 多帧字段按 field timestamp 合并。
|
||||
- 808 和 MQTT 不覆盖其他协议独有字段。
|
||||
- Redis TTL 用于在线判断,不作为数据删除依据。
|
||||
|
||||
## 数据流
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
source["车辆 / 平台"] --> gateway["Go gateway"]
|
||||
gateway --> parser["protocol parser"]
|
||||
parser --> identity["identity resolver"]
|
||||
identity --> kafkaRaw["Kafka RAW topics"]
|
||||
identity --> kafkaEvent["Kafka unified event"]
|
||||
kafkaRaw --> history["history-writer"]
|
||||
kafkaRaw --> stats["stat-writer"]
|
||||
kafkaEvent --> realtime["realtime consumer"]
|
||||
history --> td["TDengine raw_frames / locations / mileage_points"]
|
||||
stats --> mysql["MySQL vehicle_daily_metric"]
|
||||
realtime --> redis["Redis latest state"]
|
||||
```
|
||||
|
||||
## 错误处理
|
||||
|
||||
- 协议坏帧仍写 RAW topic,`parse_status=BAD_FRAME`。
|
||||
- 可部分解析的帧写 `parse_status=PARTIAL`,保留 parse error。
|
||||
- Kafka 写失败时接入层先按配置重试;重试耗尽后不写 unified event,必须打错误日志和 metrics。
|
||||
- 启用 `KAFKA_SPOOL_DIR` 后,Kafka 重试耗尽会落本地 spool;如果 spool 写失败,才视为接入层最终失败。
|
||||
- TDengine 写失败不提交 Kafka offset。
|
||||
- MySQL 统计写失败不提交 Kafka offset。
|
||||
- Redis 写失败不影响历史和统计,但要通过 metrics 暴露。
|
||||
- 808 相同 phone 新连接时,旧连接应被替换并记录 latest_seen。
|
||||
|
||||
## 部署设计
|
||||
|
||||
生产 ECS 运行 Go 二进制或容器:
|
||||
|
||||
- `vehicle-gateway`
|
||||
- `vehicle-history-writer`
|
||||
- `vehicle-stat-writer`
|
||||
- `vehicle-realtime-api`
|
||||
|
||||
配置来源:
|
||||
|
||||
- 环境变量优先。
|
||||
- Nacos 可作为后续配置中心,但 Go 第一阶段必须能只靠环境变量启动,便于 Portainer 部署和故障恢复。
|
||||
|
||||
TDengine 连接:
|
||||
|
||||
- 优先 WebSocket `taosWS`。
|
||||
- 若当前 ECS 只开放原生端口,可短期兼容 `taosSql`,但代码配置必须显式标记为兼容模式。
|
||||
|
||||
## 验证计划
|
||||
|
||||
### 本地验证
|
||||
|
||||
- 用样例 808 报文验证:
|
||||
- phone BCD 解析。
|
||||
- 0200 经纬度、速度、方向、时间。
|
||||
- 表 27 `0x01` GPS 总里程。
|
||||
- 用真实 32960 报文验证:
|
||||
- 登录应答正确。
|
||||
- 实时帧和补发帧都能写 RAW。
|
||||
- 位置和总里程字段进入统一 fields。
|
||||
- 用宇通 MQTT payload 验证:
|
||||
- payload 完整保存。
|
||||
- 核心字段归一化。
|
||||
- 不连 Kafka 时可输出 JSON log。
|
||||
- 连接 Kafka 时 RAW topic 可消费到 envelope。
|
||||
|
||||
### ECS 验证
|
||||
|
||||
- 部署前先旁路接收或短窗口切流。
|
||||
- 验证端口:
|
||||
- `32960`
|
||||
- `808`
|
||||
- MQTT 订阅连接。
|
||||
- 验证 Kafka topic 有持续写入。
|
||||
- 验证 TDengine:
|
||||
- `raw_frames` 有三种协议数据。
|
||||
- `vehicle_locations` 能按 VIN、协议、时间分页查询。
|
||||
- `vehicle_mileage_points` 有 32960 和 808 总里程采样。
|
||||
- 验证 MySQL:
|
||||
- `vehicle_daily_metric` 有 32960 和 808 的 `daily_mileage_km`、`daily_total_mileage_km`。
|
||||
- 验证 Redis:
|
||||
- 查询 VIN 在线。
|
||||
- 查询 VIN 合并实时快照。
|
||||
- 查询 VIN 分协议实时快照。
|
||||
|
||||
## 验收标准
|
||||
|
||||
- Go gateway 能在 ECS 上接收真实 32960、808、宇通 MQTT 数据。
|
||||
- 三种协议 RAW 都进入 Kafka。
|
||||
- Kafka 短暂写失败时,gateway 按配置重试;单元测试覆盖首次失败后成功和重试耗尽返回错误。
|
||||
- Kafka 长时间不可用时,gateway 可把 RAW/unified 写入本地 spool;单元测试覆盖 RAW 已落盘时 unified 也落盘、恢复后按顺序补发并删除文件。
|
||||
- 三种协议 RAW 和 parsed JSON 都进入 TDengine。
|
||||
- 32960 和 808 的位置进入 `vehicle_locations`。
|
||||
- 32960 和 808 的总里程采样进入 `vehicle_mileage_points`。
|
||||
- 32960 和 808 的每日里程、每日总里程进入 MySQL 窄表。
|
||||
- Redis 可以查询 VIN 在线和实时数据。
|
||||
- 任一消费者宕机后可通过 Kafka offset 继续恢复。
|
||||
- 查询和统计验证使用 ECS 上的真实数据完成。
|
||||
|
||||
## 实施顺序
|
||||
|
||||
1. 创建 `go/vehicle-gateway` 单 module,迁移现有 Go 原型中可用的解析、Kafka、TDengine、MySQL、Redis 代码。
|
||||
2. 先写协议解析单元测试,覆盖 808 样例报文和 32960 核心字段。
|
||||
3. 完成统一 envelope 和 Kafka sink。
|
||||
4. 完成 TDengine schema bootstrap 与 history writer。
|
||||
5. 完成 MySQL schema bootstrap 与 stat writer。
|
||||
6. 完成 Redis realtime writer/API。
|
||||
7. 本地启动 gateway,用 ECS 转发流量验证。
|
||||
8. 构建 Linux 镜像,部署 ECS。
|
||||
9. 按验收标准逐项验证并记录证据。
|
||||
@@ -5,6 +5,8 @@ The project should stay small at runtime: protocol apps ingest and publish,
|
||||
history consumes and indexes, analytics derives metrics, and business systems
|
||||
read through explicit APIs or Kafka.
|
||||
|
||||
Only supported message backbone: Kafka.
|
||||
|
||||
## Active Scope
|
||||
|
||||
Active production protocols:
|
||||
@@ -15,9 +17,12 @@ Active production protocols:
|
||||
| JT/T 808 | `jt808-ingest-app` | `vehicle.event.jt808.v1` | `vehicle.raw.jt808.v1` |
|
||||
| Yutong MQTT | `yutong-mqtt-app` | `vehicle.event.mqtt-yutong.v1` | `vehicle.raw.mqtt-yutong.v1` |
|
||||
|
||||
Xinda Push is legacy and is not part of the default production surface.
|
||||
It may remain in source for reference, but new optimization work should target
|
||||
the active protocols above.
|
||||
Xinda Push source, Maven profile, deployment bindings, and history consumers
|
||||
are removed. New optimization work should target the active protocols above.
|
||||
|
||||
JSATL12 attachment upload is also outside the default production reactor. Keep
|
||||
it available through the `optional-attachments` profile until an attachment app
|
||||
is explicitly deployed.
|
||||
|
||||
## Goals
|
||||
|
||||
@@ -28,8 +33,7 @@ the active protocols above.
|
||||
- Store complete raw payload metadata and parsed JSON in TDengine `raw_frames`.
|
||||
- Store query-friendly location rows separately from raw payload JSON.
|
||||
- Keep derived statistics in metric tables, not protocol-specific daily tables.
|
||||
- Use Redis only for hot state or streaming calculation state that can be
|
||||
rebuilt from Kafka.
|
||||
- Use Redis only for optional latest-state APIs that can be rebuilt from Kafka.
|
||||
- Keep optional compatibility modules outside the production hot path.
|
||||
|
||||
## Non-Goals
|
||||
@@ -37,9 +41,10 @@ the active protocols above.
|
||||
- Ingest apps do not write business tables.
|
||||
- Ingest apps do not serve history query APIs.
|
||||
- Redis is not the long-term historical store.
|
||||
- `event-file-store` Parquet/DuckDB is not the default history store.
|
||||
- `telemetry_fields` parsing is not handled by this project when a separate
|
||||
field parsing service owns that job.
|
||||
- File-based event indexes are not part of the current build surface.
|
||||
- `telemetry_fields` parsing and field-trend APIs are owned by a separate
|
||||
field parsing service. This project stores RAW parsed JSON plus compact
|
||||
location rows and does not expose a telemetry-fields history API.
|
||||
|
||||
## Runtime Responsibilities
|
||||
|
||||
@@ -66,7 +71,7 @@ repositories, and statistic calculators.
|
||||
|
||||
### Event Contract
|
||||
|
||||
`sink-mq` owns the Kafka protobuf envelope and consumer plumbing.
|
||||
The Kafka sink owns the protobuf envelope and consumer plumbing.
|
||||
Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped
|
||||
envelopes, and downstream failures are converted to structured results and DLQ
|
||||
records instead of blocking a partition.
|
||||
@@ -104,11 +109,9 @@ Production default:
|
||||
|
||||
- `TDENGINE_HISTORY_ENABLED=true` in deployment.
|
||||
- `EVENT_FILE_STORE_ENABLED=false`.
|
||||
- `TDENGINE_TELEMETRY_FIELDS_ENABLED=false`.
|
||||
|
||||
`event-file-store` remains an optional compatibility path for old low-level
|
||||
record APIs or local investigations. It should not be treated as the primary
|
||||
history backend.
|
||||
File-based event indexes have been removed; this app should keep history
|
||||
queries on TDengine and raw replay on `archive://...` references.
|
||||
|
||||
### Analytics App
|
||||
|
||||
@@ -120,22 +123,25 @@ Responsibilities:
|
||||
- Calculate 808 daily mileage from the reported GPS total mileage only:
|
||||
|
||||
```text
|
||||
daily_mileage_km = last_total_mileage_km - first_total_mileage_km
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
calculation_method = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
- Store the metric in `vehicle_stat_metric` with
|
||||
`metric_key = daily_mileage_km`; the first and latest GPS total mileage used
|
||||
for the subtraction stay on the same metric row as calculation source columns.
|
||||
`metric_key = daily_mileage_km`; the local-day minimum and maximum GPS total
|
||||
mileage used for the subtraction stay on the same metric row as calculation
|
||||
source columns.
|
||||
|
||||
There is no JT808-specific daily mileage table and no in-memory production
|
||||
state-store mode. Restart recovery reads the same metric row, so it does not
|
||||
require a separate Redis mileage state.
|
||||
Runtime state: none outside `vehicle_stat_metric`. There is no JT808-specific
|
||||
daily mileage table. Restart recovery reads the same metric row.
|
||||
|
||||
### Latest State
|
||||
|
||||
Module: `vehicle-state-service`
|
||||
|
||||
This module is outside the default production reactor. Build it with
|
||||
`-Poptional-latest-state` when Redis latest-state APIs are explicitly needed.
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume normalized Kafka envelopes.
|
||||
@@ -147,6 +153,8 @@ source of truth for historical queries.
|
||||
|
||||
## Data Flow
|
||||
|
||||
Default production flow:
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
vehicle["Vehicles / Platforms"] --> ingest["Protocol apps"]
|
||||
@@ -154,12 +162,14 @@ flowchart LR
|
||||
ingest --> kafka["Kafka<br/>event + raw envelopes"]
|
||||
kafka --> history["vehicle-history-app"]
|
||||
kafka --> analytics["vehicle-analytics-app"]
|
||||
kafka --> state["vehicle-state-service"]
|
||||
history --> tdengine["TDengine<br/>raw_frames + locations"]
|
||||
analytics --> mysql["MySQL<br/>vehicle_stat_metric"]
|
||||
state --> redisState["Redis<br/>latest state"]
|
||||
```
|
||||
|
||||
Optional latest-state flow, enabled only with `-Poptional-latest-state`, consumes
|
||||
the same Kafka envelopes and rebuilds Redis snapshots outside the default
|
||||
production deployment.
|
||||
|
||||
## Failure Handling
|
||||
|
||||
- Protocol apps continue accepting traffic when history, analytics, or Redis
|
||||
@@ -176,13 +186,17 @@ flowchart LR
|
||||
|
||||
- Active production deployment contains GB32960, JT808, Yutong MQTT,
|
||||
vehicle-history, and vehicle-analytics apps.
|
||||
- Xinda Push is absent from default deployment and history consumer bindings.
|
||||
- History APIs read from TDengine by default; Parquet/DuckDB is optional
|
||||
compatibility only.
|
||||
- Xinda Push source and deployment bindings are removed.
|
||||
- JSATL12 is absent from the default Maven reactor and available through
|
||||
`optional-attachments`.
|
||||
- vehicle-state-service is absent from the default Maven reactor and available
|
||||
through `optional-latest-state`.
|
||||
- raw-archive-store prototype has been removed; raw bytes are written through
|
||||
`sink-archive`.
|
||||
- History APIs read from TDengine; file-based event indexes are not maintained.
|
||||
- Raw frame queries can return complete parsed JSON through `payloadJson.parsed`.
|
||||
- Location queries page over compact rows and reference raw records instead of
|
||||
duplicating full raw JSON.
|
||||
- JT808 daily mileage is stored only in `vehicle_stat_metric`.
|
||||
- Runtime state needed for 808 mileage is also in `vehicle_stat_metric`, with no
|
||||
Redis or production memory state-store mode.
|
||||
- Runtime state needed for 808 mileage is also in `vehicle_stat_metric`.
|
||||
- Build validation covers the active modules and their composition tests.
|
||||
|
||||
29
go/vehicle-gateway/Dockerfile
Normal file
29
go/vehicle-gateway/Dockerfile
Normal file
@@ -0,0 +1,29 @@
|
||||
FROM golang:1.26-bookworm AS build
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/gateway ./cmd/gateway \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/history-writer ./cmd/history-writer \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/stat-writer ./cmd/stat-writer \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/realtime-api ./cmd/realtime-api
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/gateway /app/gateway
|
||||
COPY --from=build /out/history-writer /app/history-writer
|
||||
COPY --from=build /out/stat-writer /app/stat-writer
|
||||
COPY --from=build /out/realtime-api /app/realtime-api
|
||||
|
||||
ENV TZ=Asia/Shanghai
|
||||
|
||||
EXPOSE 808 32960 20210
|
||||
|
||||
CMD ["/app/gateway"]
|
||||
223
go/vehicle-gateway/cmd/gateway/main.go
Normal file
223
go/vehicle-gateway/cmd/gateway/main.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/gateway"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-gateway")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
sink, err := buildSink(ctx, logger)
|
||||
if err != nil {
|
||||
logger.Error("build sink failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer sink.Close()
|
||||
resolver, closeResolver, err := buildIdentityResolver(ctx, logger)
|
||||
if err != nil {
|
||||
logger.Error("build identity resolver failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer closeResolver()
|
||||
|
||||
protocols := []gateway.TCPProtocol{
|
||||
{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: env("GB32960_TCP_ADDR", ":32960"),
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
Respond: gb32960.AutoResponse,
|
||||
},
|
||||
{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Addr: env("JT808_TCP_ADDR", ":808"),
|
||||
Extract: jt808.ExtractFrames,
|
||||
Parse: jt808.ParseFrame,
|
||||
Respond: jt808.NewAutoResponder(env("JT808_REGISTER_AUTH_CODE", "g7gps")).Respond,
|
||||
},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, len(protocols))
|
||||
for _, protocol := range protocols {
|
||||
server, err := gateway.NewTCPServer(gateway.TCPServerConfig{
|
||||
Protocol: protocol,
|
||||
Sink: sink,
|
||||
Resolver: resolver,
|
||||
Logger: logger,
|
||||
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
|
||||
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
|
||||
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := server.ListenAndServe(ctx); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}()
|
||||
}
|
||||
if envBool("YUTONG_MQTT_ENABLED", false) {
|
||||
client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{
|
||||
EndpointName: env("YUTONG_MQTT_ENDPOINT", env("YUTONG_MQTT_ENDPOINT_NAME", "yutong")),
|
||||
Broker: env("YUTONG_MQTT_URI", ""),
|
||||
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
|
||||
Username: env("YUTONG_MQTT_USERNAME", ""),
|
||||
Password: env("YUTONG_MQTT_PASSWORD", ""),
|
||||
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
|
||||
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
|
||||
CleanSession: envBool("YUTONG_MQTT_CLEAN_SESSION", false),
|
||||
KeepAlive: time.Duration(envInt("YUTONG_MQTT_KEEP_ALIVE_SECONDS", 20)) * time.Second,
|
||||
ConnectTimeout: time.Duration(envInt("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", 10)) * time.Second,
|
||||
TLSCACertPath: env("YUTONG_MQTT_TLS_CA_PEM", ""),
|
||||
TLSClientCertPath: env("YUTONG_MQTT_TLS_CLIENT_PEM", ""),
|
||||
TLSClientKeyPath: env("YUTONG_MQTT_TLS_CLIENT_KEY", ""),
|
||||
TLSHostnameVerification: envBool("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", true),
|
||||
Sink: sink,
|
||||
Resolver: resolver,
|
||||
Logger: logger,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("build yutong mqtt client failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := client.Start(ctx); err != nil {
|
||||
logger.Error("start yutong mqtt client failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
logger.Info("yutong mqtt client started")
|
||||
}
|
||||
|
||||
logger.Info("vehicle gateway started")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case err := <-errs:
|
||||
logger.Error("vehicle gateway listener failed", "error", err)
|
||||
stop()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.Resolver, func(), error) {
|
||||
dsn := env("IDENTITY_MYSQL_DSN", strings.TrimSpace(os.Getenv("MYSQL_DSN")))
|
||||
if strings.TrimSpace(dsn) == "" {
|
||||
logger.Warn("identity mysql dsn is empty; using noop identity resolver")
|
||||
return identity.NoopResolver{}, func() {}, nil
|
||||
}
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding")
|
||||
logger.Info("identity mysql resolver enabled", "table", table)
|
||||
return identity.NewMySQLResolver(db, table), func() { _ = db.Close() }, nil
|
||||
}
|
||||
|
||||
func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error) {
|
||||
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
|
||||
if len(brokers) == 0 {
|
||||
logger.Warn("KAFKA_BROKERS is empty; using log sink")
|
||||
return eventbus.NewLogSink(logger), nil
|
||||
}
|
||||
sink, err := eventbus.NewKafkaSink(eventbus.KafkaConfig{
|
||||
Brokers: brokers,
|
||||
RawTopics: map[envelope.Protocol]string{
|
||||
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1"),
|
||||
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1"),
|
||||
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1"),
|
||||
},
|
||||
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1"),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out eventbus.Sink = eventbus.NewRetryingSink(sink, eventbus.RetryConfig{
|
||||
Attempts: envInt("KAFKA_PUBLISH_ATTEMPTS", 3),
|
||||
Backoff: time.Duration(envInt("KAFKA_PUBLISH_BACKOFF_MS", 100)) * time.Millisecond,
|
||||
AttemptTimeout: time.Duration(envInt("KAFKA_PUBLISH_TIMEOUT_MS", 3000)) * time.Millisecond,
|
||||
})
|
||||
spoolDir := strings.TrimSpace(os.Getenv("KAFKA_SPOOL_DIR"))
|
||||
if spoolDir != "" {
|
||||
durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{Directory: spoolDir})
|
||||
interval := time.Duration(envInt("KAFKA_SPOOL_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond
|
||||
go durable.ReplayLoop(ctx, interval, func(err error) {
|
||||
logger.Warn("kafka spool replay failed", "error", err)
|
||||
})
|
||||
logger.Info("kafka durable spool enabled", "dir", spoolDir, "replay_interval_ms", interval.Milliseconds())
|
||||
out = durable
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
var out int
|
||||
for _, r := range value {
|
||||
if r < '0' || r > '9' {
|
||||
return fallback
|
||||
}
|
||||
out = out*10 + int(r-'0')
|
||||
}
|
||||
if out <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func envBool(key string, fallback bool) bool {
|
||||
value := strings.ToLower(strings.TrimSpace(os.Getenv(key)))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value == "1" || value == "true" || value == "yes" || value == "on"
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
123
go/vehicle-gateway/cmd/history-writer/main.go
Normal file
123
go/vehicle-gateway/cmd/history-writer/main.go
Normal file
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
_ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-history-writer")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
cfg := loadConfig()
|
||||
db, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
|
||||
if err != nil {
|
||||
logger.Error("tdengine open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
logger.Error("tdengine ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
writer := history.NewWriter(db)
|
||||
if cfg.EnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
|
||||
logger.Error("tdengine schema bootstrap failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
GroupID: cfg.KafkaGroup,
|
||||
GroupTopics: cfg.KafkaTopics,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
|
||||
logger.Info("history writer started",
|
||||
"driver", cfg.TDengineDriver,
|
||||
"group", cfg.KafkaGroup,
|
||||
"topics", strings.Join(cfg.KafkaTopics, ","))
|
||||
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := writer.AppendAll(ctx, env); err != nil {
|
||||
logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
KafkaBrokers []string
|
||||
KafkaTopics []string
|
||||
KafkaGroup string
|
||||
TDengineDriver string
|
||||
TDengineDSN string
|
||||
TDengineDatabase string
|
||||
EnsureSchema bool
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
return config{
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1")),
|
||||
KafkaGroup: env("KAFKA_GROUP", "go-history-writer"),
|
||||
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
|
||||
TDengineDSN: env("TDENGINE_DSN", ""),
|
||||
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
|
||||
EnsureSchema: env("TDENGINE_ENSURE_SCHEMA", "true") != "false",
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
198
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
198
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
@@ -0,0 +1,198 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/segmentio/kafka-go"
|
||||
_ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-realtime-api")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: env("REDIS_ADDR", "127.0.0.1:6379"),
|
||||
Username: env("REDIS_USERNAME", ""),
|
||||
Password: env("REDIS_PASSWORD", ""),
|
||||
DB: envInt("REDIS_DB", 0),
|
||||
})
|
||||
defer client.Close()
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
logger.Error("redis ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
repository := realtime.NewRepository(client, realtime.Config{
|
||||
OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", 600)) * time.Second,
|
||||
})
|
||||
if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
|
||||
go consumeKafka(ctx, logger, repository, brokers)
|
||||
} else {
|
||||
logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository))
|
||||
closeStats := func() {}
|
||||
if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" {
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
logger.Error("stats mysql open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
logger.Error("stats mysql ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
closeStats = func() { _ = db.Close() }
|
||||
mux.Handle("/api/stats/daily-metrics", stats.NewMetricHandler(stats.NewMetricRepository(db)))
|
||||
logger.Info("stats mysql query enabled")
|
||||
} else {
|
||||
mux.HandleFunc("/api/stats/daily-metrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "MYSQL_DSN is not configured"})
|
||||
})
|
||||
logger.Warn("MYSQL_DSN is empty; stats query api disabled")
|
||||
}
|
||||
defer closeStats()
|
||||
closeHistory := func() {}
|
||||
if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" {
|
||||
driver := env("TDENGINE_DRIVER", "taosWS")
|
||||
db, err := sql.Open(driver, dsn)
|
||||
if err != nil {
|
||||
logger.Error("history tdengine open failed", "driver", driver, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
logger.Error("history tdengine ping failed", "driver", driver, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
closeHistory = func() { _ = db.Close() }
|
||||
database := env("TDENGINE_DATABASE", history.DefaultDatabase)
|
||||
mux.Handle("/api/history/raw-frames", history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)))
|
||||
mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database)))
|
||||
mux.Handle("/api/history/mileage-points", history.NewMileagePointHandler(history.NewMileagePointRepository(db, database)))
|
||||
logger.Info("history query enabled", "driver", driver, "database", database)
|
||||
} else {
|
||||
historyUnavailable := func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"})
|
||||
}
|
||||
mux.HandleFunc("/api/history/raw-frames", historyUnavailable)
|
||||
mux.HandleFunc("/api/history/locations", historyUnavailable)
|
||||
mux.HandleFunc("/api/history/mileage-points", historyUnavailable)
|
||||
logger.Warn("TDENGINE_DSN is empty; history query api disabled")
|
||||
}
|
||||
defer closeHistory()
|
||||
|
||||
server := &http.Server{
|
||||
Addr: env("HTTP_ADDR", ":20210"),
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = server.Shutdown(shutdownCtx)
|
||||
}()
|
||||
|
||||
logger.Info("realtime api started", "addr", server.Addr)
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
logger.Error("http server failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func consumeKafka(ctx context.Context, logger interface {
|
||||
Info(string, ...any)
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, repository *realtime.Repository, brokers []string) {
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: brokers,
|
||||
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
|
||||
GroupTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.event.unified.v1")),
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
logger.Info("realtime kafka consumer started", "topics", env("KAFKA_TOPICS", "vehicle.event.unified.v1"))
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := repository.Update(ctx, env); err != nil {
|
||||
logger.Error("redis realtime update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
122
go/vehicle-gateway/cmd/stat-writer/main.go
Normal file
122
go/vehicle-gateway/cmd/stat-writer/main.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-stat-writer")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
cfg := loadConfig()
|
||||
db, err := sql.Open("mysql", cfg.MySQLDSN)
|
||||
if err != nil {
|
||||
logger.Error("mysql open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
logger.Error("mysql ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
writer := stats.NewWriter(db, cfg.Location)
|
||||
if cfg.EnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx); err != nil {
|
||||
logger.Error("mysql schema bootstrap failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
GroupID: cfg.KafkaGroup,
|
||||
GroupTopics: cfg.KafkaTopics,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
|
||||
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","))
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := writer.Append(ctx, env); err != nil {
|
||||
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
KafkaBrokers []string
|
||||
KafkaTopics []string
|
||||
KafkaGroup string
|
||||
MySQLDSN string
|
||||
EnsureSchema bool
|
||||
Location *time.Location
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
loc, err := time.LoadLocation(env("LOCAL_TZ", "Asia/Shanghai"))
|
||||
if err != nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
return config{
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1")),
|
||||
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
|
||||
MySQLDSN: env("MYSQL_DSN", ""),
|
||||
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",
|
||||
Location: loc,
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
30
go/vehicle-gateway/go.mod
Normal file
30
go/vehicle-gateway/go.mod
Normal file
@@ -0,0 +1,30 @@
|
||||
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/alicebob/miniredis/v2 v2.35.0
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/taosdata/driver-go/v3 v3.8.1
|
||||
golang.org/x/text v0.29.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.15.9 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
github.com/yuin/gopher-lua v1.1.1 // indirect
|
||||
golang.org/x/net v0.44.0 // indirect
|
||||
golang.org/x/sync v0.17.0 // indirect
|
||||
)
|
||||
73
go/vehicle-gateway/go.sum
Normal file
73
go/vehicle-gateway/go.sum
Normal file
@@ -0,0 +1,73 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE=
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
|
||||
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
||||
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/taosdata/driver-go/v3 v3.8.1 h1:kkd4ABsGiU+oXDbsw/sic985LKAvnpF9Gb/TEunTnLE=
|
||||
github.com/taosdata/driver-go/v3 v3.8.1/go.mod h1:S6OGOinfR0xxxaMGsvBi9cLkYxEIW1p6qqr8QJATTlg=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
|
||||
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
95
go/vehicle-gateway/internal/envelope/envelope.go
Normal file
95
go/vehicle-gateway/internal/envelope/envelope.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Protocol string
|
||||
|
||||
const (
|
||||
ProtocolGB32960 Protocol = "GB32960"
|
||||
ProtocolJT808 Protocol = "JT808"
|
||||
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||
)
|
||||
|
||||
type ParseStatus string
|
||||
|
||||
const (
|
||||
ParseOK ParseStatus = "OK"
|
||||
ParsePartial ParseStatus = "PARTIAL"
|
||||
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||
)
|
||||
|
||||
const (
|
||||
FieldSpeedKMH = "speed_kmh"
|
||||
FieldTotalMileageKM = "total_mileage_km"
|
||||
FieldLongitude = "longitude"
|
||||
FieldLatitude = "latitude"
|
||||
FieldSOCPercent = "soc_percent"
|
||||
)
|
||||
|
||||
type FrameEnvelope struct {
|
||||
EventID string `json:"event_id"`
|
||||
TraceID string `json:"trace_id"`
|
||||
Protocol Protocol `json:"protocol"`
|
||||
MessageID string `json:"message_id"`
|
||||
Sequence uint16 `json:"sequence"`
|
||||
VIN string `json:"vin,omitempty"`
|
||||
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
Parsed map[string]any `json:"parsed,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
ParseStatus ParseStatus `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) VehicleKey() string {
|
||||
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
return string(e.Protocol) + ":unknown"
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) StableEventID() string {
|
||||
if strings.TrimSpace(e.EventID) != "" {
|
||||
return e.EventID
|
||||
}
|
||||
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||
sum := sha256.Sum256([]byte(input))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) KafkaKey() []byte {
|
||||
return []byte(e.VehicleKey())
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||
if e.EventID == "" {
|
||||
e.EventID = e.StableEventID()
|
||||
}
|
||||
if e.ParseStatus == "" {
|
||||
e.ParseStatus = ParseOK
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
55
go/vehicle-gateway/internal/envelope/envelope_test.go
Normal file
55
go/vehicle-gateway/internal/envelope/envelope_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||
e := FrameEnvelope{
|
||||
Protocol: ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013307795425",
|
||||
Sequence: 1,
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7e02000000ff7e",
|
||||
}
|
||||
a := e.StableEventID()
|
||||
b := e.StableEventID()
|
||||
if a == "" || a != b {
|
||||
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeMarshalDefaults(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolGB32960, VIN: "LNBVIN00000000002", MessageID: "0x02"}
|
||||
payload, err := e.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalJSONBytes() error = %v", err)
|
||||
}
|
||||
var decoded FrameEnvelope
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if decoded.EventID == "" {
|
||||
t.Fatal("event id should be filled")
|
||||
}
|
||||
if decoded.ParseStatus != ParseOK {
|
||||
t.Fatalf("parse status = %q", decoded.ParseStatus)
|
||||
}
|
||||
}
|
||||
187
go/vehicle-gateway/internal/eventbus/durable_sink.go
Normal file
187
go/vehicle-gateway/internal/eventbus/durable_sink.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type DurableConfig struct {
|
||||
Directory string
|
||||
}
|
||||
|
||||
type DurableSink struct {
|
||||
delegate Sink
|
||||
dir string
|
||||
|
||||
mu sync.Mutex
|
||||
seq uint64
|
||||
rawPending map[string]struct{}
|
||||
}
|
||||
|
||||
type durableRecord struct {
|
||||
Kind string `json:"kind"`
|
||||
Envelope envelope.FrameEnvelope `json:"envelope"`
|
||||
}
|
||||
|
||||
func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
|
||||
if delegate == nil {
|
||||
panic("durable delegate sink must not be nil")
|
||||
}
|
||||
return &DurableSink{
|
||||
delegate: delegate,
|
||||
dir: strings.TrimSpace(cfg.Directory),
|
||||
rawPending: map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if err := s.delegate.PublishRaw(ctx, env); err == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.spool("raw", env); err != nil {
|
||||
return err
|
||||
}
|
||||
s.markRawPending(env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DurableSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if s.isRawPending(env) {
|
||||
return s.spool("unified", env)
|
||||
}
|
||||
if err := s.delegate.PublishUnified(ctx, env); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.spool("unified", env)
|
||||
}
|
||||
|
||||
func (s *DurableSink) ReplayOnce(ctx context.Context) error {
|
||||
files, err := filepath.Glob(filepath.Join(s.dir, "*.json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(files)
|
||||
for _, file := range files {
|
||||
record, err := readDurableRecord(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.publishRecord(ctx, record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(file); err != nil {
|
||||
return err
|
||||
}
|
||||
if record.Kind == "raw" {
|
||||
s.clearRawPending(record.Envelope)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DurableSink) ReplayLoop(ctx context.Context, interval time.Duration, onError func(error)) {
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := s.ReplayOnce(ctx); err != nil && onError != nil {
|
||||
onError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) Close() error {
|
||||
return s.delegate.Close()
|
||||
}
|
||||
|
||||
func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) error {
|
||||
switch record.Kind {
|
||||
case "raw":
|
||||
return s.delegate.PublishRaw(ctx, record.Envelope)
|
||||
case "unified":
|
||||
return s.delegate.PublishUnified(ctx, record.Envelope)
|
||||
default:
|
||||
return fmt.Errorf("unknown durable record kind %q", record.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
|
||||
if s.dir == "" {
|
||||
return fmt.Errorf("durable spool directory is empty")
|
||||
}
|
||||
if env.EventID == "" {
|
||||
env.EventID = env.StableEventID()
|
||||
}
|
||||
if env.ParseStatus == "" {
|
||||
env.ParseStatus = envelope.ParseOK
|
||||
}
|
||||
if err := os.MkdirAll(s.dir, 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := s.nextFileName(env, kind)
|
||||
path := filepath.Join(s.dir, name)
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, payload, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.seq++
|
||||
eventID := env.StableEventID()
|
||||
if len(eventID) > 12 {
|
||||
eventID = eventID[:12]
|
||||
}
|
||||
return fmt.Sprintf("%020d-%06d-%s-%s.json", time.Now().UnixNano(), s.seq, kind, eventID)
|
||||
}
|
||||
|
||||
func (s *DurableSink) markRawPending(env envelope.FrameEnvelope) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.rawPending[env.StableEventID()] = struct{}{}
|
||||
}
|
||||
|
||||
func (s *DurableSink) clearRawPending(env envelope.FrameEnvelope) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.rawPending, env.StableEventID())
|
||||
}
|
||||
|
||||
func (s *DurableSink) isRawPending(env envelope.FrameEnvelope) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, ok := s.rawPending[env.StableEventID()]
|
||||
return ok
|
||||
}
|
||||
|
||||
func readDurableRecord(path string) (durableRecord, error) {
|
||||
var record durableRecord
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return record, err
|
||||
}
|
||||
return record, json.Unmarshal(payload, &record)
|
||||
}
|
||||
153
go/vehicle-gateway/internal/eventbus/durable_sink_test.go
Normal file
153
go/vehicle-gateway/internal/eventbus/durable_sink_test.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
|
||||
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
|
||||
env := durableTestEnvelope()
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
|
||||
if delegate.rawCalls != 1 {
|
||||
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
|
||||
}
|
||||
if delegate.unifiedCalls != 0 {
|
||||
t.Fatalf("unified should not be delegated while raw is spooled, calls = %d", delegate.unifiedCalls)
|
||||
}
|
||||
files := spoolFiles(t, dir)
|
||||
if len(files) != 2 {
|
||||
t.Fatalf("spool files = %d, want 2: %#v", len(files), files)
|
||||
}
|
||||
assertSpoolKind(t, files[0], "raw")
|
||||
assertSpoolKind(t, files[1], "unified")
|
||||
}
|
||||
|
||||
func TestDurableSinkReplayPublishesInFileOrderAndDeletesFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
|
||||
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
|
||||
env := durableTestEnvelope()
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
delegate.rawErrors = nil
|
||||
|
||||
if err := sink.ReplayOnce(context.Background()); err != nil {
|
||||
t.Fatalf("ReplayOnce() error = %v", err)
|
||||
}
|
||||
|
||||
got := delegate.calls
|
||||
want := []string{"raw", "raw", "unified"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("calls = %#v, want %#v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("calls[%d] = %q, want %q; all=%#v", i, got[i], want[i], got)
|
||||
}
|
||||
}
|
||||
if files := spoolFiles(t, dir); len(files) != 0 {
|
||||
t.Fatalf("spool files after replay = %#v, want none", files)
|
||||
}
|
||||
}
|
||||
|
||||
func durableTestEnvelope() envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013079963379",
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
Sequence: 183,
|
||||
EventTimeMS: 1782903940000,
|
||||
ReceivedAtMS: 1782917751903,
|
||||
RawHex: "0200002201307996337900B7",
|
||||
}
|
||||
}
|
||||
|
||||
func spoolFiles(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
matches, err := filepath.Glob(filepath.Join(dir, "*.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob spool files: %v", err)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
func assertSpoolKind(t *testing.T, path string, kind string) {
|
||||
t.Helper()
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read spool file: %v", err)
|
||||
}
|
||||
if !containsString(string(payload), `"kind":"`+kind+`"`) {
|
||||
t.Fatalf("spool file %s payload = %s, want kind %q", path, payload, kind)
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(value string, needle string) bool {
|
||||
return len(needle) == 0 || (len(value) >= len(needle) && indexString(value, needle) >= 0)
|
||||
}
|
||||
|
||||
func indexString(value string, needle string) int {
|
||||
for i := 0; i+len(needle) <= len(value); i++ {
|
||||
if value[i:i+len(needle)] == needle {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
var errSpoolTest = errors.New("delegate unavailable")
|
||||
|
||||
type scriptedSink struct {
|
||||
rawErrors []error
|
||||
unifiedErrors []error
|
||||
rawCalls int
|
||||
unifiedCalls int
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (s *scriptedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
s.rawCalls++
|
||||
s.calls = append(s.calls, "raw")
|
||||
if len(s.rawErrors) > 0 {
|
||||
err := s.rawErrors[0]
|
||||
s.rawErrors = s.rawErrors[1:]
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scriptedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
s.unifiedCalls++
|
||||
s.calls = append(s.calls, "unified")
|
||||
if len(s.unifiedErrors) > 0 {
|
||||
err := s.unifiedErrors[0]
|
||||
s.unifiedErrors = s.unifiedErrors[1:]
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scriptedSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
94
go/vehicle-gateway/internal/eventbus/kafka_sink.go
Normal file
94
go/vehicle-gateway/internal/eventbus/kafka_sink.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Sink interface {
|
||||
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type KafkaSink struct {
|
||||
writer kafkaWriter
|
||||
rawTopics map[envelope.Protocol]string
|
||||
unifiedTopic string
|
||||
}
|
||||
|
||||
type kafkaWriter interface {
|
||||
WriteMessages(context.Context, ...kafka.Message) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type KafkaConfig struct {
|
||||
Brokers []string
|
||||
RawTopics map[envelope.Protocol]string
|
||||
UnifiedTopic string
|
||||
}
|
||||
|
||||
func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) {
|
||||
if len(cfg.Brokers) == 0 {
|
||||
return nil, errors.New("kafka brokers are required")
|
||||
}
|
||||
return newKafkaSinkWithWriter(&kafka.Writer{
|
||||
Addr: kafka.TCP(cfg.Brokers...),
|
||||
Balancer: &kafka.Hash{},
|
||||
AllowAutoTopicCreation: false,
|
||||
}, cfg), nil
|
||||
}
|
||||
|
||||
func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
|
||||
rawTopics := map[envelope.Protocol]string{
|
||||
envelope.ProtocolGB32960: "vehicle.raw.gb32960.v1",
|
||||
envelope.ProtocolJT808: "vehicle.raw.jt808.v1",
|
||||
envelope.ProtocolYutongMQTT: "vehicle.raw.yutong-mqtt.v1",
|
||||
}
|
||||
for protocol, topic := range cfg.RawTopics {
|
||||
if topic != "" {
|
||||
rawTopics[protocol] = topic
|
||||
}
|
||||
}
|
||||
unifiedTopic := cfg.UnifiedTopic
|
||||
if unifiedTopic == "" {
|
||||
unifiedTopic = "vehicle.event.unified.v1"
|
||||
}
|
||||
return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic}
|
||||
}
|
||||
|
||||
func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
topic, ok := s.rawTopics[env.Protocol]
|
||||
if !ok || topic == "" {
|
||||
return fmt.Errorf("raw topic not configured for protocol %s", env.Protocol)
|
||||
}
|
||||
return s.publish(ctx, topic, env)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.publish(ctx, s.unifiedTopic, env)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) Close() error {
|
||||
if s == nil || s.writer == nil {
|
||||
return nil
|
||||
}
|
||||
return s.writer.Close()
|
||||
}
|
||||
|
||||
func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.writer.WriteMessages(ctx, kafka.Message{
|
||||
Topic: topic,
|
||||
Key: env.KafkaKey(),
|
||||
Value: payload,
|
||||
})
|
||||
}
|
||||
77
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
Normal file
77
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestKafkaSinkRoutesRawTopics(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "013307795425", MessageID: "0x0200"}
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if len(writer.messages) != 1 {
|
||||
t.Fatalf("messages = %d", len(writer.messages))
|
||||
}
|
||||
msg := writer.messages[0]
|
||||
if msg.Topic != "vehicle.raw.jt808.v1" {
|
||||
t.Fatalf("topic = %q", msg.Topic)
|
||||
}
|
||||
if string(msg.Key) != "JT808:013307795425" {
|
||||
t.Fatalf("key = %q", string(msg.Key))
|
||||
}
|
||||
var decoded envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(msg.Value, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if decoded.EventID == "" || decoded.ParseStatus != envelope.ParseOK {
|
||||
t.Fatalf("unexpected payload defaults: %#v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKafkaSinkRoutesUnifiedTopic(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{UnifiedTopic: "custom.unified"})
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000001", MessageID: "0x02"}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
if writer.messages[0].Topic != "custom.unified" {
|
||||
t.Fatalf("topic = %q", writer.messages[0].Topic)
|
||||
}
|
||||
if string(writer.messages[0].Key) != "LNBVIN00000000001" {
|
||||
t.Fatalf("key = %q", string(writer.messages[0].Key))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKafkaSinkRejectsUnknownProtocol(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")})
|
||||
if err == nil {
|
||||
t.Fatal("expected unknown protocol error")
|
||||
}
|
||||
}
|
||||
|
||||
type recordingWriter struct {
|
||||
messages []kafka.Message
|
||||
}
|
||||
|
||||
func (w *recordingWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
|
||||
w.messages = append(w.messages, messages...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *recordingWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
33
go/vehicle-gateway/internal/eventbus/log_sink.go
Normal file
33
go/vehicle-gateway/internal/eventbus/log_sink.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type LogSink struct {
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewLogSink(logger *slog.Logger) *LogSink {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &LogSink{logger: logger}
|
||||
}
|
||||
|
||||
func (s *LogSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.logger.Info("raw envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID, "status", env.ParseStatus)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LogSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.logger.Info("unified envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LogSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
80
go/vehicle-gateway/internal/eventbus/retry_sink.go
Normal file
80
go/vehicle-gateway/internal/eventbus/retry_sink.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type RetryConfig struct {
|
||||
Attempts int
|
||||
Backoff time.Duration
|
||||
AttemptTimeout time.Duration
|
||||
}
|
||||
|
||||
type RetryingSink struct {
|
||||
delegate Sink
|
||||
cfg RetryConfig
|
||||
}
|
||||
|
||||
func NewRetryingSink(delegate Sink, cfg RetryConfig) *RetryingSink {
|
||||
if delegate == nil {
|
||||
panic("retry delegate sink must not be nil")
|
||||
}
|
||||
if cfg.Attempts <= 0 {
|
||||
cfg.Attempts = 1
|
||||
}
|
||||
return &RetryingSink{delegate: delegate, cfg: cfg}
|
||||
}
|
||||
|
||||
func (s *RetryingSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.withRetry(ctx, func(attemptCtx context.Context) error {
|
||||
return s.delegate.PublishRaw(attemptCtx, env)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *RetryingSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.withRetry(ctx, func(attemptCtx context.Context) error {
|
||||
return s.delegate.PublishUnified(attemptCtx, env)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *RetryingSink) Close() error {
|
||||
return s.delegate.Close()
|
||||
}
|
||||
|
||||
func (s *RetryingSink) withRetry(ctx context.Context, publish func(context.Context) error) error {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= s.cfg.Attempts; attempt++ {
|
||||
attemptCtx := ctx
|
||||
cancel := func() {}
|
||||
if s.cfg.AttemptTimeout > 0 {
|
||||
attemptCtx, cancel = context.WithTimeout(ctx, s.cfg.AttemptTimeout)
|
||||
}
|
||||
err := publish(attemptCtx)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
lastErr = err
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if attempt == s.cfg.Attempts {
|
||||
break
|
||||
}
|
||||
if s.cfg.Backoff > 0 {
|
||||
timer := time.NewTimer(s.cfg.Backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
80
go/vehicle-gateway/internal/eventbus/retry_sink_test.go
Normal file
80
go/vehicle-gateway/internal/eventbus/retry_sink_test.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestRetryingSinkRetriesRawUntilSuccess(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 1}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 3})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if delegate.rawCalls != 2 {
|
||||
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryingSinkReturnsLastErrorAfterAttempts(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 3}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 2})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if !errors.Is(err, errFlakyPublish) {
|
||||
t.Fatalf("PublishRaw() error = %v, want %v", err, errFlakyPublish)
|
||||
}
|
||||
if delegate.rawCalls != 2 {
|
||||
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryingSinkStopsWhenContextCanceledDuringBackoff(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 10}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 5, Backoff: time.Minute})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
err := sink.PublishRaw(ctx, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("PublishRaw() error = %v, want context canceled", err)
|
||||
}
|
||||
if delegate.rawCalls != 1 {
|
||||
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
var errFlakyPublish = errors.New("publish failed")
|
||||
|
||||
type flakySink struct {
|
||||
rawFailures int
|
||||
unifiedFailures int
|
||||
rawCalls int
|
||||
unifiedCalls int
|
||||
}
|
||||
|
||||
func (s *flakySink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
s.rawCalls++
|
||||
if s.rawCalls <= s.rawFailures {
|
||||
return errFlakyPublish
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *flakySink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
s.unifiedCalls++
|
||||
if s.unifiedCalls <= s.unifiedFailures {
|
||||
return errFlakyPublish
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *flakySink) Close() error {
|
||||
return nil
|
||||
}
|
||||
218
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
218
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
@@ -0,0 +1,218 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
|
||||
)
|
||||
|
||||
type MQTTClientConfig struct {
|
||||
EndpointName string
|
||||
Broker string
|
||||
ClientID string
|
||||
Username string
|
||||
Password string
|
||||
Topics []string
|
||||
QoS byte
|
||||
CleanSession bool
|
||||
KeepAlive time.Duration
|
||||
ConnectTimeout time.Duration
|
||||
TLSCACertPath string
|
||||
TLSClientCertPath string
|
||||
TLSClientKeyPath string
|
||||
TLSHostnameVerification bool
|
||||
Sink eventbus.Sink
|
||||
Resolver identity.Resolver
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
type MQTTClient struct {
|
||||
cfg MQTTClientConfig
|
||||
client mqtt.Client
|
||||
}
|
||||
|
||||
func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
|
||||
if strings.TrimSpace(cfg.Broker) == "" {
|
||||
return nil, errors.New("mqtt broker is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ClientID) == "" {
|
||||
return nil, errors.New("mqtt client id is required")
|
||||
}
|
||||
if len(cfg.Topics) == 0 {
|
||||
return nil, errors.New("mqtt topics are required")
|
||||
}
|
||||
if cfg.Sink == nil {
|
||||
return nil, errors.New("sink is required")
|
||||
}
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
if cfg.Resolver == nil {
|
||||
cfg.Resolver = identity.NoopResolver{}
|
||||
}
|
||||
if cfg.EndpointName == "" {
|
||||
cfg.EndpointName = "yutong"
|
||||
}
|
||||
return &MQTTClient{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) Start(ctx context.Context) error {
|
||||
opts, err := c.buildOptions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.client = mqtt.NewClient(opts)
|
||||
token := c.client.Connect()
|
||||
if token.Wait() && token.Error() != nil {
|
||||
return token.Error()
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
if c.client != nil && c.client.IsConnected() {
|
||||
c.client.Disconnect(250)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) buildOptions(ctx context.Context) (*mqtt.ClientOptions, error) {
|
||||
keepAlive := c.cfg.KeepAlive
|
||||
if keepAlive <= 0 {
|
||||
keepAlive = 20 * time.Second
|
||||
}
|
||||
connectTimeout := c.cfg.ConnectTimeout
|
||||
if connectTimeout <= 0 {
|
||||
connectTimeout = 10 * time.Second
|
||||
}
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker(c.cfg.Broker).
|
||||
SetClientID(c.cfg.ClientID).
|
||||
SetUsername(c.cfg.Username).
|
||||
SetPassword(c.cfg.Password).
|
||||
SetCleanSession(c.cfg.CleanSession).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectRetry(true).
|
||||
SetConnectRetryInterval(5 * time.Second).
|
||||
SetResumeSubs(true).
|
||||
SetOrderMatters(false).
|
||||
SetKeepAlive(keepAlive).
|
||||
SetConnectTimeout(connectTimeout)
|
||||
|
||||
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
|
||||
c.handleMessage(ctx, message.Topic(), message.Payload())
|
||||
})
|
||||
opts.OnConnect = func(client mqtt.Client) {
|
||||
for _, topic := range c.cfg.Topics {
|
||||
token := client.Subscribe(topic, c.cfg.QoS, nil)
|
||||
if token.Wait() && token.Error() != nil {
|
||||
c.cfg.Logger.Error("mqtt subscribe failed", "broker", c.cfg.Broker, "topic", topic, "error", token.Error())
|
||||
continue
|
||||
}
|
||||
c.cfg.Logger.Info("mqtt subscribed", "broker", c.cfg.Broker, "topic", topic, "qos", c.cfg.QoS)
|
||||
}
|
||||
}
|
||||
opts.OnConnectionLost = func(_ mqtt.Client, err error) {
|
||||
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
|
||||
}
|
||||
|
||||
tlsConfig, err := c.buildTLSConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tlsConfig != nil {
|
||||
opts.SetTLSConfig(tlsConfig)
|
||||
}
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
|
||||
caPath := strings.TrimSpace(c.cfg.TLSCACertPath)
|
||||
certPath := strings.TrimSpace(c.cfg.TLSClientCertPath)
|
||||
keyPath := strings.TrimSpace(c.cfg.TLSClientKeyPath)
|
||||
if caPath == "" && certPath == "" && keyPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
config := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: !c.cfg.TLSHostnameVerification,
|
||||
}
|
||||
if caPath != "" {
|
||||
caPEM, err := os.ReadFile(caPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read mqtt ca certificate: %w", err)
|
||||
}
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("parse mqtt ca certificate %s", caPath)
|
||||
}
|
||||
config.RootCAs = roots
|
||||
}
|
||||
if certPath != "" || keyPath != "" {
|
||||
if certPath == "" || keyPath == "" {
|
||||
return nil, errors.New("mqtt client certificate and key must be configured together")
|
||||
}
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load mqtt client certificate: %w", err)
|
||||
}
|
||||
config.Certificates = []tls.Certificate{cert}
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := yutongmqtt.ParseMessage(c.cfg.EndpointName, topic, payload, receivedAtMS)
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
MessageID: "MQTT",
|
||||
VehicleKeyHint: "unknown",
|
||||
SourceEndpoint: "mqtt://" + c.cfg.EndpointName + topic,
|
||||
EventTimeMS: receivedAtMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawText: string(payload),
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(payload)),
|
||||
ParseStatus: envelope.ParseBadFrame,
|
||||
ParseError: err.Error(),
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
} else {
|
||||
resolved, resolveErr := c.cfg.Resolver.Resolve(ctx, env)
|
||||
if resolveErr != nil {
|
||||
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
|
||||
env.ParseStatus = envelope.ParsePartial
|
||||
} else {
|
||||
env = resolved
|
||||
}
|
||||
}
|
||||
if err := c.cfg.Sink.PublishRaw(ctx, env); err != nil {
|
||||
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return
|
||||
}
|
||||
if err := c.cfg.Sink.PublishUnified(ctx, env); err != nil {
|
||||
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||
}
|
||||
}
|
||||
183
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
183
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "tcp://127.0.0.1:1883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
|
||||
"device":"LTEST000000000001",
|
||||
"time":"20260413100000",
|
||||
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
|
||||
}`))
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
|
||||
t.Fatalf("unexpected raw envelope: %#v", sink.raw[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "tcp://127.0.0.1:1883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
client.handleMessage(context.Background(), "/ytforward/shln/bad", []byte("{bad-json"))
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "ssl://mqtt.example.test:8883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
QoS: 1,
|
||||
Sink: &recordingSink{},
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
TLSCACertPath: caPath,
|
||||
TLSClientCertPath: certPath,
|
||||
TLSClientKeyPath: keyPath,
|
||||
TLSHostnameVerification: false,
|
||||
CleanSession: true,
|
||||
KeepAlive: 20 * time.Second,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
opts, err := client.buildOptions(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("buildOptions() error = %v", err)
|
||||
}
|
||||
if opts.TLSConfig == nil {
|
||||
t.Fatal("TLSConfig is nil")
|
||||
}
|
||||
if opts.TLSConfig.RootCAs == nil {
|
||||
t.Fatal("RootCAs is nil")
|
||||
}
|
||||
if len(opts.TLSConfig.Certificates) != 1 {
|
||||
t.Fatalf("client certificates = %d, want 1", len(opts.TLSConfig.Certificates))
|
||||
}
|
||||
if !opts.TLSConfig.InsecureSkipVerify {
|
||||
t.Fatal("InsecureSkipVerify should be true when hostname verification is disabled")
|
||||
}
|
||||
if !opts.CleanSession {
|
||||
t.Fatal("CleanSession should be true")
|
||||
}
|
||||
if opts.Order {
|
||||
t.Fatal("OrderMatters should be false so MQTT network handling is not blocked by Kafka/DB work")
|
||||
}
|
||||
if !opts.ResumeSubs {
|
||||
t.Fatal("ResumeSubs should be true to avoid subscribe failures during reconnect churn")
|
||||
}
|
||||
if got := opts.KeepAlive; got != 20 {
|
||||
t.Fatalf("KeepAlive = %d, want 20", got)
|
||||
}
|
||||
if got := opts.ConnectTimeout; got != 10*time.Second {
|
||||
t.Fatalf("ConnectTimeout = %v, want 10s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestTLSMaterial(t *testing.T, dir string) (string, string, string) {
|
||||
t.Helper()
|
||||
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate ca key: %v", err)
|
||||
}
|
||||
caTemplate := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "test-ca"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
}
|
||||
caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
|
||||
if err != nil {
|
||||
t.Fatalf("create ca cert: %v", err)
|
||||
}
|
||||
clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate client key: %v", err)
|
||||
}
|
||||
clientTemplate := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(2),
|
||||
Subject: pkix.Name{CommonName: "test-client"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
||||
}
|
||||
clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey)
|
||||
if err != nil {
|
||||
t.Fatalf("create client cert: %v", err)
|
||||
}
|
||||
caPath := filepath.Join(dir, "ca.pem")
|
||||
certPath := filepath.Join(dir, "client.pem")
|
||||
keyPath := filepath.Join(dir, "client-key.pem")
|
||||
writePEM(t, caPath, "CERTIFICATE", caDER)
|
||||
writePEM(t, certPath, "CERTIFICATE", clientDER)
|
||||
keyDER := x509.MarshalPKCS1PrivateKey(clientKey)
|
||||
writePEM(t, keyPath, "RSA PRIVATE KEY", keyDER)
|
||||
return caPath, certPath, keyPath
|
||||
}
|
||||
|
||||
func writePEM(t *testing.T, path, typ string, der []byte) {
|
||||
t.Helper()
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create %s: %v", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err := pem.Encode(file, &pem.Block{Type: typ, Bytes: der}); err != nil {
|
||||
t.Fatalf("write pem %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
238
go/vehicle-gateway/internal/gateway/tcp_server.go
Normal file
238
go/vehicle-gateway/internal/gateway/tcp_server.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
)
|
||||
|
||||
type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
|
||||
|
||||
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
|
||||
|
||||
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
|
||||
|
||||
type TCPProtocol struct {
|
||||
Protocol envelope.Protocol
|
||||
Addr string
|
||||
Extract FrameExtractor
|
||||
Parse FrameParser
|
||||
Respond FrameResponder
|
||||
}
|
||||
|
||||
type TCPServer struct {
|
||||
protocol TCPProtocol
|
||||
sink eventbus.Sink
|
||||
resolver identity.Resolver
|
||||
logger *slog.Logger
|
||||
readBufferSize int
|
||||
idleTimeout time.Duration
|
||||
maxConnections int
|
||||
}
|
||||
|
||||
type TCPServerConfig struct {
|
||||
Protocol TCPProtocol
|
||||
Sink eventbus.Sink
|
||||
Resolver identity.Resolver
|
||||
Logger *slog.Logger
|
||||
ReadBufferSize int
|
||||
IdleTimeout time.Duration
|
||||
MaxConnections int
|
||||
}
|
||||
|
||||
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
|
||||
if cfg.Protocol.Protocol == "" {
|
||||
return nil, errors.New("protocol is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Protocol.Addr) == "" {
|
||||
return nil, errors.New("listen addr is required")
|
||||
}
|
||||
if cfg.Protocol.Extract == nil {
|
||||
return nil, errors.New("frame extractor is required")
|
||||
}
|
||||
if cfg.Protocol.Parse == nil {
|
||||
return nil, errors.New("frame parser is required")
|
||||
}
|
||||
if cfg.Sink == nil {
|
||||
return nil, errors.New("sink is required")
|
||||
}
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
if cfg.Resolver == nil {
|
||||
cfg.Resolver = identity.NoopResolver{}
|
||||
}
|
||||
if cfg.ReadBufferSize <= 0 {
|
||||
cfg.ReadBufferSize = 32 * 1024
|
||||
}
|
||||
if cfg.IdleTimeout <= 0 {
|
||||
cfg.IdleTimeout = 2 * time.Minute
|
||||
}
|
||||
if cfg.MaxConnections <= 0 {
|
||||
cfg.MaxConnections = 10_000
|
||||
}
|
||||
return &TCPServer{
|
||||
protocol: cfg.Protocol,
|
||||
sink: cfg.Sink,
|
||||
resolver: cfg.Resolver,
|
||||
logger: cfg.Logger,
|
||||
readBufferSize: cfg.ReadBufferSize,
|
||||
idleTimeout: cfg.IdleTimeout,
|
||||
maxConnections: cfg.MaxConnections,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *TCPServer) ListenAndServe(ctx context.Context) error {
|
||||
var lc net.ListenConfig
|
||||
listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
_ = listener.Close()
|
||||
}()
|
||||
|
||||
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
|
||||
sem := make(chan struct{}, s.maxConnections)
|
||||
var wg sync.WaitGroup
|
||||
defer wg.Wait()
|
||||
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
s.handleConnection(ctx, conn)
|
||||
}()
|
||||
default:
|
||||
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
||||
defer conn.Close()
|
||||
source := conn.RemoteAddr().String()
|
||||
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
|
||||
log.Info("tcp connection opened")
|
||||
defer log.Info("tcp connection closed")
|
||||
|
||||
readBuffer := make([]byte, s.readBufferSize)
|
||||
var pending []byte
|
||||
for {
|
||||
_ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout))
|
||||
n, err := conn.Read(readBuffer)
|
||||
if n > 0 {
|
||||
pending = append(pending, readBuffer[:n]...)
|
||||
frames, remainder, extractErr := s.protocol.Extract(pending)
|
||||
if extractErr != nil {
|
||||
log.Warn("frame extraction failed", "error", extractErr)
|
||||
return
|
||||
}
|
||||
pending = remainder
|
||||
for _, frame := range frames {
|
||||
s.handleFrame(ctx, conn, frame, source)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
log.Warn("tcp connection idle timeout")
|
||||
return
|
||||
}
|
||||
log.Warn("tcp read failed", "error", err)
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string) {
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := s.protocol.Parse(raw, receivedAtMS, source)
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: s.protocol.Protocol,
|
||||
SourceEndpoint: source,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
EventTimeMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
ParseStatus: envelope.ParseBadFrame,
|
||||
ParseError: err.Error(),
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
} else {
|
||||
resolved, resolveErr := s.resolver.Resolve(ctx, env)
|
||||
if resolveErr != nil {
|
||||
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
|
||||
env.ParseStatus = envelope.ParsePartial
|
||||
} else {
|
||||
env = resolved
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.sink.PublishRaw(ctx, env); err != nil {
|
||||
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return
|
||||
}
|
||||
if err := s.sink.PublishUnified(ctx, env); err != nil {
|
||||
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if s.protocol.Respond == nil {
|
||||
return
|
||||
}
|
||||
response, ok, err := s.protocol.Respond(raw, env)
|
||||
if err != nil {
|
||||
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if !ok || len(response) == 0 {
|
||||
return
|
||||
}
|
||||
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := conn.Write(response); err != nil {
|
||||
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (p TCPProtocol) String() string {
|
||||
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
|
||||
}
|
||||
177
go/vehicle-gateway/internal/gateway/tcp_server_test.go
Normal file
177
go/vehicle-gateway/internal/gateway/tcp_server_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||
)
|
||||
|
||||
func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
|
||||
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Addr: ":0",
|
||||
Extract: jt808.ExtractFrames,
|
||||
Parse: jt808.ParseFrame,
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(frame); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].Phone != "013307795425" {
|
||||
t.Fatalf("phone = %q", sink.raw[0].Phone)
|
||||
}
|
||||
if sink.raw[0].Fields[envelope.FieldTotalMileageKM] != 10241.2 {
|
||||
t.Fatalf("total mileage = %#v", sink.raw[0].Fields[envelope.FieldTotalMileageKM])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
|
||||
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
good[len(good)-1] ^= 0xff
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(good); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||
}
|
||||
if sink.raw[0].ParseError == "" {
|
||||
t.Fatal("parse error should be recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
|
||||
frame := buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
|
||||
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
return []byte("ACK"), true, nil
|
||||
},
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(frame); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
buf := make([]byte, 3)
|
||||
if _, err := io.ReadFull(client, buf); err != nil {
|
||||
t.Fatalf("read response error = %v", err)
|
||||
}
|
||||
if string(buf) != "ACK" {
|
||||
t.Fatalf("response = %q", string(buf))
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer {
|
||||
t.Helper()
|
||||
server, err := NewTCPServer(TCPServerConfig{
|
||||
Protocol: protocol,
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
ReadBufferSize: 1024,
|
||||
IdleTimeout: time.Second,
|
||||
MaxConnections: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewTCPServer() error = %v", err)
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
func runPipe(t *testing.T, server *TCPServer) (net.Conn, <-chan struct{}) {
|
||||
t.Helper()
|
||||
client, srv := net.Pipe()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
server.handleConnection(context.Background(), srv)
|
||||
}()
|
||||
return client, done
|
||||
}
|
||||
|
||||
type recordingSink struct {
|
||||
raw []envelope.FrameEnvelope
|
||||
unified []envelope.FrameEnvelope
|
||||
}
|
||||
|
||||
func (s *recordingSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.raw = append(s.raw, env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *recordingSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.unified = append(s.unified, env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *recordingSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type testWriter struct {
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (w testWriter) Write(p []byte) (int, error) {
|
||||
w.t.Log(string(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func buildGBFrame(command byte, response byte, vin string, body []byte) []byte {
|
||||
frame := []byte{'#', '#', command, response}
|
||||
vinBytes := []byte(vin)
|
||||
if len(vinBytes) < 17 {
|
||||
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
|
||||
}
|
||||
frame = append(frame, vinBytes[:17]...)
|
||||
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
|
||||
frame = append(frame, body...)
|
||||
var bcc byte
|
||||
for _, value := range frame[2:] {
|
||||
bcc ^= value
|
||||
}
|
||||
return append(frame, bcc)
|
||||
}
|
||||
908
go/vehicle-gateway/internal/history/query.go
Normal file
908
go/vehicle-gateway/internal/history/query.go
Normal file
@@ -0,0 +1,908 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Queryer interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
type RawFrameQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
MessageID string
|
||||
OrderBy string
|
||||
IncludeTotal bool
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type RawFrameRow struct {
|
||||
TS string `json:"ts"`
|
||||
FrameID string `json:"frame_id"`
|
||||
EventID string `json:"event_id"`
|
||||
MessageID int64 `json:"message_id"`
|
||||
MessageIDHex string `json:"message_id_hex"`
|
||||
EventTime string `json:"event_time"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
RawSizeBytes int64 `json:"raw_size_bytes"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
ParsedJSON string `json:"parsed_json,omitempty"`
|
||||
FieldsJSON string `json:"fields_json,omitempty"`
|
||||
ParseStatus string `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type LocationQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type LocationRow struct {
|
||||
TS string `json:"ts"`
|
||||
EventID string `json:"event_id"`
|
||||
FrameID string `json:"frame_id"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
Longitude float64 `json:"longitude"`
|
||||
Latitude float64 `json:"latitude"`
|
||||
AltitudeM *float64 `json:"altitude_m,omitempty"`
|
||||
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
|
||||
DirectionDeg *int64 `json:"direction_deg,omitempty"`
|
||||
AlarmFlag *int64 `json:"alarm_flag,omitempty"`
|
||||
StatusFlag *int64 `json:"status_flag,omitempty"`
|
||||
TotalMileageKM *float64 `json:"total_mileage_km,omitempty"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type MileagePointQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type MileagePointRow struct {
|
||||
TS string `json:"ts"`
|
||||
EventID string `json:"event_id"`
|
||||
FrameID string `json:"frame_id"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
TotalMileageKM float64 `json:"total_mileage_km"`
|
||||
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type RawFrameRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
func NewRawFrameRepository(db Queryer, database string) *RawFrameRepository {
|
||||
if db == nil {
|
||||
panic("raw frame query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &RawFrameRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
type LocationRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
type MileagePointRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
func NewLocationRepository(db Queryer, database string) *LocationRepository {
|
||||
if db == nil {
|
||||
panic("location query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &LocationRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
func NewMileagePointRepository(db Queryer, database string) *MileagePointRepository {
|
||||
if db == nil {
|
||||
panic("mileage point query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &MileagePointRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) {
|
||||
query = normalizeRawFrameQuery(query)
|
||||
sqlText, args := buildRawFrameSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]RawFrameRow, 0)
|
||||
for rows.Next() {
|
||||
var row RawFrameRow
|
||||
var ts scanDateTime
|
||||
var eventTime scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.FrameID,
|
||||
&row.EventID,
|
||||
&row.MessageID,
|
||||
&eventTime,
|
||||
&receivedAt,
|
||||
&row.RawSizeBytes,
|
||||
&row.RawHex,
|
||||
&row.RawText,
|
||||
&row.ParsedJSON,
|
||||
&row.FieldsJSON,
|
||||
&row.ParseStatus,
|
||||
&row.ParseError,
|
||||
&row.SourceEndpoint,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.EventTime = eventTime.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.MessageIDHex = fmt.Sprintf("0x%04X", row.MessageID)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
|
||||
query = normalizeRawFrameQuery(query)
|
||||
sqlText, args := buildRawFrameCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]LocationRow, error) {
|
||||
query = normalizeLocationQuery(query)
|
||||
sqlText, args := buildLocationSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]LocationRow, 0)
|
||||
for rows.Next() {
|
||||
var row LocationRow
|
||||
var ts scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
var altitude sql.NullFloat64
|
||||
var speed sql.NullFloat64
|
||||
var direction sql.NullInt64
|
||||
var alarm sql.NullInt64
|
||||
var status sql.NullInt64
|
||||
var mileage sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.EventID,
|
||||
&row.FrameID,
|
||||
&receivedAt,
|
||||
&row.Longitude,
|
||||
&row.Latitude,
|
||||
&altitude,
|
||||
&speed,
|
||||
&direction,
|
||||
&alarm,
|
||||
&status,
|
||||
&mileage,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.AltitudeM = nullableFloat(altitude)
|
||||
row.SpeedKMH = nullableFloat(speed)
|
||||
row.DirectionDeg = nullableInt(direction)
|
||||
row.AlarmFlag = nullableInt(alarm)
|
||||
row.StatusFlag = nullableInt(status)
|
||||
row.TotalMileageKM = nullableFloat(mileage)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *LocationRepository) Count(ctx context.Context, query LocationQuery) (int64, error) {
|
||||
query = normalizeLocationQuery(query)
|
||||
sqlText, args := buildLocationCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) Query(ctx context.Context, query MileagePointQuery) ([]MileagePointRow, error) {
|
||||
query = normalizeMileagePointQuery(query)
|
||||
sqlText, args := buildMileagePointSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]MileagePointRow, 0)
|
||||
for rows.Next() {
|
||||
var row MileagePointRow
|
||||
var ts scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
var speed sql.NullFloat64
|
||||
var longitude sql.NullFloat64
|
||||
var latitude sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.EventID,
|
||||
&row.FrameID,
|
||||
&receivedAt,
|
||||
&row.TotalMileageKM,
|
||||
&speed,
|
||||
&longitude,
|
||||
&latitude,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.SpeedKMH = nullableFloat(speed)
|
||||
row.Longitude = nullableFloat(longitude)
|
||||
row.Latitude = nullableFloat(latitude)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) Count(ctx context.Context, query MileagePointQuery) (int64, error) {
|
||||
query = normalizeMileagePointQuery(query)
|
||||
sqlText, args := buildMileagePointCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func countRows(ctx context.Context, db Queryer, sqlText string, args ...any) (int64, error) {
|
||||
rows, err := db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var total int64
|
||||
if rows.Next() {
|
||||
if err := rows.Scan(&total); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return total, rows.Err()
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "raw_frames"
|
||||
}
|
||||
return r.database + ".raw_frames"
|
||||
}
|
||||
|
||||
func (r *LocationRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "vehicle_locations"
|
||||
}
|
||||
return r.database + ".vehicle_locations"
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "vehicle_mileage_points"
|
||||
}
|
||||
return r.database + ".vehicle_mileage_points"
|
||||
}
|
||||
|
||||
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.MessageID = strings.TrimSpace(query.MessageID)
|
||||
query.OrderBy = normalizeRawFrameOrderBy(query.OrderBy)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func normalizeLocationQuery(query LocationQuery) LocationQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func normalizeMileagePointQuery(query MileagePointQuery) MileagePointQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
|
||||
where := rawFrameWhere(query)
|
||||
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY " + rawFrameOrderColumn(query.OrderBy) + " DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := rawFrameWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
|
||||
where := locationWhere(query)
|
||||
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildLocationCountSQL(table string, query LocationQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := locationWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildMileagePointSQL(table string, query MileagePointQuery) (string, []any) {
|
||||
where := mileagePointWhere(query)
|
||||
sqlText := `SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildMileagePointCountSQL(table string, query MileagePointQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := mileagePointWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func rawFrameWhere(query RawFrameQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.MessageID != "" {
|
||||
if parsed, ok := parseMessageID(query.MessageID); ok {
|
||||
add("message_id = " + strconv.FormatInt(parsed, 10))
|
||||
}
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
func normalizeRawFrameOrderBy(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
switch value {
|
||||
case "receivedat", "received_at":
|
||||
return "receivedAt"
|
||||
default:
|
||||
return "ts"
|
||||
}
|
||||
}
|
||||
|
||||
func rawFrameOrderColumn(value string) string {
|
||||
if normalizeRawFrameOrderBy(value) == "receivedAt" {
|
||||
return "received_at"
|
||||
}
|
||||
return "ts"
|
||||
}
|
||||
|
||||
func locationWhere(query LocationQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
func mileagePointWhere(query MileagePointQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
type RawFrameHandler struct {
|
||||
repository *RawFrameRepository
|
||||
}
|
||||
|
||||
type LocationHandler struct {
|
||||
repository *LocationRepository
|
||||
}
|
||||
|
||||
type MileagePointHandler struct {
|
||||
repository *MileagePointRepository
|
||||
}
|
||||
|
||||
func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler {
|
||||
if repository == nil {
|
||||
panic("raw frame repository must not be nil")
|
||||
}
|
||||
return &RawFrameHandler{repository: repository}
|
||||
}
|
||||
|
||||
func NewLocationHandler(repository *LocationRepository) *LocationHandler {
|
||||
if repository == nil {
|
||||
panic("location repository must not be nil")
|
||||
}
|
||||
return &LocationHandler{repository: repository}
|
||||
}
|
||||
|
||||
func NewMileagePointHandler(repository *MileagePointRepository) *MileagePointHandler {
|
||||
if repository == nil {
|
||||
panic("mileage point repository must not be nil")
|
||||
}
|
||||
return &MileagePointHandler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/raw-frames" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseRawFrameQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
var total int64
|
||||
if query.IncludeTotal {
|
||||
total, err = h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !query.IncludeTotal {
|
||||
total = int64(len(rows))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/locations" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseLocationQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *MileagePointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/mileage-points" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseMileagePointQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return RawFrameQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return RawFrameQuery{}, err
|
||||
}
|
||||
query := RawFrameQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
MessageID: values.Get("messageId"),
|
||||
OrderBy: values.Get("orderBy"),
|
||||
IncludeTotal: values.Get("includeTotal") != "false",
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
if strings.TrimSpace(query.MessageID) != "" {
|
||||
if _, ok := parseMessageID(query.MessageID); !ok {
|
||||
return RawFrameQuery{}, errors.New("messageId must be decimal or hex like 0x0200")
|
||||
}
|
||||
}
|
||||
return normalizeRawFrameQuery(query), nil
|
||||
}
|
||||
|
||||
func parseMileagePointQuery(r *http.Request) (MileagePointQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return MileagePointQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return MileagePointQuery{}, err
|
||||
}
|
||||
query := MileagePointQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return MileagePointQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
return normalizeMileagePointQuery(query), nil
|
||||
}
|
||||
|
||||
func parseLocationQuery(r *http.Request) (LocationQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return LocationQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return LocationQuery{}, err
|
||||
}
|
||||
query := LocationQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
return normalizeLocationQuery(query), nil
|
||||
}
|
||||
|
||||
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil || value < min || value > max {
|
||||
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseMessageID(value string) (int64, bool) {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
if value == "" {
|
||||
return 0, false
|
||||
}
|
||||
if strings.HasPrefix(value, "0x") {
|
||||
parsed, err := strconv.ParseInt(strings.TrimPrefix(value, "0x"), 16, 32)
|
||||
return parsed, err == nil
|
||||
}
|
||||
parsed, err := strconv.ParseInt(value, 10, 32)
|
||||
return parsed, err == nil
|
||||
}
|
||||
|
||||
func validDateTime(value string) bool {
|
||||
value = normalizeDateTimeLiteral(value)
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
for _, layout := range []string{"2006-01-02", "2006-01-02 15:04:05", time.RFC3339} {
|
||||
if _, err := time.Parse(layout, value); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeDateTimeLiteral(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} {
|
||||
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
|
||||
return parsed.UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
}
|
||||
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
|
||||
return parsed.UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func safeIdentifier(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type scanDateTime struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDateTime) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatSQLTime(value any, layout string) string {
|
||||
switch typed := value.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case time.Time:
|
||||
return typed.Format(layout)
|
||||
case []byte:
|
||||
return strings.TrimSpace(string(typed))
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
|
||||
func nullableFloat(value sql.NullFloat64) *float64 {
|
||||
if !value.Valid {
|
||||
return nil
|
||||
}
|
||||
return &value.Float64
|
||||
}
|
||||
|
||||
func nullableInt(value sql.NullInt64) *int64 {
|
||||
if !value.Valid {
|
||||
return nil
|
||||
}
|
||||
return &value.Int64
|
||||
}
|
||||
|
||||
func writeHistoryError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
423
go/vehicle-gateway/internal/history/query_test.go
Normal file
423
go/vehicle-gateway/internal/history/query_test.go
Normal file
@@ -0,0 +1,423 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
"go_frame", "event-1", 0x0200,
|
||||
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
time.Date(2026, 7, 1, 23, 25, 37, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
64, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
|
||||
"OK", "", "222.66.200.68:29646", "JT808", "013079963379", "LKLG7C4E3NA774736", "013079963379", "9963379",
|
||||
))
|
||||
|
||||
repository := NewRawFrameRepository(db, "lingniu_vehicle_ts")
|
||||
rows, err := repository.Query(context.Background(), RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
DateFrom: "2026-07-01 00:00:00",
|
||||
DateTo: "2026-07-01 23:59:59",
|
||||
Limit: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query() error = %v", err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("row count = %d", len(rows))
|
||||
}
|
||||
if rows[0].MessageID != 512 || rows[0].MessageIDHex != "0x0200" || rows[0].ParsedJSON == "" {
|
||||
t.Fatalf("unexpected row: %#v", rows[0])
|
||||
}
|
||||
if rows[0].TS != "2026-07-01 23:25:36" {
|
||||
t.Fatalf("timestamp = %q", rows[0].TS)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(38))
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-01 22:28:25", "go_frame", "event-2", 2, "2026-07-01 22:28:25", "2026-07-01 22:28:25",
|
||||
128, "2323", "", `{"command":"REALTIME"}`, `{"total_mileage_km":53490.9}`,
|
||||
"OK", "", "8.134.95.166:53702", "GB32960", "LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"message_id_hex":"0x0002"`, `"total":38`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(11))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "go_frame", "event-3", 0x0200, "2026-07-02 00:18:22", "2026-07-02 00:22:43",
|
||||
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"total_mileage_km":8792.8}`,
|
||||
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"phone":"013307811350"`, `"total":11`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
|
||||
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
|
||||
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=NOT_EXISTS&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
|
||||
t.Fatalf("expected empty items array, got: %s", body)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocationHandlerReturnsLocationsByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(17))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
|
||||
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
|
||||
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
|
||||
"JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
|
||||
8792.8, 8.0, 121.07764, 30.585928,
|
||||
"JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewMileagePointHandler(NewMileagePointRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/mileage-points?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"total_mileage_km":8792.8`, `"speed_kmh":8`, `"total":19`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) {
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?dateFrom=2026-07-01T00:00:00&dateTo=2026-07-02T00:00:00", nil)
|
||||
|
||||
query, err := parseRawFrameQuery(request)
|
||||
if err != nil {
|
||||
t.Fatalf("parseRawFrameQuery() error = %v", err)
|
||||
}
|
||||
if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" {
|
||||
t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) {
|
||||
for raw, want := range map[string]string{
|
||||
"2026-07-01T00:00:00": "2026-06-30 16:00:00",
|
||||
"2026-07-01 00:00:00": "2026-06-30 16:00:00",
|
||||
"2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00",
|
||||
"2026-06-30T16:00:00Z": "2026-06-30 16:00:00",
|
||||
} {
|
||||
if got := normalizeDateTimeLiteral(raw); got != want {
|
||||
t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
|
||||
for raw, want := range map[string]int64{
|
||||
"512": 512,
|
||||
"0x0200": 512,
|
||||
"0X0100": 256,
|
||||
} {
|
||||
got, ok := parseMessageID(raw)
|
||||
if !ok || got != want {
|
||||
t.Fatalf("parseMessageID(%q) = %d,%v want %d,true", raw, got, ok, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
DateFrom: "2026-07-02 00:00:00",
|
||||
DateTo: "2026-07-02 23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"ts >= '2026-07-01 16:00:00'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildLocationSQL("lingniu_vehicle_ts.vehicle_locations", LocationQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
DateFrom: "2026-07-02 00:00:00",
|
||||
DateTo: "2026-07-02 23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"ts >= '2026-07-01 16:00:00'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
VIN: "VIN'1",
|
||||
MessageID: "0x0200",
|
||||
DateFrom: "2026-07-01T00:00:00",
|
||||
DateTo: "2026-07-01T23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"vin = 'VIN''1'",
|
||||
"message_id = 512",
|
||||
"ts >= '2026-06-30 16:00:00'",
|
||||
"ts <= '2026-07-01 15:59:59'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRawFrameSQLCanOrderByReceivedAt(t *testing.T) {
|
||||
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
OrderBy: "receivedAt",
|
||||
Limit: 1,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
if !strings.Contains(sqlText, "ORDER BY received_at DESC LIMIT 1 OFFSET 0") {
|
||||
t.Fatalf("sql should order by received_at: %s", sqlText)
|
||||
}
|
||||
}
|
||||
71
go/vehicle-gateway/internal/history/schema.go
Normal file
71
go/vehicle-gateway/internal/history/schema.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package history
|
||||
|
||||
const DefaultDatabase = "lingniu_vehicle_ts"
|
||||
|
||||
func SchemaStatements(database string) []string {
|
||||
if database == "" {
|
||||
database = DefaultDatabase
|
||||
}
|
||||
return []string{
|
||||
"CREATE DATABASE IF NOT EXISTS " + database + " KEEP 7300 DURATION 10 BUFFER 256",
|
||||
"USE " + database,
|
||||
`CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
event_id NCHAR(64),
|
||||
message_id INT,
|
||||
event_time TIMESTAMP,
|
||||
received_at TIMESTAMP,
|
||||
raw_size_bytes INT,
|
||||
raw_hex BINARY(16374),
|
||||
raw_text BINARY(16374),
|
||||
parsed_json BINARY(16374),
|
||||
fields_json BINARY(4096),
|
||||
parse_status NCHAR(16),
|
||||
parse_error BINARY(1024),
|
||||
source_endpoint NCHAR(128)
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
`CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg INT,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
total_mileage_km DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
}
|
||||
}
|
||||
339
go/vehicle-gateway/internal/history/writer.go
Normal file
339
go/vehicle-gateway/internal/history/writer.go
Normal file
@@ -0,0 +1,339 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Execer interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
exec Execer
|
||||
cache tableCache
|
||||
}
|
||||
|
||||
type tableCache struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]struct{}
|
||||
}
|
||||
|
||||
func NewWriter(exec Execer) *Writer {
|
||||
if exec == nil {
|
||||
panic("history execer must not be nil")
|
||||
}
|
||||
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
|
||||
}
|
||||
|
||||
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
||||
for _, statement := range SchemaStatements(database) {
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if err := w.AppendRawFrame(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.AppendLocation(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.AppendMileagePoint(ctx, env)
|
||||
}
|
||||
|
||||
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
table := tableName("raw", env)
|
||||
if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
||||
raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
|
||||
VALUES (%s)`, table, joinLiterals(rawValues(env))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
longitude, okLon := floatField(env, envelope.FieldLongitude)
|
||||
latitude, okLat := floatField(env, envelope.FieldLatitude)
|
||||
if !okLon || !okLat {
|
||||
return nil
|
||||
}
|
||||
table := tableName("loc", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_locations", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
||||
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
||||
VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
table := tableName("mil", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
|
||||
VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
|
||||
key := stable + "." + table
|
||||
w.cache.mu.Lock()
|
||||
_, ok := w.cache.seen[key]
|
||||
w.cache.mu.Unlock()
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
|
||||
table, stable,
|
||||
quote(string(env.Protocol)),
|
||||
quote(env.VehicleKey()),
|
||||
quote(env.VIN),
|
||||
quote(env.Phone),
|
||||
quote(env.DeviceID))
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
w.cache.mu.Lock()
|
||||
w.cache.seen[key] = struct{}{}
|
||||
w.cache.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawValues(env envelope.FrameEnvelope) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
eventTime := millis(env.EventTimeMS)
|
||||
return []any{
|
||||
received,
|
||||
frameID(env),
|
||||
env.StableEventID(),
|
||||
messageIDInt(env.MessageID),
|
||||
eventTime,
|
||||
received,
|
||||
rawSizeBytes(env),
|
||||
env.RawHex,
|
||||
env.RawText,
|
||||
jsonString(env.Parsed),
|
||||
jsonString(env.Fields),
|
||||
string(env.ParseStatus),
|
||||
env.ParseError,
|
||||
env.SourceEndpoint,
|
||||
}
|
||||
}
|
||||
|
||||
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
longitude,
|
||||
latitude,
|
||||
floatFieldOrNil(env, "altitude_m"),
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
intFieldOrNil(env, "direction_deg"),
|
||||
intFieldOrNil(env, "alarm_flag"),
|
||||
intFieldOrNil(env, "status_flag"),
|
||||
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
|
||||
}
|
||||
}
|
||||
|
||||
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
totalMileage,
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
floatFieldOrNil(env, envelope.FieldLongitude),
|
||||
floatFieldOrNil(env, envelope.FieldLatitude),
|
||||
}
|
||||
}
|
||||
|
||||
func frameID(env envelope.FrameEnvelope) string {
|
||||
return "go_" + env.StableEventID()
|
||||
}
|
||||
|
||||
func tableName(prefix string, env envelope.FrameEnvelope) string {
|
||||
return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
|
||||
}
|
||||
|
||||
func hash16(value string) string {
|
||||
sum := sha1.Sum([]byte(value))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
func messageIDInt(value string) int64 {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
value = strings.TrimPrefix(value, "0x")
|
||||
parsed, err := strconv.ParseInt(value, 16, 32)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
parsed, _ = strconv.ParseInt(value, 10, 32)
|
||||
return parsed
|
||||
}
|
||||
|
||||
func rawSizeBytes(env envelope.FrameEnvelope) int {
|
||||
rawHex := strings.TrimSpace(env.RawHex)
|
||||
if len(rawHex)%2 != 0 {
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
if rawHex != "" {
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
return len([]byte(env.RawText))
|
||||
}
|
||||
|
||||
func jsonString(value any) string {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||
if env.Fields == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
value, ok := floatField(env, key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
if env.Fields == nil {
|
||||
return nil
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return nil
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return typed
|
||||
case int64:
|
||||
return typed
|
||||
case uint16:
|
||||
return int64(typed)
|
||||
case uint32:
|
||||
return int64(typed)
|
||||
case float64:
|
||||
return int64(typed)
|
||||
case string:
|
||||
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
|
||||
if env.EventTimeMS > 0 {
|
||||
return millis(env.EventTimeMS)
|
||||
}
|
||||
return millis(env.ReceivedAtMS)
|
||||
}
|
||||
|
||||
func millis(value int64) time.Time {
|
||||
if value <= 0 {
|
||||
return time.UnixMilli(time.Now().UnixMilli()).UTC()
|
||||
}
|
||||
return time.UnixMilli(value).UTC()
|
||||
}
|
||||
|
||||
func quote(value string) string {
|
||||
return strings.ReplaceAll(value, "'", "''")
|
||||
}
|
||||
|
||||
func joinLiterals(values []any) string {
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, literal(value))
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
func literal(value any) string {
|
||||
if value == nil {
|
||||
return "NULL"
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case time.Time:
|
||||
return "'" + typed.UTC().Format("2006-01-02 15:04:05.000") + "'"
|
||||
case string:
|
||||
return "'" + quote(typed) + "'"
|
||||
case envelope.ParseStatus:
|
||||
return "'" + quote(string(typed)) + "'"
|
||||
case int:
|
||||
return strconv.FormatInt(int64(typed), 10)
|
||||
case int64:
|
||||
return strconv.FormatInt(typed, 10)
|
||||
case uint16:
|
||||
return strconv.FormatUint(uint64(typed), 10)
|
||||
case uint32:
|
||||
return strconv.FormatUint(uint64(typed), 10)
|
||||
case float64:
|
||||
return strconv.FormatFloat(typed, 'f', -1, 64)
|
||||
case float32:
|
||||
return strconv.FormatFloat(float64(typed), 'f', -1, 64)
|
||||
default:
|
||||
return "'" + quote(fmt.Sprint(typed)) + "'"
|
||||
}
|
||||
}
|
||||
146
go/vehicle-gateway/internal/history/writer_test.go
Normal file
146
go/vehicle-gateway/internal/history/writer_test.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
|
||||
statements := strings.Join(SchemaStatements("test_ts"), "\n")
|
||||
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} {
|
||||
if !strings.Contains(statements, want) {
|
||||
t.Fatalf("schema missing %q:\n%s", want, statements)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec)
|
||||
env := sampleEnvelope()
|
||||
|
||||
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||
t.Fatalf("AppendAll() error = %v", err)
|
||||
}
|
||||
|
||||
if got := countSQL(exec.calls, "USING raw_frames"); got != 1 {
|
||||
t.Fatalf("raw child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
|
||||
t.Fatalf("location child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 {
|
||||
t.Fatalf("mileage child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||
t.Fatalf("raw insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
|
||||
t.Fatalf("location insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 {
|
||||
t.Fatalf("mileage insert count = %d", got)
|
||||
}
|
||||
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
|
||||
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {
|
||||
t.Fatalf("raw insert should quote string literals: %s", rawInsert)
|
||||
}
|
||||
if strings.Contains(rawInsert, "VALUES (?,") {
|
||||
t.Fatalf("raw insert should not use placeholders for tdengine websocket plain insert: %s", rawInsert)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec)
|
||||
env := sampleEnvelope()
|
||||
env.Fields = map[string]any{}
|
||||
|
||||
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||
t.Fatalf("AppendAll() error = %v", err)
|
||||
}
|
||||
|
||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||
t.Fatalf("raw insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
|
||||
t.Fatalf("location insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
|
||||
t.Fatalf("mileage insert count = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawSizeBytesUsesRawTextWhenHexIsEmpty(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{RawText: `{"code":"0F80","data":{"speed":12}}`}
|
||||
if got, want := rawSizeBytes(env), len([]byte(env.RawText)); got != want {
|
||||
t.Fatalf("raw text size = %d, want %d", got, want)
|
||||
}
|
||||
|
||||
env.RawHex = "7E0200"
|
||||
if got, want := rawSizeBytes(env), 3; got != want {
|
||||
t.Fatalf("raw hex size = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func sampleEnvelope() envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Sequence: 1,
|
||||
VIN: "LNBVIN00000000001",
|
||||
Phone: "013307795425",
|
||||
SourceEndpoint: "127.0.0.1:18080",
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7E0200",
|
||||
Parsed: map[string]any{"message": "location"},
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 121.069881,
|
||||
envelope.FieldLatitude: 30.590151,
|
||||
envelope.FieldSpeedKMH: 23.0,
|
||||
envelope.FieldTotalMileageKM: 10241.2,
|
||||
"direction_deg": uint16(79),
|
||||
"alarm_flag": uint32(0),
|
||||
"status_flag": uint32(72),
|
||||
},
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
}
|
||||
|
||||
func countSQL(calls []execCall, pattern string) int {
|
||||
var count int
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call.query, pattern) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func findSQL(calls []execCall, pattern string) string {
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call.query, pattern) {
|
||||
return call.query
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type execCall struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type recordingExec struct {
|
||||
calls []execCall
|
||||
}
|
||||
|
||||
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||
return nil, nil
|
||||
}
|
||||
237
go/vehicle-gateway/internal/identity/resolver.go
Normal file
237
go/vehicle-gateway/internal/identity/resolver.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Resolver interface {
|
||||
Resolve(context.Context, envelope.FrameEnvelope) (envelope.FrameEnvelope, error)
|
||||
}
|
||||
|
||||
type NoopResolver struct{}
|
||||
|
||||
func (NoopResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||
return env, nil
|
||||
}
|
||||
|
||||
type MySQLResolver struct {
|
||||
db *sql.DB
|
||||
table string
|
||||
}
|
||||
|
||||
func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver {
|
||||
if db == nil {
|
||||
panic("identity db must not be nil")
|
||||
}
|
||||
table = strings.TrimSpace(table)
|
||||
if table == "" || !safeIdentifier(table) {
|
||||
table = "vehicle_identity_binding"
|
||||
}
|
||||
return &MySQLResolver{db: db, table: table}
|
||||
}
|
||||
|
||||
func safeIdentifier(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||
if strings.TrimSpace(env.VIN) != "" {
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
candidates := CandidateKeys(env)
|
||||
for _, candidate := range candidates {
|
||||
vin, err := r.lookup(ctx, candidate.Column, candidate.Value)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
return env, err
|
||||
}
|
||||
if strings.TrimSpace(vin) != "" {
|
||||
env.VIN = strings.TrimSpace(vin)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{
|
||||
"resolved": true,
|
||||
"source": candidate.Column,
|
||||
"value": candidate.Value,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
}
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) {
|
||||
query := "SELECT vin FROM " + r.table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' ORDER BY updated_at DESC LIMIT 1"
|
||||
var vin string
|
||||
err := r.db.QueryRowContext(ctx, query, value).Scan(&vin)
|
||||
return vin, err
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) trackJT808(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if env.Protocol != envelope.ProtocolJT808 || strings.TrimSpace(env.Phone) == "" || strings.TrimSpace(env.MessageID) == "" {
|
||||
return nil
|
||||
}
|
||||
registration := mapValue(env.Parsed, "registration")
|
||||
authentication := mapValue(env.Parsed, "authentication")
|
||||
deviceID := firstNonEmpty(env.DeviceID, textValue(registration, "device_id"))
|
||||
plate := firstNonEmpty(env.Plate, textValue(registration, "plate"))
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
if vin == "" {
|
||||
vin = "unknown"
|
||||
}
|
||||
firstRegisteredAt := nilTime()
|
||||
latestRegisteredAt := nilTime()
|
||||
if env.MessageID == "0x0100" {
|
||||
firstRegisteredAt = "CURRENT_TIMESTAMP"
|
||||
latestRegisteredAt = "CURRENT_TIMESTAMP"
|
||||
}
|
||||
latestAuthenticatedAt := nilTime()
|
||||
if env.MessageID == "0x0102" {
|
||||
latestAuthenticatedAt = "CURRENT_TIMESTAMP"
|
||||
}
|
||||
|
||||
query := `INSERT INTO jt808_registration
|
||||
(phone, device_id, plate, vin, province, city, manufacturer, device_type, plate_color,
|
||||
auth_token, auth_imei, auth_software_version, source_endpoint,
|
||||
first_registered_at, latest_registered_at, latest_authenticated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ` + firstRegisteredAt + `, ` + latestRegisteredAt + `, ` + latestAuthenticatedAt + `)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
device_id = IF(VALUES(device_id) <> '', VALUES(device_id), device_id),
|
||||
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
|
||||
vin = IF(VALUES(vin) <> 'unknown', VALUES(vin), vin),
|
||||
province = IF(VALUES(province) <> '', VALUES(province), province),
|
||||
city = IF(VALUES(city) <> '', VALUES(city), city),
|
||||
manufacturer = IF(VALUES(manufacturer) <> '', VALUES(manufacturer), manufacturer),
|
||||
device_type = IF(VALUES(device_type) <> '', VALUES(device_type), device_type),
|
||||
plate_color = IF(VALUES(plate_color) <> '', VALUES(plate_color), plate_color),
|
||||
auth_token = IF(VALUES(auth_token) <> '', VALUES(auth_token), auth_token),
|
||||
auth_imei = IF(VALUES(auth_imei) <> '', VALUES(auth_imei), auth_imei),
|
||||
auth_software_version = IF(VALUES(auth_software_version) <> '', VALUES(auth_software_version), auth_software_version),
|
||||
source_endpoint = IF(VALUES(source_endpoint) <> '', VALUES(source_endpoint), source_endpoint),
|
||||
latest_seen_at = CURRENT_TIMESTAMP,
|
||||
first_registered_at = COALESCE(first_registered_at, VALUES(first_registered_at)),
|
||||
latest_registered_at = COALESCE(VALUES(latest_registered_at), latest_registered_at),
|
||||
latest_authenticated_at = COALESCE(VALUES(latest_authenticated_at), latest_authenticated_at)`
|
||||
_, err := r.db.ExecContext(ctx, query,
|
||||
env.Phone,
|
||||
deviceID,
|
||||
plate,
|
||||
vin,
|
||||
textValue(registration, "province"),
|
||||
textValue(registration, "city"),
|
||||
textValue(registration, "manufacturer"),
|
||||
textValue(registration, "device_type"),
|
||||
textValue(registration, "plate_color"),
|
||||
textValue(authentication, "token"),
|
||||
textValue(authentication, "imei"),
|
||||
textValue(authentication, "software_version"),
|
||||
env.SourceEndpoint,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if vin == "unknown" {
|
||||
return nil
|
||||
}
|
||||
_, err = r.db.ExecContext(ctx, `UPDATE IGNORE `+r.table+`
|
||||
SET
|
||||
phone = CASE WHEN (phone IS NULL OR phone = '') AND ? <> '' THEN ? ELSE phone END,
|
||||
device_id = CASE WHEN (device_id IS NULL OR device_id = '') AND ? <> '' THEN ? ELSE device_id END,
|
||||
plate = CASE WHEN (plate IS NULL OR plate = '') AND ? <> '' THEN ? ELSE plate END
|
||||
WHERE vin = ?`, env.Phone, env.Phone, deviceID, deviceID, plate, plate, vin)
|
||||
return err
|
||||
}
|
||||
|
||||
type CandidateKey struct {
|
||||
Column string
|
||||
Value string
|
||||
}
|
||||
|
||||
func CandidateKeys(env envelope.FrameEnvelope) []CandidateKey {
|
||||
var out []CandidateKey
|
||||
add := func(column string, value string) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || strings.EqualFold(value, "unknown") {
|
||||
return
|
||||
}
|
||||
for _, existing := range out {
|
||||
if existing.Column == column && existing.Value == value {
|
||||
return
|
||||
}
|
||||
}
|
||||
out = append(out, CandidateKey{Column: column, Value: value})
|
||||
}
|
||||
add("phone", env.Phone)
|
||||
if trimmed := strings.TrimLeft(strings.TrimSpace(env.Phone), "0"); trimmed != "" {
|
||||
add("phone", trimmed)
|
||||
}
|
||||
add("device_id", env.DeviceID)
|
||||
add("plate", env.Plate)
|
||||
add("vin", env.VehicleKeyHint)
|
||||
return out
|
||||
}
|
||||
|
||||
func mapValue(values map[string]any, key string) map[string]any {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
nested, _ := values[key].(map[string]any)
|
||||
return nested
|
||||
}
|
||||
|
||||
func textValue(values map[string]any, key string) string {
|
||||
if values == nil {
|
||||
return ""
|
||||
}
|
||||
value, ok := values[key]
|
||||
if !ok || value == nil {
|
||||
return ""
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func nilTime() string {
|
||||
return "NULL"
|
||||
}
|
||||
171
go/vehicle-gateway/internal/identity/resolver_test.go
Normal file
171
go/vehicle-gateway/internal/identity/resolver_test.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestCandidateKeysPrioritizesPhoneDevicePlate(t *testing.T) {
|
||||
keys := CandidateKeys(envelope.FrameEnvelope{
|
||||
Phone: "013307795425",
|
||||
DeviceID: "D1",
|
||||
Plate: "豫A12345",
|
||||
VehicleKeyHint: "D1",
|
||||
})
|
||||
got := []string{}
|
||||
for _, key := range keys {
|
||||
got = append(got, key.Column+"="+key.Value)
|
||||
}
|
||||
want := []string{"phone=013307795425", "phone=13307795425", "device_id=D1", "plate=豫A12345", "vin=D1"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("candidate count = %d got=%#v", len(got), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateKeysAddsTrimmedPhoneVariant(t *testing.T) {
|
||||
keys := CandidateKeys(envelope.FrameEnvelope{
|
||||
Phone: "013079963379",
|
||||
})
|
||||
got := []string{}
|
||||
for _, key := range keys {
|
||||
got = append(got, key.Column+"="+key.Value)
|
||||
}
|
||||
want := []string{"phone=013079963379", "phone=13079963379"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("candidate count = %d got=%#v", len(got), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001"))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNBVIN00000000001" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "phone" {
|
||||
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverFallsBackToDeviceID(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
|
||||
WithArgs("D1").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000002"))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
DeviceID: "D1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNBVIN00000000002" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverTracksJT808RegistrationAndBackfillsBinding(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013079963379").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13079963379").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
|
||||
WithArgs("DEV0001").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("TEST123").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LKLG7C4E3NA774736"))
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE IGNORE vehicle_identity_binding").
|
||||
WithArgs("013079963379", "013079963379", "DEV0001", "DEV0001", "TEST123", "TEST123", "LKLG7C4E3NA774736").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0100",
|
||||
Phone: "013079963379",
|
||||
DeviceID: "DEV0001",
|
||||
Plate: "TEST123",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
Parsed: map[string]any{
|
||||
"registration": map[string]any{
|
||||
"province": uint16(16),
|
||||
"city": uint16(32),
|
||||
"manufacturer": "YUTNG",
|
||||
"device_type": "ZK6105CHEVNPG4",
|
||||
"device_id": "DEV0001",
|
||||
"plate_color": uint8(2),
|
||||
"plate": "TEST123",
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LKLG7C4E3NA774736" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) {
|
||||
t.Helper()
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
return db, mock
|
||||
}
|
||||
13
go/vehicle-gateway/internal/observability/logger.go
Normal file
13
go/vehicle-gateway/internal/observability/logger.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
// NewLogger returns a JSON logger with a stable service field so ECS logs from
|
||||
// different Go processes can be filtered without relying on container names.
|
||||
func NewLogger(service string) *slog.Logger {
|
||||
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||
return slog.New(handler).With("service", service)
|
||||
}
|
||||
840
go/vehicle-gateway/internal/protocol/gb32960/parser.go
Normal file
840
go/vehicle-gateway/internal/protocol/gb32960/parser.go
Normal file
@@ -0,0 +1,840 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFrameTooShort = errors.New("gb32960 frame too short")
|
||||
ErrBadStartSymbol = errors.New("gb32960 bad start symbol")
|
||||
ErrBodyLength = errors.New("gb32960 body length mismatch")
|
||||
ErrBCC = errors.New("gb32960 bcc mismatch")
|
||||
)
|
||||
|
||||
const (
|
||||
minFrameLen = 25
|
||||
headerLen = 24
|
||||
)
|
||||
|
||||
// ExtractFrames splits GB/T 32960 TCP streams using the protocol length field.
|
||||
// Returned frames include the start symbols and trailing BCC byte.
|
||||
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
|
||||
offset := 0
|
||||
for {
|
||||
start := indexStart(stream[offset:])
|
||||
if start < 0 {
|
||||
return frames, nil, nil
|
||||
}
|
||||
offset += start
|
||||
remaining := stream[offset:]
|
||||
if len(remaining) < headerLen {
|
||||
return frames, append([]byte(nil), remaining...), nil
|
||||
}
|
||||
bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
|
||||
frameLen := frameLength(remaining, bodyLen)
|
||||
if len(remaining) < frameLen {
|
||||
return frames, append([]byte(nil), remaining...), nil
|
||||
}
|
||||
frames = append(frames, append([]byte(nil), remaining[:frameLen]...))
|
||||
offset += frameLen
|
||||
if offset >= len(stream) {
|
||||
return frames, nil, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||
if len(raw) < minFrameLen {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
version, ok := version(raw[0], raw[1])
|
||||
if !ok {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
|
||||
}
|
||||
bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
|
||||
actualBodyLen := len(raw) - headerLen - 1
|
||||
if len(raw) != headerLen+bodyLen+1 && !isExtendedPlatformLogin(raw[2], bodyLen, actualBodyLen) {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
|
||||
}
|
||||
if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrBCC, got, want)
|
||||
}
|
||||
|
||||
command := raw[2]
|
||||
responseFlag := raw[3]
|
||||
vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
|
||||
body := raw[headerLen : headerLen+actualBodyLen]
|
||||
parsed := map[string]any{
|
||||
"header": map[string]any{
|
||||
"version": version,
|
||||
"command": fmt.Sprintf("0x%02X", command),
|
||||
"response_flag": fmt.Sprintf("0x%02X", responseFlag),
|
||||
"vin": vin,
|
||||
"encrypt": raw[21],
|
||||
"body_length": bodyLen,
|
||||
"actual_body_length": actualBodyLen,
|
||||
},
|
||||
}
|
||||
fields := map[string]any{}
|
||||
eventTimeMS := receivedAtMS
|
||||
|
||||
if command == 0x02 || command == 0x03 {
|
||||
eventTime, units := parseDataBody(version, body, fields)
|
||||
if !eventTime.IsZero() {
|
||||
eventTimeMS = eventTime.UnixMilli()
|
||||
fields["device_time"] = eventTime.Format(time.RFC3339)
|
||||
parsed["device_time"] = eventTime.Format(time.RFC3339)
|
||||
}
|
||||
parsed["data_units"] = units
|
||||
} else if command == 0x05 {
|
||||
login := parsePlatformLogin(body)
|
||||
parsed["platform_login"] = login
|
||||
if deviceTime, ok := login["login_time"].(string); ok && deviceTime != "" {
|
||||
fields["device_time"] = deviceTime
|
||||
parsed["device_time"] = deviceTime
|
||||
}
|
||||
if account, ok := login["username"].(string); ok && account != "" {
|
||||
fields["platform_account"] = account
|
||||
}
|
||||
}
|
||||
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
MessageID: fmt.Sprintf("0x%02X", command),
|
||||
VIN: vin,
|
||||
SourceEndpoint: sourceEndpoint,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
|
||||
if len(body) < 6 {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
eventTime := parseGBTime(body[:6])
|
||||
cursor := 6
|
||||
var units []map[string]any
|
||||
for cursor < len(body) {
|
||||
unitType := body[cursor]
|
||||
cursor++
|
||||
switch unitType {
|
||||
case 0x01:
|
||||
size := 20
|
||||
if version == "V2025" {
|
||||
size = 18
|
||||
}
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x01", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseVehicleData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x01", "name": "vehicle", "value": unit})
|
||||
fields["vehicle_status"] = unit["vehicle_status"]
|
||||
fields["charge_status"] = unit["charge_status"]
|
||||
fields["running_mode"] = unit["running_mode"]
|
||||
fields[envelope.FieldSpeedKMH] = unit["speed_kmh"]
|
||||
fields[envelope.FieldTotalMileageKM] = unit["total_mileage_km"]
|
||||
fields[envelope.FieldSOCPercent] = unit["soc_percent"]
|
||||
cursor += size
|
||||
case 0x05:
|
||||
if len(body[cursor:]) < 9 {
|
||||
units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parsePositionData(body[cursor : cursor+9])
|
||||
units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
|
||||
fields["position_status"] = unit["position_status"]
|
||||
fields[envelope.FieldLongitude] = unit["longitude"]
|
||||
fields[envelope.FieldLatitude] = unit["latitude"]
|
||||
cursor += 9
|
||||
case 0x02:
|
||||
if len(body[cursor:]) < 1 {
|
||||
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
size := 1 + int(body[cursor])*12
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseDriveMotorData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x02", "name": "drive_motor", "value": unit})
|
||||
cursor += size
|
||||
case 0x03:
|
||||
if len(body[cursor:]) < 8 {
|
||||
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
probeCount := int(binary.BigEndian.Uint16(body[cursor+6 : cursor+8]))
|
||||
size := 8 + probeCount + 10
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseFuelCellData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x03", "name": "fuel_cell", "value": unit})
|
||||
fields["fuel_cell_hydrogen_consumption_kg_per_100km"] = unit["hydrogen_consumption_kg_per_100km"]
|
||||
fields["fuel_cell_voltage_v"] = unit["fuel_cell_voltage_v"]
|
||||
fields["fuel_cell_current_a"] = unit["fuel_cell_current_a"]
|
||||
cursor += size
|
||||
case 0x04:
|
||||
if len(body[cursor:]) < 5 {
|
||||
units = append(units, map[string]any{"type": "0x04", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseEngineData(body[cursor : cursor+5])
|
||||
units = append(units, map[string]any{"type": "0x04", "name": "engine", "value": unit})
|
||||
cursor += 5
|
||||
case 0x06:
|
||||
if len(body[cursor:]) < 14 {
|
||||
units = append(units, map[string]any{"type": "0x06", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseExtremeData(body[cursor : cursor+14])
|
||||
units = append(units, map[string]any{"type": "0x06", "name": "extreme", "value": unit})
|
||||
cursor += 14
|
||||
case 0x07:
|
||||
size, ok := alarmDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x07", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseAlarmData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x07", "name": "alarm", "value": unit})
|
||||
cursor += size
|
||||
case 0x08:
|
||||
size, ok := voltageDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x08", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseVoltageData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x08", "name": "voltage", "value": unit})
|
||||
cursor += size
|
||||
case 0x09:
|
||||
size, ok := temperatureDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x09", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseTemperatureData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x09", "name": "temperature", "value": unit})
|
||||
cursor += size
|
||||
case 0x30:
|
||||
size, ok := gdFCStackDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x30", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCStackData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x30", "name": "gd_fc_stack", "value": unit})
|
||||
if summaries, ok := unit["summaries"].([]map[string]any); ok && len(summaries) > 0 {
|
||||
fields["gd_fc_stack_hydrogen_inlet_pressure_kpa"] = summaries[0]["hydrogen_inlet_pressure_kpa"]
|
||||
fields["gd_fc_stack_water_outlet_temp_c"] = summaries[0]["stack_water_outlet_temp_c"]
|
||||
fields["gd_fc_stack_cell_count"] = summaries[0]["cell_count"]
|
||||
fields["gd_fc_stack_frame_cell_count"] = summaries[0]["frame_cell_count"]
|
||||
}
|
||||
cursor += size
|
||||
case 0x31:
|
||||
size, ok := gdFCAuxiliaryDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x31", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCAuxiliaryData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x31", "name": "gd_fc_auxiliary", "value": unit})
|
||||
cursor += size
|
||||
case 0x32:
|
||||
if len(body[cursor:]) < 9 {
|
||||
units = append(units, map[string]any{"type": "0x32", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCDcDcData(body[cursor : cursor+9])
|
||||
units = append(units, map[string]any{"type": "0x32", "name": "gd_fc_dcdc", "value": unit})
|
||||
fields["gd_fc_dcdc_controller_temp_c"] = unit["controller_temp_c"]
|
||||
cursor += 9
|
||||
case 0x33:
|
||||
if len(body[cursor:]) < 5 {
|
||||
units = append(units, map[string]any{"type": "0x33", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCAirConditionerData(body[cursor : cursor+5])
|
||||
units = append(units, map[string]any{"type": "0x33", "name": "gd_fc_air_conditioner", "value": unit})
|
||||
cursor += 5
|
||||
case 0x34:
|
||||
if len(body[cursor:]) < 7 {
|
||||
units = append(units, map[string]any{"type": "0x34", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCVehicleInfoData(body[cursor : cursor+7])
|
||||
units = append(units, map[string]any{"type": "0x34", "name": "gd_fc_vehicle_info", "value": unit})
|
||||
fields["gd_fc_vehicle_hydrogen_mass_kg"] = unit["hydrogen_mass_kg"]
|
||||
cursor += 7
|
||||
case 0x80:
|
||||
if len(body[cursor:]) < 11 {
|
||||
units = append(units, map[string]any{"type": "0x80", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCDemoExtensionData(body[cursor : cursor+11])
|
||||
units = append(units, map[string]any{"type": "0x80", "name": "gd_fc_demo_extension", "value": unit})
|
||||
fields["gd_fc_demo_stack_temp_c"] = unit["stack_temp_c"]
|
||||
cursor += 11
|
||||
case 0x83:
|
||||
size, ok := gdFCVendorTLVDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x83", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCVendorTLVData(0x83, body[cursor:cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x83", "name": "gd_fc_vendor_tlv", "value": unit})
|
||||
cursor += size
|
||||
default:
|
||||
units = append(units, map[string]any{
|
||||
"type": fmt.Sprintf("0x%02X", unitType),
|
||||
"raw_tail": strings.ToUpper(hex.EncodeToString(body[cursor:])),
|
||||
"parse": "unknown_unit",
|
||||
"byte_size": len(body) - cursor,
|
||||
})
|
||||
return eventTime, units
|
||||
}
|
||||
}
|
||||
return eventTime, units
|
||||
}
|
||||
|
||||
func frameLength(frameStart []byte, bodyLen int) int {
|
||||
standardLen := headerLen + bodyLen + 1
|
||||
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
|
||||
return standardLen
|
||||
}
|
||||
// Some field platform-login frames declare the standard 41-byte body but
|
||||
// carry a 32-byte password, making the actual body 53 bytes. Only consume
|
||||
// the extended form when its own BCC verifies so the splitter does not eat
|
||||
// bytes from the next frame.
|
||||
extendedLen := headerLen + 53 + 1
|
||||
if len(frameStart) < extendedLen {
|
||||
return standardLen
|
||||
}
|
||||
extended := frameStart[:extendedLen]
|
||||
if bcc(extended[2:len(extended)-1]) == extended[len(extended)-1] {
|
||||
return extendedLen
|
||||
}
|
||||
return standardLen
|
||||
}
|
||||
|
||||
func isExtendedPlatformLogin(command byte, declaredBodyLen int, actualBodyLen int) bool {
|
||||
return command == 0x05 && declaredBodyLen == 41 && actualBodyLen == 53
|
||||
}
|
||||
|
||||
func parsePlatformLogin(body []byte) map[string]any {
|
||||
login := map[string]any{
|
||||
"raw_body_length": len(body),
|
||||
}
|
||||
if len(body) < 9 {
|
||||
login["error"] = "truncated"
|
||||
return login
|
||||
}
|
||||
loginTime := parseGBTime(body[:6])
|
||||
if !loginTime.IsZero() {
|
||||
login["login_time"] = loginTime.Format(time.RFC3339)
|
||||
}
|
||||
login["serial_no"] = binary.BigEndian.Uint16(body[6:8])
|
||||
if len(body) >= 21 {
|
||||
login["username"] = trimASCII(body[8:20])
|
||||
}
|
||||
if len(body) > 21 {
|
||||
passwordEnd := len(body) - 1
|
||||
login["password"] = trimASCII(body[20:passwordEnd])
|
||||
login["encrypt_rule"] = body[passwordEnd]
|
||||
}
|
||||
return login
|
||||
}
|
||||
|
||||
func parseVehicleData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"vehicle_status": int(data[0]),
|
||||
"charge_status": int(data[1]),
|
||||
"running_mode": int(data[2]),
|
||||
"speed_kmh": float64(binary.BigEndian.Uint16(data[3:5])) / 10,
|
||||
"total_mileage_km": float64(binary.BigEndian.Uint32(data[5:9])) / 10,
|
||||
"total_voltage_v": float64(binary.BigEndian.Uint16(data[9:11])) / 10,
|
||||
"total_current_a": float64(binary.BigEndian.Uint16(data[11:13]))/10 - 1000,
|
||||
"soc_percent": int(data[13]),
|
||||
"dc_dc_status": int(data[14]),
|
||||
"gear": int(data[15]),
|
||||
"insulation_kohm": binary.BigEndian.Uint16(data[16:18]),
|
||||
"accelerator_pct": int(data[18]),
|
||||
"brake_pct": int(data[19]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseDriveMotorData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
motors := make([]map[string]any, 0, count)
|
||||
cursor := 1
|
||||
for i := 0; i < count; i++ {
|
||||
motors = append(motors, map[string]any{
|
||||
"serial_no": int(data[cursor]),
|
||||
"state": int(data[cursor+1]),
|
||||
"controller_temperature_c": int(data[cursor+2]) - 40,
|
||||
"speed_rpm": int(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) - 20000,
|
||||
"torque_nm": float64(int(binary.BigEndian.Uint16(data[cursor+5:cursor+7]))-20000) / 10,
|
||||
"motor_temperature_c": int(data[cursor+7]) - 40,
|
||||
"controller_voltage_v": float64(binary.BigEndian.Uint16(data[cursor+8:cursor+10])) / 10,
|
||||
"controller_current_a": float64(binary.BigEndian.Uint16(data[cursor+10:cursor+12]))/10 - 1000,
|
||||
})
|
||||
cursor += 12
|
||||
}
|
||||
return map[string]any{
|
||||
"count": count,
|
||||
"motors": motors,
|
||||
}
|
||||
}
|
||||
|
||||
func parseFuelCellData(data []byte) map[string]any {
|
||||
probeCount := int(binary.BigEndian.Uint16(data[6:8]))
|
||||
probes := make([]int, 0, probeCount)
|
||||
cursor := 8
|
||||
for i := 0; i < probeCount; i++ {
|
||||
probes = append(probes, int(data[cursor])-40)
|
||||
cursor++
|
||||
}
|
||||
out := map[string]any{
|
||||
"fuel_cell_voltage_v": float64(binary.BigEndian.Uint16(data[0:2])) / 10,
|
||||
"fuel_cell_current_a": float64(binary.BigEndian.Uint16(data[2:4])) / 10,
|
||||
"hydrogen_consumption_kg_per_100km": float64(binary.BigEndian.Uint16(data[4:6])) / 100,
|
||||
"temperature_probe_count": probeCount,
|
||||
"temperature_probe_values_c": probes,
|
||||
"max_hydrogen_temperature_c": float64(binary.BigEndian.Uint16(data[cursor:cursor+2]))/10 - 40,
|
||||
"max_hydrogen_temperature_probe_id": int(data[cursor+2]),
|
||||
"max_hydrogen_concentration_fraction": float64(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) * 0.000001,
|
||||
"max_hydrogen_concentration_probe_id": int(data[cursor+5]),
|
||||
"max_hydrogen_pressure_mpa": float64(binary.BigEndian.Uint16(data[cursor+6:cursor+8])) / 10,
|
||||
"max_hydrogen_pressure_probe_id": int(data[cursor+8]),
|
||||
"dc_dc_status": int(data[cursor+9]),
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseEngineData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"engine_status": int(data[0]),
|
||||
"crank_speed_rpm": binary.BigEndian.Uint16(data[1:3]),
|
||||
"fuel_rate": binary.BigEndian.Uint16(data[3:5]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseExtremeData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"max_voltage_subsystem_no": int(data[0]),
|
||||
"max_voltage_cell_no": int(data[1]),
|
||||
"max_voltage_v": float64(binary.BigEndian.Uint16(data[2:4])) / 1000,
|
||||
"min_voltage_subsystem_no": int(data[4]),
|
||||
"min_voltage_cell_no": int(data[5]),
|
||||
"min_voltage_v": float64(binary.BigEndian.Uint16(data[6:8])) / 1000,
|
||||
"max_temp_subsystem_no": int(data[8]),
|
||||
"max_temp_probe_no": int(data[9]),
|
||||
"max_temp_c": int(data[10]) - 40,
|
||||
"min_temp_subsystem_no": int(data[11]),
|
||||
"min_temp_probe_no": int(data[12]),
|
||||
"min_temp_c": int(data[13]) - 40,
|
||||
}
|
||||
}
|
||||
|
||||
func alarmDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 5 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 5
|
||||
for i := 0; i < 4; i++ {
|
||||
if len(data[cursor:]) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
count := int(data[cursor])
|
||||
cursor++
|
||||
cursor += count * 4
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseAlarmData(data []byte) map[string]any {
|
||||
cursor := 5
|
||||
readList := func() []string {
|
||||
count := int(data[cursor])
|
||||
cursor++
|
||||
values := make([]string, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
values = append(values, fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[cursor:cursor+4])))
|
||||
cursor += 4
|
||||
}
|
||||
return values
|
||||
}
|
||||
return map[string]any{
|
||||
"max_alarm_level": int(data[0]),
|
||||
"general_alarm_flag": fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[1:5])),
|
||||
"battery_faults": readList(),
|
||||
"motor_faults": readList(),
|
||||
"engine_faults": readList(),
|
||||
"other_faults": readList(),
|
||||
}
|
||||
}
|
||||
|
||||
func voltageDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 10 {
|
||||
return 0, false
|
||||
}
|
||||
frameCellCount := int(data[cursor+9])
|
||||
cursor += 10 + frameCellCount*2
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseVoltageData(data []byte) map[string]any {
|
||||
subCount := int(data[0])
|
||||
cursor := 1
|
||||
totalVoltage := 0.0
|
||||
maxCell := 0.0
|
||||
minCell := 0.0
|
||||
cellSeen := false
|
||||
for i := 0; i < subCount; i++ {
|
||||
totalVoltage += float64(binary.BigEndian.Uint16(data[cursor+1:cursor+3])) / 10
|
||||
cellCount := int(binary.BigEndian.Uint16(data[cursor+5 : cursor+7]))
|
||||
frameCellCount := int(data[cursor+9])
|
||||
cursor += 10
|
||||
for c := 0; c < frameCellCount; c++ {
|
||||
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
|
||||
if !cellSeen || voltage > maxCell {
|
||||
maxCell = voltage
|
||||
}
|
||||
if !cellSeen || voltage < minCell {
|
||||
minCell = voltage
|
||||
}
|
||||
cellSeen = true
|
||||
cursor += 2
|
||||
}
|
||||
_ = cellCount
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": subCount,
|
||||
"total_voltage_v": totalVoltage,
|
||||
"max_cell_v": maxCell,
|
||||
"min_cell_v": minCell,
|
||||
}
|
||||
}
|
||||
|
||||
func temperatureDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 3 {
|
||||
return 0, false
|
||||
}
|
||||
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
|
||||
cursor += 3 + probeCount
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseTemperatureData(data []byte) map[string]any {
|
||||
subCount := int(data[0])
|
||||
cursor := 1
|
||||
maxTemp := 0
|
||||
minTemp := 0
|
||||
seen := false
|
||||
for i := 0; i < subCount; i++ {
|
||||
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
|
||||
cursor += 3
|
||||
for p := 0; p < probeCount; p++ {
|
||||
temp := int(data[cursor]) - 40
|
||||
if !seen || temp > maxTemp {
|
||||
maxTemp = temp
|
||||
}
|
||||
if !seen || temp < minTemp {
|
||||
minTemp = temp
|
||||
}
|
||||
seen = true
|
||||
cursor++
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": subCount,
|
||||
"max_temp_c": maxTemp,
|
||||
"min_temp_c": minTemp,
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCStackDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 22 {
|
||||
return 0, false
|
||||
}
|
||||
frameCellCount := int(data[cursor+21])
|
||||
cursor += 22 + frameCellCount*2
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseGdFCStackData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
cursor := 1
|
||||
summaries := make([]map[string]any, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
frameCellCount := int(data[cursor+21])
|
||||
summary := map[string]any{
|
||||
"engine_work_state": int(data[cursor]),
|
||||
"stack_water_outlet_temp_c": nullableTempOffset40(data[cursor+1]),
|
||||
"hydrogen_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+2:cursor+4], 0.1, -100),
|
||||
"air_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+4:cursor+6], 0.1, -100),
|
||||
"air_inlet_temp_c": nullableTempOffset40(data[cursor+6]),
|
||||
"max_cell_voltage_id": nullableU16(data[cursor+7 : cursor+9]),
|
||||
"min_cell_voltage_id": nullableU16(data[cursor+9 : cursor+11]),
|
||||
"max_cell_voltage_v": nullableScaledU16(data[cursor+11:cursor+13], 0.001),
|
||||
"min_cell_voltage_v": nullableScaledU16(data[cursor+13:cursor+15], 0.001),
|
||||
"avg_cell_voltage_v": nullableScaledU16(data[cursor+15:cursor+17], 0.001),
|
||||
"cell_count": nullableU16(data[cursor+17 : cursor+19]),
|
||||
"frame_cell_start": nullableU16(data[cursor+19 : cursor+21]),
|
||||
"frame_cell_count": frameCellCount,
|
||||
}
|
||||
cursor += 22
|
||||
maxCell := 0.0
|
||||
minCell := 0.0
|
||||
seen := false
|
||||
for c := 0; c < frameCellCount; c++ {
|
||||
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
|
||||
if !seen || voltage > maxCell {
|
||||
maxCell = voltage
|
||||
}
|
||||
if !seen || voltage < minCell {
|
||||
minCell = voltage
|
||||
}
|
||||
seen = true
|
||||
cursor += 2
|
||||
}
|
||||
if seen {
|
||||
summary["frame_max_cell_voltage_v"] = maxCell
|
||||
summary["frame_min_cell_voltage_v"] = minCell
|
||||
}
|
||||
summaries = append(summaries, summary)
|
||||
}
|
||||
return map[string]any{
|
||||
"stack_count": count,
|
||||
"summaries": summaries,
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCAuxiliaryDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
size := 1 + int(data[0])*16
|
||||
return size, len(data) >= size
|
||||
}
|
||||
|
||||
func parseGdFCAuxiliaryData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
cursor := 1
|
||||
subsystems := make([]map[string]any, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
subsystems = append(subsystems, map[string]any{
|
||||
"air_compressor_motor_voltage_v": nullableScaledU16(data[cursor:cursor+2], 0.1),
|
||||
"air_compressor_power_kw": nullableU16WithOffset(data[cursor+2:cursor+4], -5),
|
||||
"hydrogen_pump_motor_voltage_v": nullableScaledU16(data[cursor+4:cursor+6], 0.2),
|
||||
"hydrogen_pump_power_kw": nullableU16WithOffset(data[cursor+6:cursor+8], -5),
|
||||
"water_pump_voltage_v": nullableScaledU16(data[cursor+8:cursor+10], 0.1),
|
||||
"ptc_voltage_v": nullableScaledU16(data[cursor+10:cursor+12], 0.1),
|
||||
"ptc_power_kw": nullableU16WithOffset(data[cursor+12:cursor+14], -5),
|
||||
"low_voltage_battery_voltage_v": nullableScaledU16(data[cursor+14:cursor+16], 0.1),
|
||||
})
|
||||
cursor += 16
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": count,
|
||||
"subsystems": subsystems,
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCDcDcData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"input_voltage_v": nullableScaledU16(data[0:2], 0.1),
|
||||
"input_current_a": nullableScaledU16(data[2:4], 0.1),
|
||||
"output_voltage_v": nullableScaledU16(data[4:6], 0.1),
|
||||
"output_current_a": nullableScaledU16(data[6:8], 0.1),
|
||||
"controller_temp_c": nullableTempOffset40(data[8]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCAirConditionerData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"status": int(data[0]),
|
||||
"power_kw": nullableU16WithOffset(data[1:3], -5),
|
||||
"compressor_input_voltage_v": nullableScaledU16(data[3:5], 0.1),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCVehicleInfoData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"collision_alarm": int(data[0]),
|
||||
"ambient_temp_c": nullableU16WithOffset(data[1:3], -40),
|
||||
"ambient_pressure_kpa": nullableScaledU16WithOffset(data[3:5], 0.1, -100),
|
||||
"hydrogen_mass_kg": nullableScaledU16(data[5:7], 0.1),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCDemoExtensionData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"declared_length": int(binary.BigEndian.Uint16(data[0:2])),
|
||||
"stack_temp_c": nullableTempOffset40(data[2]),
|
||||
"air_compressor_voltage_v": nullableScaledU16(data[3:5], 0.1),
|
||||
"air_compressor_current_a": nullableScaledU16WithOffset(data[5:7], 0.1, -1000),
|
||||
"hydrogen_pump_voltage_v": nullableScaledU16(data[7:9], 0.1),
|
||||
"hydrogen_pump_current_a": nullableScaledU16WithOffset(data[9:11], 0.1, -1000),
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCVendorTLVDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 2 {
|
||||
return 0, false
|
||||
}
|
||||
size := 2 + int(binary.BigEndian.Uint16(data[0:2]))
|
||||
return size, len(data) >= size
|
||||
}
|
||||
|
||||
func parseGdFCVendorTLVData(typeCode byte, data []byte) map[string]any {
|
||||
length := int(binary.BigEndian.Uint16(data[0:2]))
|
||||
return map[string]any{
|
||||
"type": fmt.Sprintf("0x%02X", typeCode),
|
||||
"declared_length": length,
|
||||
"payload_hex": strings.ToUpper(hex.EncodeToString(data[2:])),
|
||||
}
|
||||
}
|
||||
|
||||
func nullableU16(data []byte) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return int(value)
|
||||
}
|
||||
|
||||
func nullableU16WithOffset(data []byte, offset int) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return int(value) + offset
|
||||
}
|
||||
|
||||
func nullableScaledU16(data []byte, scale float64) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return float64(value) * scale
|
||||
}
|
||||
|
||||
func nullableScaledU16WithOffset(data []byte, scale float64, offset float64) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return float64(value)*scale + offset
|
||||
}
|
||||
|
||||
func nullableTempOffset40(value byte) any {
|
||||
if value == 0xfe || value == 0xff {
|
||||
return nil
|
||||
}
|
||||
return int(value) - 40
|
||||
}
|
||||
|
||||
func parsePositionData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"position_status": int(data[0]),
|
||||
"longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
|
||||
"latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
func indexStart(data []byte) int {
|
||||
for i := 0; i+1 < len(data); i++ {
|
||||
if _, ok := version(data[i], data[i+1]); ok {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func version(a byte, b byte) (string, bool) {
|
||||
switch {
|
||||
case a == '#' && b == '#':
|
||||
return "V2016", true
|
||||
case a == '$' && b == '$':
|
||||
return "V2025", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func parseGBTime(data []byte) time.Time {
|
||||
if len(data) != 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
year := 2000 + int(data[0])
|
||||
month := time.Month(data[1])
|
||||
day := int(data[2])
|
||||
hour := int(data[3])
|
||||
minute := int(data[4])
|
||||
second := int(data[5])
|
||||
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func trimASCII(data []byte) string {
|
||||
return strings.TrimRight(string(data), "\x00 ")
|
||||
}
|
||||
|
||||
func bcc(data []byte) byte {
|
||||
var out byte
|
||||
for _, value := range data {
|
||||
out ^= value
|
||||
}
|
||||
return out
|
||||
}
|
||||
258
go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
Normal file
258
go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
Normal file
@@ -0,0 +1,258 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
|
||||
|
||||
func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
|
||||
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39})
|
||||
second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
|
||||
stream := append([]byte{0x00, 0x01}, first...)
|
||||
stream = append(stream, second[:len(second)-3]...)
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 {
|
||||
t.Fatalf("expected one complete frame, got %d", len(frames))
|
||||
}
|
||||
if string(remainder) != string(second[:len(second)-3]) {
|
||||
t.Fatalf("expected partial second frame as remainder, got %s", hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x02" {
|
||||
t.Fatalf("unexpected envelope header: %#v", env)
|
||||
}
|
||||
if env.VIN != "LNBSCB3D4R1234567" {
|
||||
t.Fatalf("unexpected vin: %q", env.VIN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameRejectsBadBCC(t *testing.T) {
|
||||
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
frame[len(frame)-1] ^= 0xff
|
||||
|
||||
if _, err := ParseFrame(frame, 1782745114999, ""); err == nil {
|
||||
t.Fatal("expected bad bcc error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFramesSupportsExtendedPlatformLogin(t *testing.T) {
|
||||
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x1f, 0x00, 0x01}
|
||||
body = append(body, fixedASCII("Hyundai", 12)...)
|
||||
body = append(body, fixedASCII("f2e3445d7cda409fb4f278f6fb890734", 32)...)
|
||||
body = append(body, 0x01)
|
||||
frame := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
|
||||
|
||||
frames, remainder, err := ExtractFrames(append(frame, []byte{0x23, 0x23, 0x05}...))
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 3 {
|
||||
t.Fatalf("unexpected split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
if len(frames[0]) != headerLen+53+1 {
|
||||
t.Fatalf("extended frame len = %d", len(frames[0]))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "8.134.95.166:39376")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x05" || env.ParseStatus != envelope.ParseOK {
|
||||
t.Fatalf("unexpected env: %#v", env)
|
||||
}
|
||||
header := env.Parsed["header"].(map[string]any)
|
||||
if header["body_length"] != 41 || header["actual_body_length"] != 53 {
|
||||
t.Fatalf("unexpected length header: %#v", header)
|
||||
}
|
||||
login := env.Parsed["platform_login"].(map[string]any)
|
||||
if login["username"] != "Hyundai" || login["password"] != "f2e3445d7cda409fb4f278f6fb890734" {
|
||||
t.Fatalf("unexpected platform login: %#v", login)
|
||||
}
|
||||
if login["login_time"] != "2026-07-02T00:38:31+08:00" {
|
||||
t.Fatalf("unexpected login time: %#v", login)
|
||||
}
|
||||
if env.Fields["platform_account"] != "Hyundai" {
|
||||
t.Fatalf("platform account not exposed: %#v", env.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponseEchoesRawVINAndOriginalTime(t *testing.T) {
|
||||
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x0f, 0x00, 0x01}
|
||||
body = append(body, fixedASCII("Hyundai", 12)...)
|
||||
body = append(body, fixedASCII("ack-test-password", 32)...)
|
||||
body = append(body, 0x01)
|
||||
request := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
|
||||
env, err := ParseFrame(request, 1782914969584, "8.134.95.166:39376")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := AutoResponse(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("AutoResponse() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
if response[2] != 0x05 || response[3] != 0x01 {
|
||||
t.Fatalf("unexpected response header: %x", response[:4])
|
||||
}
|
||||
if hex.EncodeToString(response[4:21]) != "0000000000000000000000000000000000" {
|
||||
t.Fatalf("raw vin not echoed: %x", response[4:21])
|
||||
}
|
||||
if hex.EncodeToString(response[24:30]) != "1a070200260f" {
|
||||
t.Fatalf("timestamp body not preserved: %x", response[24:30])
|
||||
}
|
||||
if got, want := bcc(response[2:len(response)-1]), response[len(response)-1]; got != want {
|
||||
t.Fatalf("response bcc got=0x%02x want=0x%02x", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
|
||||
body := []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39}
|
||||
body = append(body, 0x01)
|
||||
body = append(body,
|
||||
0x01, // vehicle status
|
||||
0x03, // charge status
|
||||
0x02, // running mode
|
||||
0x01, 0x2c, // speed: 30.0km/h
|
||||
0x00, 0x01, 0x86, 0xa0, // total mileage: 10000.0km
|
||||
0x15, 0xe5, // total voltage: 560.5V
|
||||
0x27, 0x10, // total current: 0A after -1000 offset
|
||||
85, 0x01, 0x00,
|
||||
0x27, 0x10,
|
||||
0x00, 0x00)
|
||||
body = append(body, 0x05)
|
||||
body = append(body,
|
||||
0x00,
|
||||
0x07, 0x36, 0x50, 0x40, // longitude: 121.000000
|
||||
0x01, 0xd2, 0x4f, 0x00) // latitude: 30.560000
|
||||
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", body)
|
||||
|
||||
env, err := ParseFrame(frame, 1782745114999, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 30.0)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10000.0)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121.0)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.56)
|
||||
if env.Fields[envelope.FieldSOCPercent] != 85 {
|
||||
t.Fatalf("unexpected soc: %#v", env.Fields[envelope.FieldSOCPercent])
|
||||
}
|
||||
if env.EventTimeMS == 1782745114999 {
|
||||
t.Fatal("event time should come from GB32960 device timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testing.T) {
|
||||
frame, err := hex.DecodeString(realFuelCellFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frame, 1782914969584, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LB9A32A21R0LS1707" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 53490.9)
|
||||
assertFloatField(t, env, "fuel_cell_hydrogen_consumption_kg_per_100km", 1.8)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 120.800326)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 31.634907)
|
||||
assertFloatField(t, env, "gd_fc_stack_hydrogen_inlet_pressure_kpa", 97.0)
|
||||
assertFloatField(t, env, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
|
||||
assertIntField(t, env, "gd_fc_demo_stack_temp_c", 65)
|
||||
|
||||
units, ok := env.Parsed["data_units"].([]map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("data_units missing: %#v", env.Parsed["data_units"])
|
||||
}
|
||||
gotTypes := make([]string, 0, len(units))
|
||||
for _, unit := range units {
|
||||
gotTypes = append(gotTypes, fmt.Sprint(unit["type"]))
|
||||
}
|
||||
wantTypes := []string{"0x01", "0x02", "0x03", "0x04", "0x05", "0x06", "0x07", "0x08", "0x09", "0x30", "0x31", "0x32", "0x33", "0x34", "0x80", "0x83"}
|
||||
if fmt.Sprint(gotTypes) != fmt.Sprint(wantTypes) {
|
||||
t.Fatalf("unit types = %v, want %v", gotTypes, wantTypes)
|
||||
}
|
||||
if units[1]["name"] != "drive_motor" {
|
||||
t.Fatalf("unexpected motor unit: %#v", units[1])
|
||||
}
|
||||
if units[2]["name"] != "fuel_cell" {
|
||||
t.Fatalf("unexpected fuel cell unit: %#v", units[2])
|
||||
}
|
||||
if units[9]["name"] != "gd_fc_stack" {
|
||||
t.Fatalf("unexpected gd stack unit: %#v", units[9])
|
||||
}
|
||||
if units[10]["name"] != "gd_fc_auxiliary" {
|
||||
t.Fatalf("unexpected gd auxiliary unit: %#v", units[10])
|
||||
}
|
||||
if units[15]["name"] != "gd_fc_vendor_tlv" {
|
||||
t.Fatalf("unexpected gd vendor tlv unit: %#v", units[15])
|
||||
}
|
||||
tlv := units[15]["value"].(map[string]any)
|
||||
if tlv["declared_length"] != 36 {
|
||||
t.Fatalf("unexpected vendor tlv: %#v", tlv)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if math.Abs(got-want) > 0.000001 {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntField(t *testing.T, env envelope.FrameEnvelope, key string, want int) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(int)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not int: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func buildFrame(command byte, response byte, vin string, body []byte) []byte {
|
||||
return buildFrameWithDeclaredLength(command, response, vin, body, len(body))
|
||||
}
|
||||
|
||||
func buildFrameWithDeclaredLength(command byte, response byte, vin string, body []byte, declaredBodyLen int) []byte {
|
||||
frame := []byte{'#', '#', command, response}
|
||||
vinBytes := []byte(vin)
|
||||
if len(vinBytes) < 17 {
|
||||
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
|
||||
}
|
||||
frame = append(frame, vinBytes[:17]...)
|
||||
frame = append(frame, 0x01, byte(declaredBodyLen>>8), byte(declaredBodyLen))
|
||||
frame = append(frame, body...)
|
||||
return append(frame, bcc(frame[2:]))
|
||||
}
|
||||
|
||||
func fixedASCII(value string, size int) []byte {
|
||||
out := make([]byte, size)
|
||||
copy(out, []byte(value))
|
||||
return out
|
||||
}
|
||||
80
go/vehicle-gateway/internal/protocol/gb32960/response.go
Normal file
80
go/vehicle-gateway/internal/protocol/gb32960/response.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrResponseFrameTooShort = errors.New("gb32960 response frame too short")
|
||||
|
||||
const (
|
||||
responseSuccess = byte(0x01)
|
||||
responseCommand = byte(0xfe)
|
||||
encryptNone = byte(0x01)
|
||||
)
|
||||
|
||||
// AutoResponse builds the protocol ACK for accepted upstream GB/T 32960 frames.
|
||||
// The ACK is written only after the caller has durably published the frame.
|
||||
func AutoResponse(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return nil, false, nil
|
||||
}
|
||||
if len(raw) < headerLen+1 {
|
||||
return nil, false, ErrResponseFrameTooShort
|
||||
}
|
||||
if raw[3] != responseCommand {
|
||||
return nil, false, nil
|
||||
}
|
||||
command := raw[2]
|
||||
if !shouldRespond(command) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
body := []byte(nil)
|
||||
if timestamp := responseTime(raw, command); !timestamp.IsZero() {
|
||||
body = encodeGBTime(timestamp)
|
||||
}
|
||||
return buildResponse(raw[0], command, responseSuccess, raw[4:21], body), true, nil
|
||||
}
|
||||
|
||||
func shouldRespond(command byte) bool {
|
||||
switch command {
|
||||
case 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func responseTime(raw []byte, command byte) time.Time {
|
||||
bodyLen := len(raw) - headerLen - 1
|
||||
if bodyLen >= 6 {
|
||||
return parseGBTime(raw[headerLen : headerLen+6])
|
||||
}
|
||||
return time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func encodeGBTime(t time.Time) []byte {
|
||||
local := t.In(time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
return []byte{
|
||||
byte(local.Year() - 2000),
|
||||
byte(local.Month()),
|
||||
byte(local.Day()),
|
||||
byte(local.Hour()),
|
||||
byte(local.Minute()),
|
||||
byte(local.Second()),
|
||||
}
|
||||
}
|
||||
|
||||
func buildResponse(start byte, command byte, responseFlag byte, vin17 []byte, body []byte) []byte {
|
||||
out := make([]byte, 0, headerLen+len(body)+1)
|
||||
out = append(out, start, start, command, responseFlag)
|
||||
out = append(out, vin17[:17]...)
|
||||
out = append(out, encryptNone, 0, 0)
|
||||
binary.BigEndian.PutUint16(out[22:24], uint16(len(body)))
|
||||
out = append(out, body...)
|
||||
return append(out, bcc(out[2:]))
|
||||
}
|
||||
600
go/vehicle-gateway/internal/protocol/jt808/parser.go
Normal file
600
go/vehicle-gateway/internal/protocol/jt808/parser.go
Normal file
@@ -0,0 +1,600 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFrameTooShort = errors.New("jt808 frame too short")
|
||||
ErrChecksum = errors.New("jt808 checksum mismatch")
|
||||
ErrEscape = errors.New("jt808 invalid escape sequence")
|
||||
)
|
||||
|
||||
type Location struct {
|
||||
AlarmFlag uint32
|
||||
StatusFlag uint32
|
||||
Latitude float64
|
||||
Longitude float64
|
||||
AltitudeM uint16
|
||||
SpeedKMH float64
|
||||
DirectionDeg uint16
|
||||
Time time.Time
|
||||
TotalMileageKM *float64
|
||||
Additional []AdditionalItem
|
||||
}
|
||||
|
||||
type AdditionalItem struct {
|
||||
ID byte
|
||||
Length int
|
||||
ValueHex string
|
||||
Parsed any
|
||||
}
|
||||
|
||||
// ExtractFrames splits JT/T 808 TCP streams by 0x7e delimiters and unescapes
|
||||
// complete frame payloads. Returned frames exclude the leading and trailing 0x7e.
|
||||
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
|
||||
start := -1
|
||||
for i, value := range stream {
|
||||
if value != 0x7e {
|
||||
continue
|
||||
}
|
||||
if start < 0 {
|
||||
start = i
|
||||
continue
|
||||
}
|
||||
if i == start+1 {
|
||||
start = i
|
||||
continue
|
||||
}
|
||||
frame, err := unescape(stream[start+1 : i])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
frames = append(frames, frame)
|
||||
start = i
|
||||
}
|
||||
if start >= 0 && start < len(stream)-1 {
|
||||
return frames, append([]byte(nil), stream[start:]...), nil
|
||||
}
|
||||
return frames, nil, nil
|
||||
}
|
||||
|
||||
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||
if len(raw) < 13 {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
if got, want := checksum(raw[:len(raw)-1]), raw[len(raw)-1]; got != want {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrChecksum, got, want)
|
||||
}
|
||||
|
||||
messageID := binary.BigEndian.Uint16(raw[0:2])
|
||||
props := binary.BigEndian.Uint16(raw[2:4])
|
||||
bodySize := props & 0x03ff
|
||||
versioned := props&0x4000 != 0
|
||||
headerLen := 12
|
||||
phoneStart := 4
|
||||
phoneEnd := 10
|
||||
serialStart := 10
|
||||
packageInfo := map[string]any(nil)
|
||||
protocolVersion := byte(0)
|
||||
if versioned {
|
||||
headerLen = 17
|
||||
if len(raw) < headerLen {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: versioned header len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
protocolVersion = raw[4]
|
||||
phoneStart = 5
|
||||
phoneEnd = 15
|
||||
serialStart = 15
|
||||
}
|
||||
subpackage := props&0x2000 != 0
|
||||
if subpackage {
|
||||
headerLen += 4
|
||||
}
|
||||
if len(raw) < headerLen+int(bodySize)+1 {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
|
||||
}
|
||||
if subpackage {
|
||||
packageInfo = map[string]any{
|
||||
"total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]),
|
||||
"index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]),
|
||||
}
|
||||
}
|
||||
|
||||
phoneBCD := raw[phoneStart:phoneEnd]
|
||||
phone := bcdString(phoneBCD)
|
||||
phoneTrimZero := strings.TrimLeft(phone, "0")
|
||||
if versioned && phoneTrimZero != "" {
|
||||
phone = phoneTrimZero
|
||||
}
|
||||
body := raw[headerLen : headerLen+int(bodySize)]
|
||||
parsed := map[string]any{
|
||||
"header": map[string]any{
|
||||
"message_id": fmt.Sprintf("0x%04X", messageID),
|
||||
"protocol_version": protocolVersion,
|
||||
"versioned": versioned,
|
||||
"properties": props,
|
||||
"body_size": bodySize,
|
||||
"phone_bcd_hex": strings.ToUpper(hex.EncodeToString(phoneBCD)),
|
||||
"phone": phone,
|
||||
"phone_trim_zero": phoneTrimZero,
|
||||
"sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
|
||||
"subpackage": subpackage,
|
||||
"package_info": packageInfo,
|
||||
"encryption_flags": (props >> 10) & 0x07,
|
||||
},
|
||||
}
|
||||
fields := map[string]any{}
|
||||
eventTimeMS := receivedAtMS
|
||||
|
||||
if messageID == 0x0200 {
|
||||
location, err := parseLocation(body)
|
||||
if err != nil {
|
||||
return envelope.FrameEnvelope{}, err
|
||||
}
|
||||
parsed["location"] = locationToMap(location)
|
||||
fields[envelope.FieldLatitude] = location.Latitude
|
||||
fields[envelope.FieldLongitude] = location.Longitude
|
||||
fields[envelope.FieldSpeedKMH] = location.SpeedKMH
|
||||
fields["altitude_m"] = location.AltitudeM
|
||||
fields["direction_deg"] = location.DirectionDeg
|
||||
fields["alarm_flag"] = location.AlarmFlag
|
||||
fields["status_flag"] = location.StatusFlag
|
||||
if !location.Time.IsZero() {
|
||||
eventTimeMS = location.Time.UnixMilli()
|
||||
fields["device_time"] = location.Time.Format(time.RFC3339)
|
||||
}
|
||||
if location.TotalMileageKM != nil {
|
||||
fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
|
||||
}
|
||||
} else if messageID == 0x0100 {
|
||||
registration, err := parseRegistration(body, versioned)
|
||||
if err != nil {
|
||||
return envelope.FrameEnvelope{}, err
|
||||
}
|
||||
parsed["registration"] = registration
|
||||
if deviceID, ok := registration["device_id"].(string); ok {
|
||||
fields["device_id"] = deviceID
|
||||
}
|
||||
if plate, ok := registration["plate"].(string); ok {
|
||||
fields["plate"] = plate
|
||||
}
|
||||
} else if messageID == 0x0102 {
|
||||
authentication := parseAuthentication(body, versioned)
|
||||
parsed["authentication"] = authentication
|
||||
if token, ok := authentication["token"].(string); ok {
|
||||
fields["auth_token"] = token
|
||||
}
|
||||
}
|
||||
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: fmt.Sprintf("0x%04X", messageID),
|
||||
Sequence: binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
|
||||
Phone: phone,
|
||||
SourceEndpoint: sourceEndpoint,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
if registration, ok := parsed["registration"].(map[string]any); ok {
|
||||
env.DeviceID, _ = registration["device_id"].(string)
|
||||
env.Plate, _ = registration["plate"].(string)
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func parseRegistration(body []byte, versioned bool) (map[string]any, error) {
|
||||
schemaVersion := "2011_2013"
|
||||
manufacturerSize := 5
|
||||
deviceTypeSize := 20
|
||||
deviceIDSize := 7
|
||||
if versioned {
|
||||
schemaVersion = "2019"
|
||||
manufacturerSize = 11
|
||||
deviceTypeSize = 30
|
||||
deviceIDSize = 30
|
||||
}
|
||||
minLen := 2 + 2 + manufacturerSize + deviceTypeSize + deviceIDSize + 1
|
||||
if len(body) < minLen {
|
||||
return nil, fmt.Errorf("%w: registration body len=%d", ErrFrameTooShort, len(body))
|
||||
}
|
||||
manufacturerStart := 4
|
||||
deviceTypeStart := manufacturerStart + manufacturerSize
|
||||
deviceIDStart := deviceTypeStart + deviceTypeSize
|
||||
plateColorStart := deviceIDStart + deviceIDSize
|
||||
registration := map[string]any{
|
||||
"schema_version": schemaVersion,
|
||||
"province": binary.BigEndian.Uint16(body[0:2]),
|
||||
"city": binary.BigEndian.Uint16(body[2:4]),
|
||||
"manufacturer": fixedText(body[manufacturerStart:deviceTypeStart]),
|
||||
"device_type": fixedText(body[deviceTypeStart:deviceIDStart]),
|
||||
"device_id": fixedText(body[deviceIDStart:plateColorStart]),
|
||||
"plate_color": body[plateColorStart],
|
||||
"plate": freeText(body[plateColorStart+1:]),
|
||||
}
|
||||
return registration, nil
|
||||
}
|
||||
|
||||
func parseAuthentication(body []byte, versioned bool) map[string]any {
|
||||
if versioned {
|
||||
if out, ok := parseVersionedAuthentication(body); ok {
|
||||
return out
|
||||
}
|
||||
if out, ok := parseDelimitedVersionedAuthentication(body); ok {
|
||||
return out
|
||||
}
|
||||
}
|
||||
return map[string]any{"token": freeText(body)}
|
||||
}
|
||||
|
||||
func parseVersionedAuthentication(body []byte) (map[string]any, bool) {
|
||||
if len(body) < 17 {
|
||||
return nil, false
|
||||
}
|
||||
tokenLen := int(body[0])
|
||||
imeiStart := 1 + tokenLen
|
||||
softwareLenStart := imeiStart + 15
|
||||
if tokenLen <= 0 || len(body) < softwareLenStart+1 {
|
||||
return nil, false
|
||||
}
|
||||
softwareLen := int(body[softwareLenStart])
|
||||
softwareStart := softwareLenStart + 1
|
||||
if len(body) < softwareStart+softwareLen {
|
||||
return nil, false
|
||||
}
|
||||
return map[string]any{
|
||||
"schema_version": "2019",
|
||||
"token": freeText(body[1:imeiStart]),
|
||||
"imei": fixedText(body[imeiStart:softwareLenStart]),
|
||||
"software_version": freeText(body[softwareStart : softwareStart+softwareLen]),
|
||||
}, true
|
||||
}
|
||||
|
||||
func parseDelimitedVersionedAuthentication(body []byte) (map[string]any, bool) {
|
||||
parts := bytes.Split(body, []byte{0xff, 0xff})
|
||||
if len(parts) < 2 {
|
||||
return nil, false
|
||||
}
|
||||
out := map[string]any{
|
||||
"schema_version": "2019_delimited",
|
||||
"token": freeText(parts[0]),
|
||||
"imei": freeText(parts[1]),
|
||||
}
|
||||
if len(parts) > 2 {
|
||||
out["software_version"] = freeText(parts[2])
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
func parseLocation(body []byte) (Location, error) {
|
||||
if len(body) < 28 {
|
||||
return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
|
||||
}
|
||||
status := binary.BigEndian.Uint32(body[4:8])
|
||||
latitude := float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000
|
||||
longitude := float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000
|
||||
if status&(1<<2) != 0 {
|
||||
latitude = -latitude
|
||||
}
|
||||
if status&(1<<3) != 0 {
|
||||
longitude = -longitude
|
||||
}
|
||||
location := Location{
|
||||
AlarmFlag: binary.BigEndian.Uint32(body[0:4]),
|
||||
StatusFlag: status,
|
||||
Latitude: latitude,
|
||||
Longitude: longitude,
|
||||
AltitudeM: binary.BigEndian.Uint16(body[16:18]),
|
||||
SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10,
|
||||
DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
|
||||
Time: parseBCDTime(body[22:28]),
|
||||
}
|
||||
location.Additional = parseAdditional(body[28:], &location)
|
||||
return location, nil
|
||||
}
|
||||
|
||||
func parseAdditional(data []byte, location *Location) []AdditionalItem {
|
||||
var out []AdditionalItem
|
||||
for len(data) >= 2 {
|
||||
id := data[0]
|
||||
size := int(data[1])
|
||||
data = data[2:]
|
||||
if len(data) < size {
|
||||
out = append(out, AdditionalItem{
|
||||
ID: id,
|
||||
Length: size,
|
||||
ValueHex: strings.ToUpper(hex.EncodeToString(data)),
|
||||
Parsed: "truncated",
|
||||
})
|
||||
return out
|
||||
}
|
||||
value := data[:size]
|
||||
item := AdditionalItem{
|
||||
ID: id,
|
||||
Length: size,
|
||||
ValueHex: strings.ToUpper(hex.EncodeToString(value)),
|
||||
}
|
||||
switch {
|
||||
case id == 0x01 && size == 4:
|
||||
rawMileage := binary.BigEndian.Uint32(value)
|
||||
mileage := float64(rawMileage) / 10
|
||||
location.TotalMileageKM = &mileage
|
||||
item.Parsed = map[string]any{
|
||||
"name": envelope.FieldTotalMileageKM,
|
||||
"raw_value_tenth_km": rawMileage,
|
||||
"value": mileage,
|
||||
"unit": "km",
|
||||
}
|
||||
case id == 0x02 && size == 2:
|
||||
rawFuel := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "fuel_l",
|
||||
"raw_value_tenth_l": rawFuel,
|
||||
"value": float64(rawFuel) / 10,
|
||||
"unit": "L",
|
||||
}
|
||||
case id == 0x03 && size == 2:
|
||||
rawSpeed := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "recorder_speed_kmh",
|
||||
"raw_value_tenth_kmh": rawSpeed,
|
||||
"value": float64(rawSpeed) / 10,
|
||||
"unit": "km/h",
|
||||
}
|
||||
case id == 0x04 && size == 2:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "manual_alarm_event_id",
|
||||
"value": binary.BigEndian.Uint16(value),
|
||||
}
|
||||
case id == 0x05 && size == 30:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "tire_pressure",
|
||||
"values": parseTirePressure(value),
|
||||
}
|
||||
case id == 0x06 && size == 2:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "carriage_temperature_c",
|
||||
"value": int16(binary.BigEndian.Uint16(value)),
|
||||
"unit": "C",
|
||||
}
|
||||
case id == 0x11 && (size == 1 || size == 5):
|
||||
parsed := map[string]any{
|
||||
"name": "overspeed_alarm",
|
||||
"location_type": value[0],
|
||||
}
|
||||
if size == 5 {
|
||||
parsed["area_id"] = binary.BigEndian.Uint32(value[1:5])
|
||||
}
|
||||
item.Parsed = parsed
|
||||
case id == 0x12 && size == 6:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "area_route_alarm",
|
||||
"location_type": value[0],
|
||||
"area_id": binary.BigEndian.Uint32(value[1:5]),
|
||||
"direction": value[5],
|
||||
}
|
||||
case id == 0x13 && size == 7:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "road_section_driving_time_alarm",
|
||||
"road_section_id": binary.BigEndian.Uint32(value[0:4]),
|
||||
"driving_time_seconds": binary.BigEndian.Uint16(value[4:6]),
|
||||
"result": value[6],
|
||||
"result_description": roadSectionDrivingTimeResult(value[6]),
|
||||
"driving_time_result_source": "jt808_0x0200_additional_0x13",
|
||||
}
|
||||
case id == 0x25 && size == 4:
|
||||
rawStatus := binary.BigEndian.Uint32(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "extended_vehicle_signal_status",
|
||||
"value": rawStatus,
|
||||
"bits": parseExtendedVehicleSignal(rawStatus),
|
||||
}
|
||||
case id == 0x2a && size == 2:
|
||||
rawStatus := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "io_status",
|
||||
"value": rawStatus,
|
||||
"bits": parseIOStatus(rawStatus),
|
||||
}
|
||||
case id == 0x2b && size == 4:
|
||||
analog := binary.BigEndian.Uint32(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "analog",
|
||||
"ad0": analog & 0xffff,
|
||||
"ad1": analog >> 16,
|
||||
"value": analog,
|
||||
}
|
||||
case id == 0x30 && size == 1:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "network_signal_strength",
|
||||
"value": value[0],
|
||||
}
|
||||
case id == 0x31 && size == 1:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "gnss_satellite_count",
|
||||
"value": value[0],
|
||||
}
|
||||
}
|
||||
out = append(out, item)
|
||||
data = data[size:]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func locationToMap(location Location) map[string]any {
|
||||
out := map[string]any{
|
||||
"alarm_flag": location.AlarmFlag,
|
||||
"status_flag": location.StatusFlag,
|
||||
"latitude": location.Latitude,
|
||||
"longitude": location.Longitude,
|
||||
"altitude_m": location.AltitudeM,
|
||||
"speed_kmh": location.SpeedKMH,
|
||||
"direction_deg": location.DirectionDeg,
|
||||
"additional": additionalToMaps(location.Additional),
|
||||
}
|
||||
if !location.Time.IsZero() {
|
||||
out["device_time"] = location.Time.Format(time.RFC3339)
|
||||
}
|
||||
if location.TotalMileageKM != nil {
|
||||
out[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func additionalToMaps(items []AdditionalItem) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(items))
|
||||
for _, item := range items {
|
||||
value := map[string]any{
|
||||
"id": fmt.Sprintf("0x%02X", item.ID),
|
||||
"length": item.Length,
|
||||
"value_hex": item.ValueHex,
|
||||
}
|
||||
if item.Parsed != nil {
|
||||
value["parsed"] = item.Parsed
|
||||
}
|
||||
out = append(out, value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseTirePressure(data []byte) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(data))
|
||||
for index, value := range data {
|
||||
out = append(out, map[string]any{
|
||||
"position": index,
|
||||
"value": value,
|
||||
"valid": value != 0xff,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseExtendedVehicleSignal(value uint32) map[string]bool {
|
||||
return map[string]bool{
|
||||
"low_beam": value&0x0001 != 0,
|
||||
"high_beam": value&0x0002 != 0,
|
||||
"right_turn": value&0x0004 != 0,
|
||||
"left_turn": value&0x0008 != 0,
|
||||
"brake": value&0x0010 != 0,
|
||||
"reverse_gear": value&0x0020 != 0,
|
||||
"fog_light": value&0x0040 != 0,
|
||||
"clearance_light": value&0x0080 != 0,
|
||||
"horn": value&0x0100 != 0,
|
||||
"air_conditioner": value&0x0200 != 0,
|
||||
"neutral": value&0x0400 != 0,
|
||||
"retarder": value&0x0800 != 0,
|
||||
"abs": value&0x1000 != 0,
|
||||
"heater": value&0x2000 != 0,
|
||||
"clutch": value&0x4000 != 0,
|
||||
}
|
||||
}
|
||||
|
||||
func parseIOStatus(value uint16) map[string]bool {
|
||||
return map[string]bool{
|
||||
"deep_sleep": value&0x0001 != 0,
|
||||
"sleep": value&0x0002 != 0,
|
||||
}
|
||||
}
|
||||
|
||||
func roadSectionDrivingTimeResult(value byte) string {
|
||||
if value == 0 {
|
||||
return "insufficient"
|
||||
}
|
||||
if value == 1 {
|
||||
return "too_long"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func bcdString(data []byte) string {
|
||||
out := make([]byte, 0, len(data)*2)
|
||||
for _, value := range data {
|
||||
out = append(out, '0'+((value>>4)&0x0f), '0'+(value&0x0f))
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func fixedText(data []byte) string {
|
||||
return strings.Trim(freeText(data), "\x00 ")
|
||||
}
|
||||
|
||||
func freeText(data []byte) string {
|
||||
data = bytes.Trim(data, "\x00 ")
|
||||
if len(data) == 0 {
|
||||
return ""
|
||||
}
|
||||
if utf8.Valid(data) {
|
||||
return string(data)
|
||||
}
|
||||
decoded, err := simplifiedchinese.GBK.NewDecoder().Bytes(data)
|
||||
if err == nil && utf8.Valid(decoded) {
|
||||
return string(decoded)
|
||||
}
|
||||
return strings.ToValidUTF8(string(data), "")
|
||||
}
|
||||
|
||||
func parseBCDTime(data []byte) time.Time {
|
||||
if len(data) != 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
year := 2000 + bcdByte(data[0])
|
||||
month := time.Month(bcdByte(data[1]))
|
||||
day := bcdByte(data[2])
|
||||
hour := bcdByte(data[3])
|
||||
minute := bcdByte(data[4])
|
||||
second := bcdByte(data[5])
|
||||
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func bcdByte(value byte) int {
|
||||
return int(value>>4)*10 + int(value&0x0f)
|
||||
}
|
||||
|
||||
func unescape(data []byte) ([]byte, error) {
|
||||
out := make([]byte, 0, len(data))
|
||||
for i := 0; i < len(data); i++ {
|
||||
if data[i] != 0x7d {
|
||||
out = append(out, data[i])
|
||||
continue
|
||||
}
|
||||
if i+1 >= len(data) {
|
||||
return nil, ErrEscape
|
||||
}
|
||||
i++
|
||||
switch data[i] {
|
||||
case 0x01:
|
||||
out = append(out, 0x7d)
|
||||
case 0x02:
|
||||
out = append(out, 0x7e)
|
||||
default:
|
||||
return nil, ErrEscape
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func checksum(data []byte) byte {
|
||||
var out byte
|
||||
for _, value := range data {
|
||||
out ^= value
|
||||
}
|
||||
return out
|
||||
}
|
||||
503
go/vehicle-gateway/internal/protocol/jt808/parser_test.go
Normal file
503
go/vehicle-gateway/internal/protocol/jt808/parser_test.go
Normal file
@@ -0,0 +1,503 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
|
||||
const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E"
|
||||
const fullAdditionalLocationFrame = "7e020000800123456789017fff000004000000080006eeb6ad02633df701380003006320070719235901040000000b02020016030200210402002c051e3737370000000000000000000000000000000000000000000000000000001105420000004212064d0000004d4d1307000000580058582504000000632a02000a2b040000001430011e31012806020001927e"
|
||||
|
||||
func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
|
||||
stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 {
|
||||
t.Fatalf("expected one complete frame, got %d", len(frames))
|
||||
}
|
||||
if hex.EncodeToString(remainder) != "7e02000000" {
|
||||
t.Fatalf("expected partial remainder, got %s", hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolJT808 {
|
||||
t.Fatalf("protocol = %q", env.Protocol)
|
||||
}
|
||||
if env.MessageID != "0x0200" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "013307795425" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.Sequence != 1 {
|
||||
t.Fatalf("sequence = %d", env.Sequence)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.590151)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121.069881)
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 23.0)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10241.2)
|
||||
if env.Fields["direction_deg"] != uint16(79) {
|
||||
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
|
||||
}
|
||||
|
||||
location, ok := env.Parsed["location"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("parsed location missing: %#v", env.Parsed)
|
||||
}
|
||||
additional, ok := location["additional"].([]map[string]any)
|
||||
if !ok || len(additional) == 0 {
|
||||
t.Fatalf("additional fields missing: %#v", location["additional"])
|
||||
}
|
||||
if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
|
||||
t.Fatalf("unexpected first additional item: %#v", additional[0])
|
||||
}
|
||||
parsed, ok := additional[0]["parsed"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("mileage parsed field missing: %#v", additional[0])
|
||||
}
|
||||
if parsed["raw_value_tenth_km"] != uint32(102412) || parsed["value"] != 10241.2 {
|
||||
t.Fatalf("unexpected mileage parsed value: %#v", parsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
|
||||
stream, err := hex.DecodeString(real2019LocationFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782914967713, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0200" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "14894135060" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.Sequence != 1995 {
|
||||
t.Fatalf("sequence = %d", env.Sequence)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.570155)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 117.178187)
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 33.4)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 7868.9)
|
||||
if env.Fields["altitude_m"] != uint16(13) {
|
||||
t.Fatalf("altitude = %#v", env.Fields["altitude_m"])
|
||||
}
|
||||
if env.Fields["direction_deg"] != uint16(232) {
|
||||
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
|
||||
}
|
||||
|
||||
header, ok := env.Parsed["header"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("parsed header missing: %#v", env.Parsed)
|
||||
}
|
||||
if header["versioned"] != true || header["protocol_version"] != byte(1) {
|
||||
t.Fatalf("unexpected versioned header: %#v", header)
|
||||
}
|
||||
if header["phone_bcd_hex"] != "00000000014894135060" {
|
||||
t.Fatalf("phone bcd = %#v", header["phone_bcd_hex"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesCommonLocationAdditionalItems(t *testing.T) {
|
||||
stream, err := hex.DecodeString(fullAdditionalLocationFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782914967713, "127.0.0.1:808")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Phone != "012345678901" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.1)
|
||||
|
||||
location := env.Parsed["location"].(map[string]any)
|
||||
additional := location["additional"].([]map[string]any)
|
||||
byID := map[string]map[string]any{}
|
||||
for _, item := range additional {
|
||||
byID[item["id"].(string)] = item["parsed"].(map[string]any)
|
||||
}
|
||||
if byID["0x01"]["raw_value_tenth_km"] != uint32(11) || byID["0x01"]["value"] != 1.1 {
|
||||
t.Fatalf("mileage parsed = %#v", byID["0x01"])
|
||||
}
|
||||
if byID["0x05"]["name"] != "tire_pressure" {
|
||||
t.Fatalf("tire pressure missing: %#v", byID["0x05"])
|
||||
}
|
||||
tireValues := byID["0x05"]["values"].([]map[string]any)
|
||||
if tireValues[0]["value"] != byte(0x37) || tireValues[0]["valid"] != true {
|
||||
t.Fatalf("tire pressure first value = %#v", tireValues[0])
|
||||
}
|
||||
if byID["0x11"]["location_type"] != byte(0x42) || byID["0x11"]["area_id"] != uint32(0x42) {
|
||||
t.Fatalf("overspeed parsed = %#v", byID["0x11"])
|
||||
}
|
||||
if byID["0x12"]["location_type"] != byte(0x4d) || byID["0x12"]["area_id"] != uint32(0x4d) || byID["0x12"]["direction"] != byte(0x4d) {
|
||||
t.Fatalf("area route parsed = %#v", byID["0x12"])
|
||||
}
|
||||
if byID["0x13"]["road_section_id"] != uint32(0x58) || byID["0x13"]["driving_time_seconds"] != uint16(0x58) {
|
||||
t.Fatalf("road section parsed = %#v", byID["0x13"])
|
||||
}
|
||||
ioBits := byID["0x2A"]["bits"].(map[string]bool)
|
||||
if ioBits["deep_sleep"] || !ioBits["sleep"] {
|
||||
t.Fatalf("io bits = %#v", ioBits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameRejectsTruncatedSubpackageHeader(t *testing.T) {
|
||||
payload := []byte{0x02, 0x00, 0x20, 0x00}
|
||||
payload = append(payload, encodeBCD("013307795425", 6)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, 1)
|
||||
payload = append(payload, checksum(payload))
|
||||
|
||||
_, err := ParseFrame(payload, 1782914967713, "115.231.168.135:43625")
|
||||
if err == nil {
|
||||
t.Fatal("expected truncated subpackage header to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLocationAppliesSouthWestStatusBits(t *testing.T) {
|
||||
body := make([]byte, 28)
|
||||
binary.BigEndian.PutUint32(body[4:8], (1<<2)|(1<<3))
|
||||
binary.BigEndian.PutUint32(body[8:12], 12345678)
|
||||
binary.BigEndian.PutUint32(body[12:16], 98765432)
|
||||
|
||||
location, err := parseLocation(body)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLocation() error = %v", err)
|
||||
}
|
||||
if location.Latitude != -12.345678 {
|
||||
t.Fatalf("latitude = %v", location.Latitude)
|
||||
}
|
||||
if location.Longitude != -98.765432 {
|
||||
t.Fatalf("longitude = %v", location.Longitude)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0001", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST123")...)
|
||||
|
||||
env, err := ParseFrame(buildFrame(0x0100, "013079963379", 7, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0100" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "013079963379" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.DeviceID != "DEV0001" {
|
||||
t.Fatalf("device id = %q", env.DeviceID)
|
||||
}
|
||||
if env.Plate != "TEST123" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
registration, ok := env.Parsed["registration"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("registration missing: %#v", env.Parsed)
|
||||
}
|
||||
if registration["manufacturer"] != "YUTNG" || registration["device_type"] != "ZK6105CHEVNPG4" {
|
||||
t.Fatalf("registration fields = %#v", registration)
|
||||
}
|
||||
if registration["plate_color"] != uint8(2) {
|
||||
t.Fatalf("plate color = %#v", registration["plate_color"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesJT8082019RegistrationIdentity(t *testing.T) {
|
||||
body := make([]byte, 0, 84)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTONG00001", 11)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4-2019", 30)
|
||||
body = appendFixedASCII(body, "DEV201900000000000000000000001", 30)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST2019")...)
|
||||
|
||||
env, err := ParseFrame(buildVersionedFrame(0x0100, "14894135060", 11, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Phone != "14894135060" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.DeviceID != "DEV201900000000000000000000001" {
|
||||
t.Fatalf("device id = %q", env.DeviceID)
|
||||
}
|
||||
if env.Plate != "TEST2019" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
registration, ok := env.Parsed["registration"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("registration missing: %#v", env.Parsed)
|
||||
}
|
||||
if registration["schema_version"] != "2019" || registration["manufacturer"] != "YUTONG00001" {
|
||||
t.Fatalf("registration fields = %#v", registration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0002", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte{0xBB, 0xA6, 'A', '5', '3', '3', '0', '1'}...)
|
||||
|
||||
env, err := ParseFrame(buildFrame(0x0100, "013079963380", 9, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Plate != "沪A53301" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesJT8082019Authentication(t *testing.T) {
|
||||
body := []byte{5}
|
||||
body = append(body, []byte("TOKEN")...)
|
||||
body = appendFixedASCII(body, "123456789012345", 15)
|
||||
body = append(body, 4)
|
||||
body = append(body, []byte("v1.2")...)
|
||||
|
||||
env, err := ParseFrame(buildVersionedFrame(0x0102, "14894135060", 12, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
auth, ok := env.Parsed["authentication"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("authentication missing: %#v", env.Parsed)
|
||||
}
|
||||
if auth["token"] != "TOKEN" || auth["imei"] != "123456789012345" || auth["software_version"] != "v1.2" {
|
||||
t.Fatalf("authentication = %#v", auth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesAuthenticationToken(t *testing.T) {
|
||||
env, err := ParseFrame(buildFrame(0x0102, "064646848757", 8, []byte("g7gps")), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0102" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
auth, ok := env.Parsed["authentication"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("authentication missing: %#v", env.Parsed)
|
||||
}
|
||||
if auth["token"] != "g7gps" {
|
||||
t.Fatalf("token = %#v", auth["token"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponderBuildsRegisterAck(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0001", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST123")...)
|
||||
request := buildFrame(0x0100, "013079963379", 9, body)
|
||||
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := NewAutoResponder("g7gps").Respond(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("Respond() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(response)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
payload := frames[0]
|
||||
if binary.BigEndian.Uint16(payload[0:2]) != 0x8100 {
|
||||
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
|
||||
}
|
||||
if hex.EncodeToString(payload[4:10]) != "013079963379" {
|
||||
t.Fatalf("phone bcd = %x", payload[4:10])
|
||||
}
|
||||
if binary.BigEndian.Uint16(payload[12:14]) != 9 || payload[14] != 0 || string(payload[15:20]) != "g7gps" {
|
||||
t.Fatalf("unexpected register ack body: %x", payload[12:len(payload)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponderBuildsVersionedGeneralAck(t *testing.T) {
|
||||
request := buildVersionedFrame(0x0200, "14894135060", 12, make([]byte, 28))
|
||||
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := NewAutoResponder("").Respond(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("Respond() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(response)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
payload := frames[0]
|
||||
if binary.BigEndian.Uint16(payload[0:2]) != 0x8001 {
|
||||
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
|
||||
}
|
||||
if binary.BigEndian.Uint16(payload[2:4])&0x4000 == 0 || payload[4] != 1 {
|
||||
t.Fatalf("versioned header not preserved: %x", payload[:5])
|
||||
}
|
||||
if hex.EncodeToString(payload[5:15]) != "00000000014894135060" {
|
||||
t.Fatalf("phone bcd = %x", payload[5:15])
|
||||
}
|
||||
bodyStart := 17
|
||||
if binary.BigEndian.Uint16(payload[bodyStart:bodyStart+2]) != 12 ||
|
||||
binary.BigEndian.Uint16(payload[bodyStart+2:bodyStart+4]) != 0x0200 ||
|
||||
payload[bodyStart+4] != 0 {
|
||||
t.Fatalf("unexpected general ack body: %x", payload[bodyStart:len(payload)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
|
||||
payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
|
||||
frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
|
||||
frame = append(frame, 0x7e)
|
||||
|
||||
frames, remainder, err := ExtractFrames(frame)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
if hex.EncodeToString(frames[0][:len(payload)]) != hex.EncodeToString(payload) {
|
||||
t.Fatalf("frame was not unescaped: %s", hex.EncodeToString(frames[0]))
|
||||
}
|
||||
}
|
||||
|
||||
func buildFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
|
||||
payload := make([]byte, 0, 12+len(body)+1)
|
||||
payload = binary.BigEndian.AppendUint16(payload, messageID)
|
||||
payload = binary.BigEndian.AppendUint16(payload, uint16(len(body)))
|
||||
payload = append(payload, encodeBCD(phone, 6)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return payload
|
||||
}
|
||||
|
||||
func buildVersionedFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
|
||||
payload := make([]byte, 0, 17+len(body)+1)
|
||||
payload = binary.BigEndian.AppendUint16(payload, messageID)
|
||||
payload = binary.BigEndian.AppendUint16(payload, 0x4000|uint16(len(body)))
|
||||
payload = append(payload, 1)
|
||||
payload = append(payload, encodeBCD(phone, 10)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return payload
|
||||
}
|
||||
|
||||
func appendFixedASCII(out []byte, value string, size int) []byte {
|
||||
start := len(out)
|
||||
out = append(out, make([]byte, size)...)
|
||||
copy(out[start:], []byte(value))
|
||||
return out
|
||||
}
|
||||
|
||||
func encodeBCD(value string, size int) []byte {
|
||||
out := make([]byte, size)
|
||||
if len(value)%2 == 1 {
|
||||
value = "0" + value
|
||||
}
|
||||
if len(value) > size*2 {
|
||||
value = value[len(value)-size*2:]
|
||||
}
|
||||
for len(value) < size*2 {
|
||||
value = "0" + value
|
||||
}
|
||||
for i := 0; i < size; i++ {
|
||||
out[i] = (value[i*2]-'0')<<4 | (value[i*2+1] - '0')
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func escape(payload []byte) []byte {
|
||||
var out []byte
|
||||
for _, value := range payload {
|
||||
switch value {
|
||||
case 0x7e:
|
||||
out = append(out, 0x7d, 0x02)
|
||||
case 0x7d:
|
||||
out = append(out, 0x7d, 0x01)
|
||||
default:
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
130
go/vehicle-gateway/internal/protocol/jt808/response.go
Normal file
130
go/vehicle-gateway/internal/protocol/jt808/response.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrResponseHeaderTooShort = errors.New("jt808 response header too short")
|
||||
|
||||
const (
|
||||
msgTerminalGeneralResponse = uint16(0x0001)
|
||||
msgTerminalRegister = uint16(0x0100)
|
||||
msgPlatformGeneralResponse = uint16(0x8001)
|
||||
msgPlatformRegisterAck = uint16(0x8100)
|
||||
)
|
||||
|
||||
type AutoResponder struct {
|
||||
authCode string
|
||||
}
|
||||
|
||||
func NewAutoResponder(authCode string) AutoResponder {
|
||||
if authCode == "" {
|
||||
authCode = "g7gps"
|
||||
}
|
||||
return AutoResponder{authCode: authCode}
|
||||
}
|
||||
|
||||
func (r AutoResponder) Respond(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return nil, false, nil
|
||||
}
|
||||
header, err := parseResponseHeader(raw)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if header.messageID == msgTerminalGeneralResponse {
|
||||
return nil, false, nil
|
||||
}
|
||||
if header.messageID == msgTerminalRegister {
|
||||
body := make([]byte, 3, 3+len(r.authCode))
|
||||
binary.BigEndian.PutUint16(body[0:2], header.sequence)
|
||||
body[2] = 0
|
||||
body = append(body, []byte(r.authCode)...)
|
||||
return encodeResponse(msgPlatformRegisterAck, header, body), true, nil
|
||||
}
|
||||
body := make([]byte, 5)
|
||||
binary.BigEndian.PutUint16(body[0:2], header.sequence)
|
||||
binary.BigEndian.PutUint16(body[2:4], header.messageID)
|
||||
body[4] = 0
|
||||
return encodeResponse(msgPlatformGeneralResponse, header, body), true, nil
|
||||
}
|
||||
|
||||
type responseHeader struct {
|
||||
messageID uint16
|
||||
versioned bool
|
||||
protocolVersion byte
|
||||
phoneBCD []byte
|
||||
sequence uint16
|
||||
}
|
||||
|
||||
func parseResponseHeader(raw []byte) (responseHeader, error) {
|
||||
if len(raw) < 12 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
props := binary.BigEndian.Uint16(raw[2:4])
|
||||
versioned := props&0x4000 != 0
|
||||
phoneStart := 4
|
||||
phoneLen := 6
|
||||
protocolVersion := byte(0)
|
||||
if versioned {
|
||||
if len(raw) < 17 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
protocolVersion = raw[4]
|
||||
phoneStart = 5
|
||||
phoneLen = 10
|
||||
}
|
||||
sequenceStart := phoneStart + phoneLen
|
||||
if len(raw) < sequenceStart+2 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
phone := append([]byte(nil), raw[phoneStart:sequenceStart]...)
|
||||
return responseHeader{
|
||||
messageID: binary.BigEndian.Uint16(raw[0:2]),
|
||||
versioned: versioned,
|
||||
protocolVersion: protocolVersion,
|
||||
phoneBCD: phone,
|
||||
sequence: binary.BigEndian.Uint16(raw[sequenceStart : sequenceStart+2]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func encodeResponse(messageID uint16, header responseHeader, body []byte) []byte {
|
||||
props := uint16(len(body)) & 0x03ff
|
||||
if header.versioned {
|
||||
props |= 0x4000
|
||||
}
|
||||
payload := make([]byte, 0, 4+1+len(header.phoneBCD)+2+len(body)+1)
|
||||
payload = appendU16(payload, messageID)
|
||||
payload = appendU16(payload, props)
|
||||
if header.versioned {
|
||||
payload = append(payload, header.protocolVersion)
|
||||
}
|
||||
payload = append(payload, header.phoneBCD...)
|
||||
payload = appendU16(payload, header.sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return frame(payload)
|
||||
}
|
||||
|
||||
func appendU16(out []byte, value uint16) []byte {
|
||||
return append(out, byte(value>>8), byte(value))
|
||||
}
|
||||
|
||||
func frame(payload []byte) []byte {
|
||||
out := []byte{0x7e}
|
||||
for _, value := range payload {
|
||||
switch value {
|
||||
case 0x7e:
|
||||
out = append(out, 0x7d, 0x02)
|
||||
case 0x7d:
|
||||
out = append(out, 0x7d, 0x01)
|
||||
default:
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
out = append(out, 0x7e)
|
||||
return out
|
||||
}
|
||||
201
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
201
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package yutongmqtt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrInvalidJSON = errors.New("yutong mqtt invalid json")
|
||||
|
||||
func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS int64) (envelope.FrameEnvelope, error) {
|
||||
root := map[string]any{}
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&root); err != nil {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %v", ErrInvalidJSON, err)
|
||||
}
|
||||
|
||||
data := objectValue(root, "data")
|
||||
if data == nil {
|
||||
data = root
|
||||
}
|
||||
deviceID := firstText(root, "device", "deviceId", "terminalId", "terminalID", "imei", "IMEI", "mqttDeviceId")
|
||||
externalVIN := firstText(root, "vin", "VIN", "externalVin", "vehicleVin")
|
||||
vin := externalVIN
|
||||
if vin == "" && looksLikeVIN(deviceID) {
|
||||
vin = deviceID
|
||||
}
|
||||
plate := firstText(root, "plateNo", "carNo", "plate", "vehicleNo")
|
||||
phone := firstText(root, "sim", "phone", "mobile")
|
||||
eventTimeMS := parseTimeMS(firstText(root, "time", "deviceTime", "eventTime", "gpsTime"), receivedAtMS)
|
||||
fields := fieldsFromData(data)
|
||||
|
||||
parsed := map[string]any{
|
||||
"endpoint": endpoint,
|
||||
"topic": topic,
|
||||
"root": root,
|
||||
"data": data,
|
||||
}
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
MessageID: "MQTT",
|
||||
VIN: vin,
|
||||
VehicleKeyHint: deviceID,
|
||||
Phone: phone,
|
||||
DeviceID: deviceID,
|
||||
Plate: plate,
|
||||
SourceEndpoint: "mqtt://" + endpoint + topic,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawText: string(payload),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func fieldsFromData(data map[string]any) map[string]any {
|
||||
fields := map[string]any{}
|
||||
if speed, ok := firstFloat(data, "METER_SPEED", "speed", "speed_kmh"); ok {
|
||||
fields[envelope.FieldSpeedKMH] = speed
|
||||
}
|
||||
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
|
||||
fields[envelope.FieldTotalMileageKM] = normalizeTotalMileageKM(mileage)
|
||||
}
|
||||
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
|
||||
fields[envelope.FieldSOCPercent] = soc
|
||||
}
|
||||
if longitude, ok := firstCoord(data, "LONGITUDE", "longitude", "lng"); ok {
|
||||
fields[envelope.FieldLongitude] = longitude
|
||||
}
|
||||
if latitude, ok := firstCoord(data, "LATITUDE", "latitude", "lat"); ok {
|
||||
fields[envelope.FieldLatitude] = latitude
|
||||
}
|
||||
if direction, ok := firstFloat(data, "GPSDirection", "direction", "direction_deg"); ok {
|
||||
fields["direction_deg"] = direction
|
||||
}
|
||||
if gear, ok := firstFloat(data, "ON_GEAR", "gear"); ok {
|
||||
fields["gear"] = gear
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func objectValue(root map[string]any, key string) map[string]any {
|
||||
value, ok := root[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
typed, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return typed
|
||||
}
|
||||
|
||||
func firstText(root map[string]any, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value, ok := root[key]; ok && value != nil {
|
||||
text := strings.TrimSpace(fmt.Sprint(value))
|
||||
if text != "" && text != "<nil>" {
|
||||
return text
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstFloat(root map[string]any, keys ...string) (float64, bool) {
|
||||
for _, key := range keys {
|
||||
value, ok := root[key]
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
if parsed, ok := toFloat(value); ok {
|
||||
return parsed, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func firstCoord(root map[string]any, keys ...string) (float64, bool) {
|
||||
value, ok := firstFloat(root, keys...)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if value == 0 || int64(value) == 999999 {
|
||||
return 0, false
|
||||
}
|
||||
if value > 1000 || value < -1000 {
|
||||
value = value / 1_000_000
|
||||
}
|
||||
return value, true
|
||||
}
|
||||
|
||||
func toFloat(value any) (float64, bool) {
|
||||
switch typed := value.(type) {
|
||||
case json.Number:
|
||||
parsed, err := typed.Float64()
|
||||
return parsed, err == nil
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func parseTimeMS(raw string, fallback int64) int64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
for _, layout := range []string{
|
||||
"2006-01-02 15:04:05.000",
|
||||
"2006-01-02 15:04:05",
|
||||
} {
|
||||
if parsed, err := time.ParseInLocation(layout, raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
}
|
||||
if len(raw) == 14 {
|
||||
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
}
|
||||
if parsed, err := time.Parse(time.RFC3339, raw); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
if epoch, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
||||
if epoch > 10_000_000_000 {
|
||||
return epoch
|
||||
}
|
||||
return epoch * 1000
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func looksLikeVIN(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
return len(value) == 17
|
||||
}
|
||||
|
||||
func normalizeTotalMileageKM(value float64) float64 {
|
||||
return value / 1000
|
||||
}
|
||||
112
go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go
Normal file
112
go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package yutongmqtt
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device": "LTEST000000000001",
|
||||
"time": "20260413100000",
|
||||
"plateNo": "豫A12345",
|
||||
"data": {
|
||||
"METER_SPEED": 52.3,
|
||||
"TOTAL_MILEAGE": 123456.7,
|
||||
"BATTERY_CAPACITY_SOC": 70,
|
||||
"LONGITUDE": 116397128,
|
||||
"LATITUDE": 39916527,
|
||||
"GPSDirection": 88
|
||||
}
|
||||
}`)
|
||||
|
||||
env, err := ParseMessage("endpoint-a", "/ytforward/shln/dev1", payload, 1782745114999)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolYutongMQTT {
|
||||
t.Fatalf("protocol = %q", env.Protocol)
|
||||
}
|
||||
if env.VIN != "LTEST000000000001" || env.DeviceID != "LTEST000000000001" {
|
||||
t.Fatalf("identity fields = vin:%q device:%q", env.VIN, env.DeviceID)
|
||||
}
|
||||
if env.Plate != "豫A12345" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123.4567)
|
||||
assertFloatField(t, env, envelope.FieldSOCPercent, 70)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
|
||||
assertFloatField(t, env, "direction_deg", 88)
|
||||
if env.EventTimeMS == 1782745114999 {
|
||||
t.Fatal("event time should come from device time")
|
||||
}
|
||||
if env.RawText == "" || env.Parsed["data"] == nil {
|
||||
t.Fatalf("raw/parsed missing: %#v", env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
|
||||
payload := []byte(`{"device":"YT-DEVICE-001","time":"20260413100000","data":{"METER_SPEED":"12.3"}}`)
|
||||
env, err := ParseMessage("endpoint-a", "/ytforward/shln/YT-DEVICE-001", payload, 1782745114999)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if env.VIN != "" {
|
||||
t.Fatalf("non-vin device should not be treated as vin: %q", env.VIN)
|
||||
}
|
||||
if env.VehicleKey() != "YT-DEVICE-001" {
|
||||
t.Fatalf("vehicle key = %q", env.VehicleKey())
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
|
||||
}
|
||||
|
||||
func TestParseMessageNormalizesProductionMileageMeters(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device":"LMRKH9AC3R1004101",
|
||||
"time":"2026-07-02 01:02:04.085",
|
||||
"data":{"TOTAL_MILEAGE":56905000,"METER_SPEED":19}
|
||||
}`)
|
||||
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 56905)
|
||||
if env.EventTimeMS != 1782925324085 {
|
||||
t.Fatalf("event time = %d, want 1782925324085", env.EventTimeMS)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageTreatsSmallYutongMileageAsMeters(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device":"LMRKH9AC3R1004101",
|
||||
"time":"2026-07-02 01:02:04.085",
|
||||
"data":{"TOTAL_MILEAGE":1024,"METER_SPEED":19}
|
||||
}`)
|
||||
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.024)
|
||||
}
|
||||
|
||||
func TestParseMessageRejectsMalformedJSON(t *testing.T) {
|
||||
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
|
||||
if !errors.Is(err, ErrInvalidJSON) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
repository *Repository
|
||||
}
|
||||
|
||||
func NewHandler(repository *Repository) *Handler {
|
||||
if repository == nil {
|
||||
panic("realtime repository must not be nil")
|
||||
}
|
||||
return &Handler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
path := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/realtime/vehicles/"), "/")
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" {
|
||||
writeError(w, http.StatusNotFound, "vehicle not found")
|
||||
return
|
||||
}
|
||||
vin := parts[0]
|
||||
switch {
|
||||
case len(parts) == 1:
|
||||
snapshot, err := h.repository.GetMerged(r.Context(), vin)
|
||||
writeResult(w, snapshot, err)
|
||||
case len(parts) == 2 && parts[1] == "online":
|
||||
status, err := h.repository.IsOnline(r.Context(), vin)
|
||||
writeResult(w, status, err)
|
||||
case len(parts) == 3 && parts[1] == "protocols":
|
||||
snapshot, err := h.repository.GetProtocol(r.Context(), vin, envelope.Protocol(strings.ToUpper(parts[2])))
|
||||
writeResult(w, snapshot, err)
|
||||
default:
|
||||
writeError(w, http.StatusNotFound, "route not found")
|
||||
}
|
||||
}
|
||||
|
||||
func writeResult(w http.ResponseWriter, value any, err error) {
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
41
go/vehicle-gateway/internal/realtime/model.go
Normal file
41
go/vehicle-gateway/internal/realtime/model.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Snapshot struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Protocol envelope.Protocol `json:"protocol,omitempty"`
|
||||
Protocols []envelope.Protocol `json:"protocols,omitempty"`
|
||||
EventID string `json:"event_id,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
|
||||
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||
}
|
||||
|
||||
type OnlineStatus struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Online bool `json:"online"`
|
||||
LastSeenMS int64 `json:"last_seen_ms"`
|
||||
Protocols []envelope.Protocol `json:"protocols,omitempty"`
|
||||
TTLSeconds int64 `json:"ttl_seconds"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
OnlineTTL time.Duration
|
||||
}
|
||||
|
||||
func (c Config) ttl() time.Duration {
|
||||
if c.OnlineTTL <= 0 {
|
||||
return 10 * time.Minute
|
||||
}
|
||||
return c.OnlineTTL
|
||||
}
|
||||
232
go/vehicle-gateway/internal/realtime/repository.go
Normal file
232
go/vehicle-gateway/internal/realtime/repository.go
Normal file
@@ -0,0 +1,232 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
client *redis.Client
|
||||
cfg Config
|
||||
}
|
||||
|
||||
func NewRepository(client *redis.Client, cfg Config) *Repository {
|
||||
if client == nil {
|
||||
panic("redis client must not be nil")
|
||||
}
|
||||
return &Repository{client: client, cfg: cfg}
|
||||
}
|
||||
|
||||
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
vehicleKey := strings.TrimSpace(env.VehicleKey())
|
||||
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
|
||||
return nil
|
||||
}
|
||||
nowMS := time.Now().UnixMilli()
|
||||
eventMS := env.EventTimeMS
|
||||
if eventMS <= 0 {
|
||||
eventMS = env.ReceivedAtMS
|
||||
}
|
||||
protocolSnapshot := Snapshot{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
EventID: env.StableEventID(),
|
||||
EventTimeMS: eventMS,
|
||||
ReceivedAtMS: env.ReceivedAtMS,
|
||||
SourceEndpoint: env.SourceEndpoint,
|
||||
Fields: cloneFields(env.Fields),
|
||||
FieldTimesMS: fieldTimes(env.Fields, eventMS),
|
||||
UpdatedAtMS: nowMS,
|
||||
}
|
||||
if err := r.setJSON(ctx, protocolKey(vehicleKey, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
protocols, err := r.addProtocol(ctx, vehicleKey, env.Protocol)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
merged, err := r.GetMerged(ctx, vehicleKey)
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
return err
|
||||
}
|
||||
if merged.VehicleKey == "" {
|
||||
merged = Snapshot{VehicleKey: vehicleKey, VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
|
||||
} else if merged.VIN == "" && vin != "" {
|
||||
merged.VIN = vin
|
||||
}
|
||||
mergeFields(&merged, env.Fields, eventMS)
|
||||
if eventMS >= merged.EventTimeMS {
|
||||
merged.EventTimeMS = eventMS
|
||||
merged.EventID = env.StableEventID()
|
||||
merged.ReceivedAtMS = env.ReceivedAtMS
|
||||
merged.SourceEndpoint = env.SourceEndpoint
|
||||
}
|
||||
merged.Protocols = protocols
|
||||
merged.UpdatedAtMS = nowMS
|
||||
if err := r.setJSON(ctx, mergedKey(vehicleKey), merged, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
online := OnlineStatus{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Online: true,
|
||||
LastSeenMS: env.ReceivedAtMS,
|
||||
Protocols: protocols,
|
||||
TTLSeconds: int64(r.cfg.ttl().Seconds()),
|
||||
}
|
||||
if err := r.setJSON(ctx, onlineKey(vehicleKey), online, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vehicleKey}).Err()
|
||||
}
|
||||
|
||||
func (r *Repository) GetMerged(ctx context.Context, vehicleKey string) (Snapshot, error) {
|
||||
return r.getSnapshot(ctx, mergedKey(vehicleKey))
|
||||
}
|
||||
|
||||
func (r *Repository) GetProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) (Snapshot, error) {
|
||||
return r.getSnapshot(ctx, protocolKey(vehicleKey, protocol))
|
||||
}
|
||||
|
||||
func (r *Repository) IsOnline(ctx context.Context, vehicleKey string) (OnlineStatus, error) {
|
||||
vehicleKey = strings.TrimSpace(vehicleKey)
|
||||
var status OnlineStatus
|
||||
payload, err := r.client.Get(ctx, onlineKey(vehicleKey)).Bytes()
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}, nil
|
||||
}
|
||||
return status, err
|
||||
}
|
||||
if err := json.Unmarshal(payload, &status); err != nil {
|
||||
return status, err
|
||||
}
|
||||
ttl := r.client.TTL(ctx, onlineKey(vehicleKey)).Val()
|
||||
status.Online = ttl > 0
|
||||
status.TTLSeconds = int64(ttl.Seconds())
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func (r *Repository) getSnapshot(ctx context.Context, key string) (Snapshot, error) {
|
||||
var snapshot Snapshot
|
||||
payload, err := r.client.Get(ctx, key).Bytes()
|
||||
if err != nil {
|
||||
return snapshot, err
|
||||
}
|
||||
return snapshot, json.Unmarshal(payload, &snapshot)
|
||||
}
|
||||
|
||||
func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl time.Duration) error {
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return r.client.Set(ctx, key, payload, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Repository) addProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
|
||||
key := protocolsKey(vehicleKey)
|
||||
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = r.client.Expire(ctx, key, r.cfg.ttl()).Err()
|
||||
values, err := r.client.SMembers(ctx, key).Result()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(values)
|
||||
out := make([]envelope.Protocol, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, envelope.Protocol(value))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func mergeFields(snapshot *Snapshot, fields map[string]any, eventMS int64) {
|
||||
if snapshot.Fields == nil {
|
||||
snapshot.Fields = map[string]any{}
|
||||
}
|
||||
if snapshot.FieldTimesMS == nil {
|
||||
snapshot.FieldTimesMS = map[string]int64{}
|
||||
}
|
||||
for key, value := range fields {
|
||||
if key == envelope.FieldTotalMileageKM && !positiveNumber(value) {
|
||||
continue
|
||||
}
|
||||
if eventMS >= snapshot.FieldTimesMS[key] {
|
||||
snapshot.Fields[key] = value
|
||||
snapshot.FieldTimesMS[key] = eventMS
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func positiveNumber(value any) bool {
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed > 0
|
||||
case float32:
|
||||
return typed > 0
|
||||
case int:
|
||||
return typed > 0
|
||||
case int64:
|
||||
return typed > 0
|
||||
case uint16:
|
||||
return typed > 0
|
||||
case uint32:
|
||||
return typed > 0
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return err == nil && parsed > 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func cloneFields(fields map[string]any) map[string]any {
|
||||
if len(fields) == 0 {
|
||||
return map[string]any{}
|
||||
}
|
||||
out := make(map[string]any, len(fields))
|
||||
for key, value := range fields {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
|
||||
out := make(map[string]int64, len(fields))
|
||||
for key := range fields {
|
||||
out[key] = eventMS
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergedKey(vehicleKey string) string {
|
||||
return "vehicle:latest:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
|
||||
func protocolKey(vehicleKey string, protocol envelope.Protocol) string {
|
||||
return "vehicle:latest:" + strings.TrimSpace(vehicleKey) + ":" + string(protocol)
|
||||
}
|
||||
|
||||
func onlineKey(vehicleKey string) string {
|
||||
return "vehicle:online:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
|
||||
func protocolsKey(vehicleKey string) string {
|
||||
return "vehicle:protocols:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
250
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
250
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
@@ -0,0 +1,250 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 121.1,
|
||||
envelope.FieldTotalMileageKM: 10.5,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 900,
|
||||
ReceivedAtMS: 1200,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 120.0,
|
||||
envelope.FieldLatitude: 30.5,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
merged, err := repo.GetMerged(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.Fields[envelope.FieldLongitude] != 121.1 {
|
||||
t.Fatalf("older longitude overwrote newer value: %#v", merged.Fields)
|
||||
}
|
||||
if merged.Fields[envelope.FieldLatitude] != 30.5 {
|
||||
t.Fatalf("new latitude missing: %#v", merged.Fields)
|
||||
}
|
||||
if got := protocolNames(merged.Protocols); strings.Join(got, ",") != "GB32960,JT808" {
|
||||
t.Fatalf("merged protocols = %#v", got)
|
||||
}
|
||||
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProtocol() error = %v", err)
|
||||
}
|
||||
if protocol.Fields[envelope.FieldTotalMileageKM] != 10.5 {
|
||||
t.Fatalf("protocol snapshot = %#v", protocol)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryOnlineStatus(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
status, err := repo.IsOnline(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if status.Online {
|
||||
t.Fatal("empty vin should be offline")
|
||||
}
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
status, err = repo.IsOnline(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if !status.Online || status.LastSeenMS != 1100 {
|
||||
t.Fatalf("online status = %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryUpdatesPhoneOnlyVehicleKey(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811170",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldSpeedKMH: 23.0,
|
||||
envelope.FieldTotalMileageKM: 10003.7,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
vehicleKey := "JT808:013307811170"
|
||||
merged, err := repo.GetMerged(ctx, vehicleKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.VehicleKey != vehicleKey || merged.VIN != "" {
|
||||
t.Fatalf("unexpected identity: %#v", merged)
|
||||
}
|
||||
if merged.Fields[envelope.FieldTotalMileageKM] != 10003.7 {
|
||||
t.Fatalf("merged fields = %#v", merged.Fields)
|
||||
}
|
||||
status, err := repo.IsOnline(ctx, vehicleKey)
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if !status.Online || status.VehicleKey != vehicleKey {
|
||||
t.Fatalf("online status = %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryDoesNotOverwritePositiveMileageWithZero(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 12345.6,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 2000,
|
||||
ReceivedAtMS: 2100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldSpeedKMH: 22.0,
|
||||
envelope.FieldTotalMileageKM: 0,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
merged, err := repo.GetMerged(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.Fields[envelope.FieldTotalMileageKM] != 12345.6 {
|
||||
t.Fatalf("zero mileage overwrote positive value: %#v", merged.Fields)
|
||||
}
|
||||
if merged.Fields[envelope.FieldSpeedKMH] != 22.0 {
|
||||
t.Fatalf("new speed should still merge: %#v", merged.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReturnsMergedSnapshot(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/VIN001", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
NewHandler(repo).ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !stringsContains(rec.Body.String(), `"vin":"VIN001"`) {
|
||||
t.Fatalf("unexpected body: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReturnsPhoneOnlyVehicleKeySnapshot(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811170",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/JT808:013307811170", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
NewHandler(repo).ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !stringsContains(rec.Body.String(), `"vehicle_key":"JT808:013307811170"`) {
|
||||
t.Fatalf("unexpected body: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRepository(t *testing.T) (*Repository, func()) {
|
||||
t.Helper()
|
||||
server, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("miniredis.Run() error = %v", err)
|
||||
}
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
return NewRepository(client, Config{OnlineTTL: time.Minute}), func() {
|
||||
_ = client.Close()
|
||||
server.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func stringsContains(value string, pattern string) bool {
|
||||
return strings.Contains(value, pattern)
|
||||
}
|
||||
|
||||
func protocolNames(protocols []envelope.Protocol) []string {
|
||||
names := make([]string, 0, len(protocols))
|
||||
for _, protocol := range protocols {
|
||||
names = append(names, string(protocol))
|
||||
}
|
||||
return names
|
||||
}
|
||||
275
go/vehicle-gateway/internal/stats/daily_metric.go
Normal file
275
go/vehicle-gateway/internal/stats/daily_metric.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const (
|
||||
MetricDailyMileageKM = "daily_mileage_km"
|
||||
MetricDailyTotalMileageKM = "daily_total_mileage_km"
|
||||
)
|
||||
|
||||
type Execer interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
exec Execer
|
||||
loc *time.Location
|
||||
}
|
||||
|
||||
type MetricSample struct {
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Protocol envelope.Protocol
|
||||
StatDate string
|
||||
MetricKey string
|
||||
MetricValue float64
|
||||
TotalMileageKM float64
|
||||
}
|
||||
|
||||
func NewWriter(exec Execer, loc *time.Location) *Writer {
|
||||
if exec == nil {
|
||||
panic("stats execer must not be nil")
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
return &Writer{exec: exec, loc: loc}
|
||||
}
|
||||
|
||||
func (w *Writer) EnsureSchema(ctx context.Context) error {
|
||||
if _, err := w.exec.ExecContext(ctx, DailyMetricTableSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
if db, ok := w.exec.(metadataDB); ok {
|
||||
return migrateDailyMetricTable(ctx, db)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
samples, err := SamplesFromEnvelope(env, w.loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, sample := range samples {
|
||||
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL,
|
||||
sample.VehicleKey,
|
||||
sample.VIN,
|
||||
sample.StatDate,
|
||||
string(sample.Protocol),
|
||||
sample.MetricKey,
|
||||
sample.MetricValue,
|
||||
sample.TotalMileageKM,
|
||||
sample.TotalMileageKM); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
vehicleKey := strings.TrimSpace(env.VehicleKey())
|
||||
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
|
||||
return nil, nil
|
||||
}
|
||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
if totalMileage <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
eventMS := env.EventTimeMS
|
||||
if eventMS <= 0 {
|
||||
eventMS = env.ReceivedAtMS
|
||||
}
|
||||
if eventMS <= 0 {
|
||||
return nil, errors.New("event or received time is required")
|
||||
}
|
||||
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
|
||||
return []MetricSample{
|
||||
{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
StatDate: statDate,
|
||||
MetricKey: MetricDailyMileageKM,
|
||||
MetricValue: 0,
|
||||
TotalMileageKM: totalMileage,
|
||||
},
|
||||
{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
StatDate: statDate,
|
||||
MetricKey: MetricDailyTotalMileageKM,
|
||||
MetricValue: totalMileage,
|
||||
TotalMileageKM: totalMileage,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
const upsertDailyMetricSQL = `
|
||||
INSERT INTO vehicle_daily_metric
|
||||
(vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
|
||||
first_total_mileage_km = CASE
|
||||
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
|
||||
THEN VALUES(first_total_mileage_km)
|
||||
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
END,
|
||||
latest_total_mileage_km = CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
END,
|
||||
metric_value = CASE
|
||||
WHEN metric_key = 'daily_mileage_km'
|
||||
THEN GREATEST(
|
||||
CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE latest_total_mileage_km
|
||||
END,
|
||||
VALUES(latest_total_mileage_km)
|
||||
)
|
||||
- CASE
|
||||
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
|
||||
THEN VALUES(first_total_mileage_km)
|
||||
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
END
|
||||
WHEN metric_key = 'daily_total_mileage_km'
|
||||
THEN GREATEST(
|
||||
CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE latest_total_mileage_km
|
||||
END,
|
||||
VALUES(latest_total_mileage_km)
|
||||
)
|
||||
ELSE VALUES(metric_value)
|
||||
END,
|
||||
sample_count = sample_count + 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`
|
||||
|
||||
type metadataDB interface {
|
||||
Execer
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}
|
||||
|
||||
func migrateDailyMetricTable(ctx context.Context, db metadataDB) error {
|
||||
columnExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND column_name = 'vehicle_key'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !columnExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD COLUMN vehicle_key VARCHAR(96) NOT NULL DEFAULT '' AFTER id`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `UPDATE vehicle_daily_metric SET vehicle_key = vin WHERE vehicle_key = ''`); err != nil {
|
||||
return err
|
||||
}
|
||||
vehicleIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'uk_daily_metric_vehicle'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !vehicleIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key)`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
oldIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'uk_daily_metric'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if oldIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric DROP INDEX uk_daily_metric`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
vinIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'idx_vin'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !vinIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD KEY idx_vin (vin)`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func informationSchemaExists(ctx context.Context, db metadataDB, query string) (bool, error) {
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, query).Scan(&count); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||
if env.Fields == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
162
go/vehicle-gateway/internal/stats/daily_metric_test.go
Normal file
162
go/vehicle-gateway/internal/stats/daily_metric_test.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "LNBVIN00000000001",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 8, 30, 0, 0, loc).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 10241.2,
|
||||
},
|
||||
}
|
||||
|
||||
samples, err := SamplesFromEnvelope(env, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 2 {
|
||||
t.Fatalf("sample count = %d", len(samples))
|
||||
}
|
||||
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
|
||||
t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
|
||||
}
|
||||
if samples[0].VehicleKey != "LNBVIN00000000001" {
|
||||
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
|
||||
}
|
||||
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 {
|
||||
t.Fatalf("unexpected daily total sample: %#v", samples[1])
|
||||
}
|
||||
if samples[0].StatDate != "2026-07-01" {
|
||||
t.Fatalf("stat date = %q", samples[0].StatDate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811254",
|
||||
EventTimeMS: time.Date(2026, 7, 2, 0, 3, 0, 0, loc).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 10985.7,
|
||||
},
|
||||
}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 2 {
|
||||
t.Fatalf("sample count = %d", len(samples))
|
||||
}
|
||||
if samples[0].VehicleKey != "JT808:013307811254" {
|
||||
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
|
||||
}
|
||||
if samples[0].VIN != "" {
|
||||
t.Fatalf("vin should stay empty for unresolved JT808 identity, got %q", samples[0].VIN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples without vehicle key, got %#v", samples)
|
||||
}
|
||||
|
||||
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "LNBVIN00000000002",
|
||||
Fields: map[string]any{},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples without mileage, got %#v", samples)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeSkipsNonPositiveMileage(t *testing.T) {
|
||||
for _, value := range []any{0, 0.0, -1.0, "0"} {
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "LNBVIN00000000001",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: value,
|
||||
},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope(%#v) error = %v", value, err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples for non-positive mileage %#v, got %#v", value, samples)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
if err := writer.EnsureSchema(context.Background()); err != nil {
|
||||
t.Fatalf("EnsureSchema() error = %v", err)
|
||||
}
|
||||
if err := writer.Append(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "LNBVIN00000000002",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: "10000.0",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") {
|
||||
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
|
||||
}
|
||||
if !strings.Contains(exec.calls[0].query, "vehicle_key") {
|
||||
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
|
||||
}
|
||||
if len(exec.calls) != 3 {
|
||||
t.Fatalf("exec calls = %d", len(exec.calls))
|
||||
}
|
||||
if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") {
|
||||
t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query)
|
||||
}
|
||||
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
|
||||
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
|
||||
}
|
||||
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
|
||||
t.Fatalf("first upsert arg should be vehicle_key, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type execCall struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type recordingExec struct {
|
||||
calls []execCall
|
||||
}
|
||||
|
||||
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||
return nil, nil
|
||||
}
|
||||
326
go/vehicle-gateway/internal/stats/query.go
Normal file
326
go/vehicle-gateway/internal/stats/query.go
Normal file
@@ -0,0 +1,326 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Queryer interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
type MetricQuery struct {
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Protocol string
|
||||
MetricKey string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type MetricRow struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
StatDate string `json:"stat_date"`
|
||||
Protocol string `json:"protocol"`
|
||||
MetricKey string `json:"metric_key"`
|
||||
MetricValue float64 `json:"metric_value"`
|
||||
MetricUnit string `json:"metric_unit"`
|
||||
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
|
||||
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
|
||||
SampleCount int64 `json:"sample_count"`
|
||||
CalculationMethod string `json:"calculation_method"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type MetricRepository struct {
|
||||
db Queryer
|
||||
}
|
||||
|
||||
func NewMetricRepository(db Queryer) *MetricRepository {
|
||||
if db == nil {
|
||||
panic("metric query db must not be nil")
|
||||
}
|
||||
return &MetricRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]MetricRow, error) {
|
||||
query = normalizeMetricQuery(query)
|
||||
sqlText, args := buildMetricSQL(query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]MetricRow, 0)
|
||||
for rows.Next() {
|
||||
var row MetricRow
|
||||
var statDate scanDate
|
||||
var createdAt scanDateTime
|
||||
var updatedAt scanDateTime
|
||||
var first sql.NullFloat64
|
||||
var latest sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&statDate,
|
||||
&row.Protocol,
|
||||
&row.MetricKey,
|
||||
&row.MetricValue,
|
||||
&row.MetricUnit,
|
||||
&first,
|
||||
&latest,
|
||||
&row.SampleCount,
|
||||
&row.CalculationMethod,
|
||||
&createdAt,
|
||||
&updatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.StatDate = statDate.String
|
||||
row.CreatedAt = createdAt.String
|
||||
row.UpdatedAt = updatedAt.String
|
||||
if first.Valid {
|
||||
row.FirstTotalMileageKM = &first.Float64
|
||||
}
|
||||
if latest.Valid {
|
||||
row.LatestTotalMileageKM = &latest.Float64
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, error) {
|
||||
query = normalizeMetricQuery(query)
|
||||
sqlText, args := buildMetricCountSQL(query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var total int64
|
||||
if rows.Next() {
|
||||
if err := rows.Scan(&total); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return total, rows.Err()
|
||||
}
|
||||
|
||||
func normalizeMetricQuery(query MetricQuery) MetricQuery {
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.MetricKey = strings.TrimSpace(query.MetricKey)
|
||||
query.DateFrom = strings.TrimSpace(query.DateFrom)
|
||||
query.DateTo = strings.TrimSpace(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 50
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func buildMetricSQL(query MetricQuery) (string, []any) {
|
||||
where, args := buildMetricWhere(query)
|
||||
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric`
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
|
||||
args = append(args, query.Limit, query.Offset)
|
||||
return sqlText, args
|
||||
}
|
||||
|
||||
func buildMetricCountSQL(query MetricQuery) (string, []any) {
|
||||
where, args := buildMetricWhere(query)
|
||||
sqlText := `SELECT COUNT(*) FROM vehicle_daily_metric`
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, args
|
||||
}
|
||||
|
||||
func buildMetricWhere(query MetricQuery) ([]string, []any) {
|
||||
var where []string
|
||||
var args []any
|
||||
add := func(clause string, value any) {
|
||||
where = append(where, clause)
|
||||
args = append(args, value)
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = ?", query.VIN)
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = ?", query.VehicleKey)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = ?", query.Protocol)
|
||||
}
|
||||
if query.MetricKey != "" {
|
||||
add("metric_key = ?", query.MetricKey)
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("stat_date >= ?", query.DateFrom)
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("stat_date <= ?", query.DateTo)
|
||||
}
|
||||
return where, args
|
||||
}
|
||||
|
||||
type MetricHandler struct {
|
||||
repository *MetricRepository
|
||||
}
|
||||
|
||||
func NewMetricHandler(repository *MetricRepository) *MetricHandler {
|
||||
if repository == nil {
|
||||
panic("metric repository must not be nil")
|
||||
}
|
||||
return &MetricHandler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeMetricError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/stats/daily-metrics" {
|
||||
writeMetricError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseMetricQuery(r)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func parseMetricQuery(r *http.Request) (MetricQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit")
|
||||
if err != nil {
|
||||
return MetricQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return MetricQuery{}, err
|
||||
}
|
||||
query := MetricQuery{
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Protocol: values.Get("protocol"),
|
||||
MetricKey: values.Get("metricKey"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDate(query.DateFrom) || !validDate(query.DateTo) {
|
||||
return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD")
|
||||
}
|
||||
return normalizeMetricQuery(query), nil
|
||||
}
|
||||
|
||||
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil || value < min || value > max {
|
||||
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func validDate(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
if len(value) != len("2006-01-02") {
|
||||
return false
|
||||
}
|
||||
for i, r := range value {
|
||||
switch i {
|
||||
case 4, 7:
|
||||
if r != '-' {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func writeMetricError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
|
||||
type scanDate struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDate) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02")
|
||||
return nil
|
||||
}
|
||||
|
||||
type scanDateTime struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDateTime) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatSQLTime(value any, layout string) string {
|
||||
switch typed := value.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case time.Time:
|
||||
return typed.Format(layout)
|
||||
case []byte:
|
||||
return strings.TrimSpace(string(typed))
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
185
go/vehicle-gateway/internal/stats/query_test.go
Normal file
185
go/vehicle-gateway/internal/stats/query_test.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km",
|
||||
12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
))
|
||||
|
||||
repository := NewMetricRepository(db)
|
||||
rows, err := repository.Query(context.Background(), MetricQuery{
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
Protocol: "JT808",
|
||||
DateFrom: "2026-07-01",
|
||||
DateTo: "2026-07-01",
|
||||
Limit: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query() error = %v", err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("row count = %d", len(rows))
|
||||
}
|
||||
if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 {
|
||||
t.Fatalf("unexpected row: %#v", rows[0])
|
||||
}
|
||||
if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
|
||||
t.Fatalf("vehicle key = %q", rows[0].VehicleKey)
|
||||
}
|
||||
if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" {
|
||||
t.Fatalf("unexpected time formatting: %#v", rows[0])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km",
|
||||
53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25",
|
||||
))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":42`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("JT808:013307811254", "JT808").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("JT808:013307811254", "JT808", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"JT808:013307811254", "", "2026-07-02", "JT808", MetricDailyTotalMileageKM, 10985.7, "km",
|
||||
10985.7, 10985.7, 1, "TOTAL_MILEAGE_DIFF", "2026-07-02 00:03:00", "2026-07-02 00:03:00",
|
||||
))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vehicleKey=JT808:013307811254&protocol=JT808", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"metric_value":10985.7`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("YUTONG_MQTT").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("YUTONG_MQTT", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?protocol=YUTONG_MQTT", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
|
||||
t.Fatalf("expected empty items array, got: %s", body)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerRejectsInvalidPagination(t *testing.T) {
|
||||
handler := NewMetricHandler(NewMetricRepository(&sql.DB{}))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?limit=2001", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
22
go/vehicle-gateway/internal/stats/schema.go
Normal file
22
go/vehicle-gateway/internal/stats/schema.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package stats
|
||||
|
||||
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vehicle_key VARCHAR(96) NOT NULL,
|
||||
vin VARCHAR(32) NOT NULL DEFAULT '',
|
||||
stat_date DATE NOT NULL,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,3) NOT NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key),
|
||||
KEY idx_vin (vin),
|
||||
KEY idx_stat_date (stat_date),
|
||||
KEY idx_protocol_metric (protocol, metric_key)
|
||||
)`
|
||||
@@ -35,7 +35,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -49,10 +49,6 @@
|
||||
<groupId>com.alibaba.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springdoc</groupId>
|
||||
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
@@ -66,17 +62,6 @@
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
<goal>build-info</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
|
||||
@SpringBootApplication
|
||||
public class Gb32960IngestApplication {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(Gb32960IngestApplication.class);
|
||||
|
||||
@@ -28,12 +28,6 @@ spring:
|
||||
server:
|
||||
port: ${HTTP_PORT:20100}
|
||||
|
||||
springdoc:
|
||||
swagger-ui:
|
||||
path: /swagger-ui.html
|
||||
api-docs:
|
||||
path: /v3/api-docs
|
||||
|
||||
lingniu:
|
||||
ingest:
|
||||
gb32960:
|
||||
@@ -49,10 +43,15 @@ lingniu:
|
||||
whitelist: []
|
||||
platforms:
|
||||
- username: ${GB32960_PLATFORM_USER_HYUNDAI:Hyundai}
|
||||
password: ${GB32960_PLATFORM_PWD_HYUNDAI:f2e3445d7cda409fb4f278f6fb890734}
|
||||
password: ${GB32960_PLATFORM_PWD_HYUNDAI:}
|
||||
allowed-ips:
|
||||
- ${GB32960_PLATFORM_IP_HYUNDAI:}
|
||||
description: 外部下级平台 - Hyundai
|
||||
- username: ${GB32960_PLATFORM_USER_YUEJIN:YueJin}
|
||||
password: ${GB32960_PLATFORM_PWD_YUEJIN:}
|
||||
allowed-ips:
|
||||
- ${GB32960_PLATFORM_IP_YUEJIN:}
|
||||
description: 外部下级平台 - YueJin
|
||||
tls:
|
||||
enabled: ${GB32960_TLS_ENABLED:false}
|
||||
cert-path: ${GB32960_TLS_CERT:}
|
||||
@@ -64,6 +63,7 @@ lingniu:
|
||||
match:
|
||||
platform-accounts:
|
||||
- Hyundai
|
||||
- YueJin
|
||||
vin-prefixes: []
|
||||
vins: []
|
||||
parse:
|
||||
@@ -82,10 +82,8 @@ lingniu:
|
||||
rate-limit:
|
||||
per-vin-qps: 50
|
||||
session:
|
||||
store: ${SESSION_STORE:redis}
|
||||
ttl: ${SESSION_TTL:30m}
|
||||
identity:
|
||||
store: ${VEHICLE_IDENTITY_STORE:mysql}
|
||||
mysql:
|
||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||
@@ -93,9 +91,8 @@ lingniu:
|
||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
kafka:
|
||||
enabled: true
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
compression-type: zstd
|
||||
linger-ms: 20
|
||||
@@ -117,18 +114,16 @@ lingniu:
|
||||
enabled: ${SINK_ARCHIVE_ENABLED:true}
|
||||
type: local
|
||||
path: ${SINK_ARCHIVE_PATH:./archive/}
|
||||
event-history:
|
||||
enabled: false
|
||||
vehicle-state:
|
||||
enabled: false
|
||||
vehicle-stat:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus,env,gb32960sessions
|
||||
include: health,info,metrics,prometheus,gb32960sessions
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
|
||||
health:
|
||||
redis:
|
||||
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}
|
||||
|
||||
@@ -6,13 +6,14 @@ import com.lingniu.ingest.identity.config.VehicleIdentityAutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
|
||||
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
||||
import com.lingniu.ingest.session.SessionStore;
|
||||
import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
||||
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
@@ -30,27 +31,23 @@ class Gb32960IngestAppCompositionTest {
|
||||
IngestCoreAutoConfiguration.class,
|
||||
SessionCoreAutoConfiguration.class,
|
||||
VehicleIdentityAutoConfiguration.class,
|
||||
SinkMqAutoConfiguration.class,
|
||||
KafkaSinkAutoConfiguration.class,
|
||||
SinkArchiveAutoConfiguration.class,
|
||||
Gb32960AutoConfiguration.class))
|
||||
.withAllowBeanDefinitionOverriding(true)
|
||||
.withBean("kafkaProducer", KafkaProducer.class, Gb32960IngestAppCompositionTest::kafkaProducer)
|
||||
.withBean(SessionStore.class, () -> mock(SessionStore.class))
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.gb32960.enabled=true",
|
||||
"lingniu.ingest.gb32960.server.enabled=true",
|
||||
"lingniu.ingest.gb32960.port=0",
|
||||
"lingniu.ingest.identity.store=mysql",
|
||||
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
||||
"lingniu.ingest.session.store=memory",
|
||||
"lingniu.ingest.sink.mq.enabled=true",
|
||||
"lingniu.ingest.sink.mq.type=kafka",
|
||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||
"lingniu.ingest.sink.archive.enabled=true",
|
||||
"lingniu.ingest.sink.archive.path=target/test-archive-gb32960",
|
||||
"lingniu.ingest.event-history.enabled=false",
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=false");
|
||||
"lingniu.ingest.event-history.enabled=false");
|
||||
|
||||
@Test
|
||||
void createsGb32960ListenerWithoutHistoryStorageBoundaries() {
|
||||
@@ -63,7 +60,8 @@ class Gb32960IngestAppCompositionTest {
|
||||
assertThat(context).hasSingleBean(ArchiveStore.class);
|
||||
assertThat(context).hasSingleBean(RawArchiveEventSink.class);
|
||||
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@ class Gb32960IngestAppDefaultsTest {
|
||||
@Test
|
||||
void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() throws IOException {
|
||||
Properties properties = applicationProperties();
|
||||
String legacyKafkaPrefix = "lingniu.ingest.sink." + "mq.";
|
||||
String kafkaTypeProperty = "lingniu.ingest.sink.kafka." + "type";
|
||||
|
||||
assertThat(properties)
|
||||
.containsEntry("spring.application.name", "gb32960-ingest-app")
|
||||
@@ -28,23 +30,48 @@ class Gb32960IngestAppDefaultsTest {
|
||||
.containsEntry("lingniu.ingest.gb32960.enabled", true)
|
||||
.containsEntry("lingniu.ingest.gb32960.server.enabled", true)
|
||||
.containsEntry("lingniu.ingest.gb32960.port", "${GB32960_PORT:32960}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
|
||||
.containsEntry("lingniu.ingest.gb32960.auth.platforms[0].username",
|
||||
"${GB32960_PLATFORM_USER_HYUNDAI:Hyundai}")
|
||||
.containsEntry("lingniu.ingest.gb32960.auth.platforms[0].password",
|
||||
"${GB32960_PLATFORM_PWD_HYUNDAI:}")
|
||||
.containsEntry("lingniu.ingest.gb32960.auth.platforms[1].username",
|
||||
"${GB32960_PLATFORM_USER_YUEJIN:YueJin}")
|
||||
.containsEntry("lingniu.ingest.gb32960.auth.platforms[1].password",
|
||||
"${GB32960_PLATFORM_PWD_YUEJIN:}")
|
||||
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[0]",
|
||||
"Hyundai")
|
||||
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[1]",
|
||||
"YueJin")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
|
||||
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
|
||||
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
|
||||
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
||||
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
|
||||
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
|
||||
.containsEntry("management.endpoints.web.exposure.include",
|
||||
"health,info,metrics,prometheus,env,gb32960sessions");
|
||||
"health,info,metrics,prometheus,gb32960sessions");
|
||||
|
||||
assertThat(properties.stringPropertyNames())
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."));
|
||||
assertThat(applicationYaml()).doesNotContain("event-file-store:");
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
|
||||
.noneMatch(name -> name.startsWith(legacyKafkaPrefix))
|
||||
.noneMatch(name -> name.equals(kafkaTypeProperty))
|
||||
.noneMatch(name -> name.equals("lingniu.ingest.session.store"))
|
||||
.noneMatch(name -> name.equals("lingniu.ingest.identity.store"));
|
||||
assertThat(applicationYaml())
|
||||
.doesNotContain("event-history:")
|
||||
.doesNotContain("event-file-store:")
|
||||
.doesNotContain("vehicle-state:")
|
||||
.doesNotContain("vehicle-stat:")
|
||||
.doesNotContain("springdoc:");
|
||||
assertThat(applicationYaml())
|
||||
.contains("password: ${GB32960_PLATFORM_PWD_HYUNDAI:}")
|
||||
.contains("password: ${GB32960_PLATFORM_PWD_YUEJIN:}")
|
||||
.doesNotContain("SESSION_STORE")
|
||||
.doesNotContain("VEHICLE_IDENTITY_STORE")
|
||||
.doesNotContain("KAFKA_ENABLED");
|
||||
}
|
||||
|
||||
private static Properties applicationProperties() {
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -49,10 +49,6 @@
|
||||
<groupId>com.alibaba.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springdoc</groupId>
|
||||
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
@@ -66,17 +62,6 @@
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
<goal>build-info</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
|
||||
@SpringBootApplication
|
||||
public class Jt808IngestApplication {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(Jt808IngestApplication.class);
|
||||
|
||||
@@ -28,12 +28,6 @@ spring:
|
||||
server:
|
||||
port: ${HTTP_PORT:20400}
|
||||
|
||||
springdoc:
|
||||
swagger-ui:
|
||||
path: /swagger-ui.html
|
||||
api-docs:
|
||||
path: /v3/api-docs
|
||||
|
||||
lingniu:
|
||||
ingest:
|
||||
jt808:
|
||||
@@ -52,10 +46,8 @@ lingniu:
|
||||
rate-limit:
|
||||
per-vin-qps: 50
|
||||
session:
|
||||
store: ${SESSION_STORE:redis}
|
||||
ttl: ${SESSION_TTL:30m}
|
||||
identity:
|
||||
store: ${VEHICLE_IDENTITY_STORE:mysql}
|
||||
mysql:
|
||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||
@@ -63,9 +55,8 @@ lingniu:
|
||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
kafka:
|
||||
enabled: true
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
compression-type: zstd
|
||||
linger-ms: 20
|
||||
@@ -83,18 +74,16 @@ lingniu:
|
||||
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
|
||||
consumer:
|
||||
enabled: false
|
||||
event-history:
|
||||
enabled: false
|
||||
vehicle-state:
|
||||
enabled: false
|
||||
vehicle-stat:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus,env
|
||||
include: health,info,metrics,prometheus
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
|
||||
health:
|
||||
redis:
|
||||
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}
|
||||
|
||||
@@ -6,13 +6,14 @@ import com.lingniu.ingest.identity.config.VehicleIdentityAutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.jt808.codec.Jt808MessageDecoder;
|
||||
import com.lingniu.ingest.protocol.jt808.config.Jt808AutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.jt808.inbound.Jt808NettyServer;
|
||||
import com.lingniu.ingest.session.SessionStore;
|
||||
import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
||||
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
@@ -30,26 +31,22 @@ class Jt808IngestAppCompositionTest {
|
||||
IngestCoreAutoConfiguration.class,
|
||||
SessionCoreAutoConfiguration.class,
|
||||
VehicleIdentityAutoConfiguration.class,
|
||||
SinkMqAutoConfiguration.class,
|
||||
KafkaSinkAutoConfiguration.class,
|
||||
SinkArchiveAutoConfiguration.class,
|
||||
Jt808AutoConfiguration.class))
|
||||
.withAllowBeanDefinitionOverriding(true)
|
||||
.withBean("kafkaProducer", KafkaProducer.class, Jt808IngestAppCompositionTest::kafkaProducer)
|
||||
.withBean(SessionStore.class, () -> mock(SessionStore.class))
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.jt808.enabled=true",
|
||||
"lingniu.ingest.jt808.port=0",
|
||||
"lingniu.ingest.identity.store=mysql",
|
||||
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
||||
"lingniu.ingest.session.store=memory",
|
||||
"lingniu.ingest.sink.mq.enabled=true",
|
||||
"lingniu.ingest.sink.mq.type=kafka",
|
||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||
"lingniu.ingest.sink.archive.enabled=true",
|
||||
"lingniu.ingest.sink.archive.path=target/test-archive-jt808",
|
||||
"lingniu.ingest.event-history.enabled=false",
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=false");
|
||||
"lingniu.ingest.event-history.enabled=false");
|
||||
|
||||
@Test
|
||||
void createsJt808ListenerWithoutHistoryStorageBoundaries() {
|
||||
@@ -62,7 +59,8 @@ class Jt808IngestAppCompositionTest {
|
||||
assertThat(context).hasSingleBean(ArchiveStore.class);
|
||||
assertThat(context).hasSingleBean(RawArchiveEventSink.class);
|
||||
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -28,13 +28,11 @@ class Jt808IngestAppDefaultsTest {
|
||||
.containsEntry("server.port", "${HTTP_PORT:20400}")
|
||||
.containsEntry("lingniu.ingest.jt808.enabled", true)
|
||||
.containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:808}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
|
||||
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
|
||||
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
||||
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url",
|
||||
@@ -43,13 +41,24 @@ class Jt808IngestAppDefaultsTest {
|
||||
.containsEntry("lingniu.ingest.identity.mysql.password", "${VEHICLE_IDENTITY_MYSQL_PASSWORD:}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.initialize-schema",
|
||||
"${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}")
|
||||
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false);
|
||||
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
|
||||
|
||||
assertThat(properties.stringPropertyNames())
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."));
|
||||
assertThat(applicationYaml()).doesNotContain("event-file-store:");
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
|
||||
.noneMatch(name -> name.equals("lingniu.ingest.session.store"))
|
||||
.noneMatch(name -> name.equals("lingniu.ingest.identity.store"));
|
||||
assertThat(applicationYaml())
|
||||
.doesNotContain("event-history:")
|
||||
.doesNotContain("event-file-store:")
|
||||
.doesNotContain("vehicle-state:")
|
||||
.doesNotContain("vehicle-stat:")
|
||||
.doesNotContain("springdoc:")
|
||||
.doesNotContain("SESSION_STORE")
|
||||
.doesNotContain("VEHICLE_IDENTITY_STORE")
|
||||
.doesNotContain("KAFKA_ENABLED");
|
||||
}
|
||||
|
||||
private static Properties applicationProperties() {
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -58,17 +58,6 @@
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
<goal>build-info</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
|
||||
@SpringBootApplication
|
||||
public class VehicleAnalyticsApplication {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(VehicleAnalyticsApplication.class);
|
||||
|
||||
@@ -24,7 +24,7 @@ spring:
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
|
||||
server:
|
||||
port: ${HTTP_PORT:20300}
|
||||
port: ${HTTP_PORT:20310}
|
||||
|
||||
springdoc:
|
||||
swagger-ui:
|
||||
@@ -34,17 +34,12 @@ springdoc:
|
||||
|
||||
lingniu:
|
||||
ingest:
|
||||
gb32960:
|
||||
enabled: false
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
kafka:
|
||||
enabled: true
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
topics:
|
||||
realtime: ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
|
||||
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
|
||||
consumer:
|
||||
dlq-topic: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
|
||||
enabled: ${KAFKA_CONSUMER_ENABLED:true}
|
||||
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics}
|
||||
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
|
||||
@@ -52,14 +47,10 @@ lingniu:
|
||||
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:900000}
|
||||
bindings:
|
||||
vehicleStatEnvelopeConsumerProcessor:
|
||||
enabled: ${VEHICLE_STAT_ENABLED:true}
|
||||
enabled: true
|
||||
group-id: ${KAFKA_GROUP_STAT:vehicle-stat}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
|
||||
archive:
|
||||
enabled: false
|
||||
event-history:
|
||||
enabled: false
|
||||
vehicle-stat:
|
||||
enabled: ${VEHICLE_STAT_ENABLED:true}
|
||||
zone-id: ${VEHICLE_STAT_ZONE_ID:Asia/Shanghai}
|
||||
@@ -70,7 +61,11 @@ management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus,env
|
||||
include: health,info,metrics,prometheus
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
|
||||
metrics:
|
||||
tags:
|
||||
application: vehicle-analytics-app
|
||||
|
||||
@@ -2,9 +2,9 @@ package com.lingniu.ingest.analyticsapp;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||
import com.lingniu.ingest.vehiclestat.JdbcVehicleStatMetricRepository;
|
||||
import com.lingniu.ingest.vehiclestat.VehicleStatController;
|
||||
import com.lingniu.ingest.vehiclestat.VehicleStatEnvelopeIngestor;
|
||||
@@ -25,24 +25,21 @@ import static org.mockito.Mockito.mock;
|
||||
class VehicleAnalyticsAppCompositionTest {
|
||||
|
||||
@Test
|
||||
void createsStatsBeansWithoutProtocolListenerOrEventFileStore() {
|
||||
void createsStatsBeansWithoutProtocolListenerOrFileHistoryStore() {
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
SinkMqAutoConfiguration.class,
|
||||
KafkaSinkAutoConfiguration.class,
|
||||
VehicleStatAutoConfiguration.class))
|
||||
.withAllowBeanDefinitionOverriding(true)
|
||||
.withBean("kafkaProducer", KafkaProducer.class, VehicleAnalyticsAppCompositionTest::kafkaProducer)
|
||||
.withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class))
|
||||
.withBean(ObjectMapper.class, ObjectMapper::new)
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.sink.mq.enabled=true",
|
||||
"lingniu.ingest.sink.mq.type=kafka",
|
||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=true",
|
||||
"lingniu.ingest.vehicle-stat.jt808.enabled=true",
|
||||
"lingniu.ingest.event-history.enabled=false",
|
||||
"lingniu.ingest.gb32960.enabled=false")
|
||||
"lingniu.ingest.vehicle-stat.jt808.enabled=true")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(VehicleStatRepository.class);
|
||||
assertThat(context).hasSingleBean(JdbcVehicleStatMetricRepository.class);
|
||||
@@ -58,7 +55,9 @@ class VehicleAnalyticsAppCompositionTest {
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.vehiclestate.VehicleStateEnvelopeIngestor");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
|
||||
assertTypeNotPresent(context,
|
||||
"com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -22,32 +22,42 @@ class VehicleAnalyticsAppDefaultsTest {
|
||||
"spring.config.import[0]",
|
||||
"optional:nacos:${spring.application.name}.${NACOS_CONFIG_FILE_EXTENSION:yml}?group=${NACOS_GROUP:DEFAULT_GROUP}&refreshEnabled=${NACOS_REFRESH_ENABLED:true}")
|
||||
.containsEntry("spring.cloud.nacos.config.server-addr", "${NACOS_SERVER_ADDR:127.0.0.1:8848}")
|
||||
.containsEntry("lingniu.ingest.gb32960.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.archive.enabled", false)
|
||||
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
||||
.containsEntry("server.port", "${HTTP_PORT:20310}")
|
||||
.containsEntry("lingniu.ingest.vehicle-stat.enabled", "${VEHICLE_STAT_ENABLED:true}")
|
||||
.containsEntry(
|
||||
"lingniu.ingest.vehicle-stat.jt808.enabled",
|
||||
"${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
|
||||
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.consumer.dlq-topic",
|
||||
"${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
||||
.containsEntry(
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
|
||||
"${VEHICLE_STAT_ENABLED:true}")
|
||||
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
|
||||
true)
|
||||
.containsEntry(
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
|
||||
"${KAFKA_GROUP_STAT:vehicle-stat}");
|
||||
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
|
||||
"${KAFKA_GROUP_STAT:vehicle-stat}")
|
||||
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
|
||||
|
||||
assertThat(properties.stringPropertyNames())
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.gb32960."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.sink.archive."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.sink.kafka.topics."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
|
||||
.noneMatch(name -> name.contains("vehicleStateEnvelopeConsumerProcessor"))
|
||||
.noneMatch(name -> name.startsWith("spring.data.redis."))
|
||||
.noneMatch(name -> name.startsWith("management.health.redis."));
|
||||
assertThat(applicationYaml())
|
||||
.doesNotContain("gb32960:")
|
||||
.doesNotContain("event-history:")
|
||||
.doesNotContain("archive:")
|
||||
.doesNotContain("event-file-store:")
|
||||
.doesNotContain("vehicle-state:")
|
||||
.doesNotContain("vehicleStateEnvelopeConsumerProcessor")
|
||||
.doesNotContain("redis:");
|
||||
.doesNotContain("redis:")
|
||||
.doesNotContain("KAFKA_ENABLED");
|
||||
}
|
||||
|
||||
private static Properties applicationProperties() {
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -62,17 +62,6 @@
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
<goal>build-info</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
|
||||
@SpringBootApplication
|
||||
public class VehicleHistoryApplication {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(VehicleHistoryApplication.class);
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.lingniu.ingest.historyapp;
|
||||
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerFactory;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerWorker;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqProperties;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerFactory;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerWorker;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
@@ -20,11 +20,11 @@ public class VehicleHistoryKafkaConsumerConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka.consumer", name = "enabled", havingValue = "true")
|
||||
public KafkaEnvelopeConsumerRunner vehicleHistoryKafkaEnvelopeConsumerRunner(
|
||||
@Qualifier("eventHistoryEnvelopeConsumerProcessor") EnvelopeConsumerProcessor processor,
|
||||
@Qualifier("eventHistoryRawEnvelopeConsumerProcessor") EnvelopeConsumerProcessor rawProcessor,
|
||||
SinkMqProperties props) {
|
||||
KafkaSinkProperties props) {
|
||||
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(
|
||||
Map.of(
|
||||
"eventHistoryGb32960EnvelopeConsumerProcessor", processor,
|
||||
|
||||
@@ -29,25 +29,17 @@ springdoc:
|
||||
|
||||
lingniu:
|
||||
ingest:
|
||||
gb32960:
|
||||
enabled: true
|
||||
server:
|
||||
enabled: false
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
kafka:
|
||||
enabled: true
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
topics:
|
||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
raw-archive: ${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}
|
||||
dlq: ${KAFKA_TOPIC_GB32960_DLQ:vehicle.dlq.gb32960.v1}
|
||||
consumer:
|
||||
dlq-topic: ${KAFKA_TOPIC_HISTORY_DLQ:vehicle.dlq.history.v1}
|
||||
enabled: ${KAFKA_CONSUMER_ENABLED:true}
|
||||
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history}
|
||||
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
|
||||
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}
|
||||
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}
|
||||
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:1800000}
|
||||
concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3}
|
||||
bindings:
|
||||
eventHistoryGb32960EnvelopeConsumerProcessor:
|
||||
@@ -80,8 +72,6 @@ lingniu:
|
||||
group-id: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-raw}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}
|
||||
archive:
|
||||
enabled: false
|
||||
tdengine-history:
|
||||
enabled: ${TDENGINE_HISTORY_ENABLED:false}
|
||||
database: ${TDENGINE_HISTORY_DATABASE:vehicle_history}
|
||||
@@ -93,21 +83,18 @@ lingniu:
|
||||
minimum-idle: ${TDENGINE_MIN_IDLE:0}
|
||||
connection-timeout-millis: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:5000}
|
||||
initialization-fail-timeout-millis: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:-1}
|
||||
telemetry-fields-enabled: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:false}
|
||||
event-history:
|
||||
enabled: true
|
||||
api:
|
||||
specialized-enabled: ${EVENT_HISTORY_SPECIALIZED_API_ENABLED:false}
|
||||
vehicle-state:
|
||||
enabled: false
|
||||
vehicle-stat:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus,env
|
||||
include: health,info,metrics,prometheus
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
|
||||
health:
|
||||
redis:
|
||||
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}
|
||||
|
||||
@@ -10,11 +10,19 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class MavenModuleProfileTest {
|
||||
|
||||
private static final List<String> ACTIVE_APP_POMS = List.of(
|
||||
"modules/apps/gb32960-ingest-app/pom.xml",
|
||||
"modules/apps/jt808-ingest-app/pom.xml",
|
||||
"modules/apps/yutong-mqtt-app/pom.xml",
|
||||
"modules/apps/vehicle-history-app/pom.xml",
|
||||
"modules/apps/vehicle-analytics-app/pom.xml");
|
||||
|
||||
@Test
|
||||
void commandGatewayIsOutsideDefaultProductionReactor() throws Exception {
|
||||
Document pom = rootPom();
|
||||
@@ -30,23 +38,607 @@ class MavenModuleProfileTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void xindaIsLegacyOnlyAndOutsideDefaultProductionReactor() throws Exception {
|
||||
void jsatl12AttachmentProtocolIsOptionalAndOutsideDefaultProductionReactor() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(defaultModules(pom))
|
||||
.doesNotContain("modules/protocols/protocol-jsatl12");
|
||||
|
||||
assertThat(profileModules(pom, "optional-attachments"))
|
||||
.containsExactly("modules/protocols/protocol-jsatl12");
|
||||
}
|
||||
|
||||
@Test
|
||||
void latestStateServiceIsOptionalAndOutsideDefaultProductionReactor() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(defaultModules(pom))
|
||||
.doesNotContain("modules/services/vehicle-state-service");
|
||||
|
||||
assertThat(profileModules(pom, "optional-latest-state"))
|
||||
.containsExactly("modules/services/vehicle-state-service");
|
||||
}
|
||||
|
||||
@Test
|
||||
void rawArchiveStorePrototypeIsRemovedFromBuildSurface() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(defaultModules(pom))
|
||||
.doesNotContain("modules/sinks/raw-archive-store");
|
||||
|
||||
assertThat(profileModules(pom, "optional-raw-archive-store"))
|
||||
.as("raw bytes are written by sink-archive; raw-archive-store was an unused prototype")
|
||||
.isEmpty();
|
||||
assertThat(Files.exists(repositoryRoot().resolve("modules/sinks/raw-archive-store")))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void eventFileStoreCompatibilityPathIsRemovedFromProductionStack() throws Exception {
|
||||
Document pom = rootPom();
|
||||
String readme = Files.readString(repositoryRoot().resolve("README.md"));
|
||||
|
||||
assertThat(defaultModules(pom))
|
||||
.doesNotContain("modules/sinks/event-file-store");
|
||||
assertThat(profileModules(pom, "optional-event-file-store"))
|
||||
.as("DuckDB/Parquet event-file-store has been replaced by TDengine raw history")
|
||||
.isEmpty();
|
||||
assertThat(Files.exists(repositoryRoot().resolve("modules/sinks/event-file-store")))
|
||||
.isFalse();
|
||||
assertThat(rootDependencyManagementArtifacts(pom))
|
||||
.doesNotContain("event-file-store");
|
||||
assertThat(hasProjectProperty(pom, "duckdb.version"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(pom, "org.duckdb", "duckdb_jdbc"))
|
||||
.isFalse();
|
||||
assertThat(readme)
|
||||
.doesNotContain("event-file-store")
|
||||
.doesNotContain("DuckDB")
|
||||
.doesNotContain("Parquet");
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaSinkModuleUsesKafkaNamingInsteadOfMqNaming() throws Exception {
|
||||
Document pom = rootPom();
|
||||
Document kafkaSinkPom = modulePom("modules/sinks/sink-kafka/pom.xml");
|
||||
String readme = Files.readString(repositoryRoot().resolve("README.md"));
|
||||
String legacyModule = "modules/sinks/sink-" + "mq";
|
||||
String legacyArtifact = "sink-" + "mq";
|
||||
|
||||
assertThat(defaultModules(pom))
|
||||
.contains("modules/sinks/sink-kafka")
|
||||
.doesNotContain(legacyModule);
|
||||
assertThat(rootDependencyManagementArtifacts(pom))
|
||||
.contains("sink-kafka")
|
||||
.doesNotContain(legacyArtifact);
|
||||
assertThat(firstDirectChild(kafkaSinkPom.getDocumentElement(), "artifactId").getTextContent())
|
||||
.isEqualTo("sink-kafka");
|
||||
assertThat(readme)
|
||||
.contains("sink-kafka/")
|
||||
.doesNotContain(legacyArtifact);
|
||||
}
|
||||
|
||||
@Test
|
||||
void xindaModulesAreRemovedFromBuildSurface() throws Exception {
|
||||
Document pom = rootPom();
|
||||
String rootPomText = Files.readString(repositoryRoot().resolve("pom.xml"));
|
||||
String readme = Files.readString(repositoryRoot().resolve("README.md"));
|
||||
|
||||
assertThat(defaultModules(pom))
|
||||
.doesNotContain("modules/inbound/inbound-xinda-push")
|
||||
.doesNotContain("modules/apps/xinda-push-app");
|
||||
|
||||
assertThat(profileModules(pom, "legacy-xinda"))
|
||||
.containsExactly(
|
||||
"modules/inbound/inbound-xinda-push",
|
||||
"modules/apps/xinda-push-app");
|
||||
.as("xinda is deleted, not hidden behind an opt-in profile")
|
||||
.isEmpty();
|
||||
assertThat(Files.exists(repositoryRoot().resolve("modules/inbound/inbound-xinda-push")))
|
||||
.isFalse();
|
||||
assertThat(Files.exists(repositoryRoot().resolve("modules/apps/xinda-push-app")))
|
||||
.isFalse();
|
||||
assertThat(rootPomText)
|
||||
.doesNotContain("legacy-xinda")
|
||||
.doesNotContain("inbound-xinda-push")
|
||||
.doesNotContain("xinda-push-app");
|
||||
assertThat(readme)
|
||||
.contains("信达 Push 已废弃并删除源码")
|
||||
.doesNotContain("信达 Push 仅保留")
|
||||
.doesNotContain("废弃兼容模块");
|
||||
}
|
||||
|
||||
@Test
|
||||
void privateLingniuRepositoryIsRemovedWithXinda() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(repositoryIds(pom.getDocumentElement()))
|
||||
.doesNotContain("lingniu-nexus");
|
||||
assertThat(profileRepositoryIds(pom, "legacy-xinda"))
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void optionalModuleVersionsAreScopedToTheirProfiles() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(rootDependencyManagementArtifacts(pom))
|
||||
.doesNotContain(
|
||||
"protocol-jt1078",
|
||||
"protocol-jsatl12",
|
||||
"vehicle-state-service",
|
||||
"raw-archive-store",
|
||||
"command-gateway",
|
||||
"inbound-xinda-push",
|
||||
"xinda-push-app");
|
||||
|
||||
assertThat(profileDependencyManagementArtifacts(pom, "optional-command-gateway"))
|
||||
.containsExactly("protocol-jt1078", "command-gateway");
|
||||
assertThat(profileDependencyManagementArtifacts(pom, "optional-attachments"))
|
||||
.containsExactly("protocol-jsatl12");
|
||||
assertThat(profileDependencyManagementArtifacts(pom, "optional-latest-state"))
|
||||
.containsExactly("vehicle-state-service");
|
||||
assertThat(profileDependencyManagementArtifacts(pom, "optional-raw-archive-store"))
|
||||
.isEmpty();
|
||||
assertThat(profileDependencyManagementArtifacts(pom, "optional-event-file-store"))
|
||||
.isEmpty();
|
||||
assertThat(profileDependencyManagementArtifacts(pom, "legacy-xinda"))
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void appsDirectoryContainsOnlyRealMavenModules() throws Exception {
|
||||
try (Stream<Path> apps = Files.list(repositoryRoot().resolve("modules/apps"))) {
|
||||
assertThat(apps.filter(Files::isDirectory)
|
||||
.filter(path -> !Files.exists(path.resolve("pom.xml")))
|
||||
.map(path -> path.getFileName().toString()))
|
||||
.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void vehicleIdentityTestSupportIsCentralizedInsteadOfCopiedIntoProtocolTests() throws Exception {
|
||||
Document pom = rootPom();
|
||||
List<String> testSupportFixtures = javaFilesNamed("InMemoryVehicleIdentityService.java");
|
||||
|
||||
assertThat(defaultModules(pom))
|
||||
.contains("modules/testing/vehicle-identity-test-support");
|
||||
assertThat(rootDependencyManagementArtifacts(pom))
|
||||
.contains("vehicle-identity-test-support");
|
||||
assertThat(testSupportFixtures)
|
||||
.containsExactly("modules/testing/vehicle-identity-test-support/src/main/java/com/lingniu/ingest/identity/InMemoryVehicleIdentityService.java");
|
||||
|
||||
for (String modulePom : List.of(
|
||||
"modules/protocols/protocol-jt808/pom.xml",
|
||||
"modules/inbound/inbound-mqtt/pom.xml",
|
||||
"modules/protocols/protocol-jt1078/pom.xml",
|
||||
"modules/protocols/protocol-jsatl12/pom.xml")) {
|
||||
assertThat(hasDependency(modulePom(modulePom), "com.lingniu.ingest", "vehicle-identity-test-support"))
|
||||
.as(modulePom)
|
||||
.isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void readmeMatchesCurrentRuntimeVersionsAndSplitAppResponsibilities() throws Exception {
|
||||
Document pom = rootPom();
|
||||
String readme = Files.readString(repositoryRoot().resolve("README.md"));
|
||||
|
||||
assertThat(readme)
|
||||
.contains("Spring Boot " + projectProperty(pom, "spring-boot.version"))
|
||||
.contains("Netty " + projectProperty(pom, "netty.version"))
|
||||
.contains("Eclipse Paho " + projectProperty(pom, "paho-mqtt.version"))
|
||||
.contains("session-core/ 设备会话 + 鉴权 + Token + Redis SessionStore")
|
||||
.contains("| 并发 | Disruptor 4 + 虚拟线程 |")
|
||||
.contains("| 事件流 | **Kafka**(唯一生产消息通道,vin 分区,保证单车有序) |")
|
||||
.contains("| 冷存 | 本地文件系统 |")
|
||||
.contains("| 构建 | Maven |")
|
||||
.contains("protocol-jt1078/ JT/T 1078(optional-command-gateway profile")
|
||||
.contains("protocol-jsatl12/ 苏标主动安全报警附件(optional-attachments profile)")
|
||||
.contains("vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询(optional-latest-state profile)")
|
||||
.contains("command-gateway/ 可选 HTTP → 设备下行命令(optional-command-gateway profile)")
|
||||
.contains("vehicle-history-app/ TDengine 历史查询 + RAW JSON")
|
||||
.contains("vehicle-analytics-app/ JT808 每日里程指标消费")
|
||||
.contains("协议接入应用只负责收、解析、校验、规整和投递 Kafka")
|
||||
.contains("生产链路只支持 Kafka")
|
||||
.contains("`vehicle-analytics-app` 将 JT808 `daily_mileage_km` 写入 MySQL `vehicle_stat_metric`")
|
||||
.contains("mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true")
|
||||
.contains("生产服务只在 ECS/Portainer 上运行")
|
||||
.doesNotContain("Spring Boot 3.4")
|
||||
.doesNotContain("Netty 4.1")
|
||||
.doesNotContain("HiveMQ MQTT Client")
|
||||
.doesNotContain("RabbitMQ")
|
||||
.doesNotContain("RocketMQ")
|
||||
.doesNotContain("其他 MQ 后端")
|
||||
.doesNotContain("Spotless")
|
||||
.doesNotContain("ArchUnit")
|
||||
.doesNotContain("S3 / OSS / 本地")
|
||||
.doesNotContain("StructuredTaskScope")
|
||||
.doesNotContain("mvn clean install -DskipTests")
|
||||
.doesNotContain("-DskipTests")
|
||||
.doesNotContain("spring-boot:run")
|
||||
.doesNotContain("Caffeine 内存降级")
|
||||
.doesNotContain("Redis/Memory SessionStore")
|
||||
.doesNotContain("event-file-store")
|
||||
.doesNotContain("DuckDB")
|
||||
.doesNotContain("Parquet")
|
||||
.doesNotContain("Parquet + DuckDB 文件型明细库")
|
||||
.doesNotContain("车辆状态/日统计消费")
|
||||
.doesNotContain("本服务不写业务库")
|
||||
.doesNotContain("里程统计全部由 Kafka 下游消费者实现")
|
||||
.doesNotContain("| 消息队列 |");
|
||||
}
|
||||
|
||||
@Test
|
||||
void archiveBuildSurfaceDocumentsLocalFilesystemOnly() throws Exception {
|
||||
String archivePom = Files.readString(repositoryRoot().resolve("modules/sinks/sink-archive/pom.xml"));
|
||||
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
|
||||
|
||||
assertThat(archivePom)
|
||||
.contains("原始报文冷存:本地文件系统实现。")
|
||||
.doesNotContain("S3/OSS")
|
||||
.doesNotContain("MinIO")
|
||||
.doesNotContain("software.amazon.awssdk")
|
||||
.doesNotContain("com.aliyun.oss");
|
||||
assertThat(moduleDataFlow)
|
||||
.contains("sink-archive<br/>RawArchive 到本地文件系统冷存")
|
||||
.contains("归档文件的真实本地路径只属于 ArchiveStore")
|
||||
.contains("原始报文冷存,当前为本地文件系统实现")
|
||||
.doesNotContain("本地/S3/OSS")
|
||||
.doesNotContain("S3/OSS")
|
||||
.doesNotContain("MinIO");
|
||||
}
|
||||
|
||||
@Test
|
||||
void yutongMqttBuildSurfaceUsesOnlyPahoClient() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(hasProjectProperty(pom, "paho-mqtt.version"))
|
||||
.isTrue();
|
||||
assertThat(hasDependency(pom, "org.eclipse.paho", "org.eclipse.paho.client.mqttv3"))
|
||||
.isTrue();
|
||||
assertThat(hasProjectProperty(pom, "fusesource-mqtt.version"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(pom, "org.fusesource.mqtt-client", "mqtt-client"))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void activeAppBootPackagingConfigurationIsManagedOnceInRootPom() throws Exception {
|
||||
Document pom = rootPom();
|
||||
Element managedBootPlugin = managedPlugin(pom, "org.springframework.boot", "spring-boot-maven-plugin");
|
||||
|
||||
assertThat(managedBootPlugin).isNotNull();
|
||||
assertThat(managedBootPlugin.getTextContent())
|
||||
.contains("${spring-boot.version}")
|
||||
.contains("--sun-misc-unsafe-memory-access=allow")
|
||||
.contains("-Dio.netty.transport.noNative=true")
|
||||
.contains("repackage")
|
||||
.contains("build-info");
|
||||
|
||||
for (String appPom : ACTIVE_APP_POMS) {
|
||||
Document app = modulePom(appPom);
|
||||
Element bootPlugin = buildPlugin(app, "org.springframework.boot", "spring-boot-maven-plugin");
|
||||
|
||||
assertThat(bootPlugin)
|
||||
.as(appPom)
|
||||
.isNotNull();
|
||||
assertThat(firstDirectChild(bootPlugin, "configuration"))
|
||||
.as(appPom + " should inherit shared JVM arguments from root pluginManagement")
|
||||
.isNull();
|
||||
assertThat(firstDirectChild(bootPlugin, "executions"))
|
||||
.as(appPom + " should inherit repackage/build-info executions from root pluginManagement")
|
||||
.isNull();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void observabilityBuildSurfaceUsesMicrometerWithoutUnusedOpenTelemetryBom() throws Exception {
|
||||
Document pom = rootPom();
|
||||
Document observabilityPom = modulePom("modules/core/observability/pom.xml");
|
||||
String readme = Files.readString(repositoryRoot().resolve("README.md"));
|
||||
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
|
||||
String traceSpan = Files.readString(repositoryRoot()
|
||||
.resolve("modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/annotation/TraceSpan.java"));
|
||||
|
||||
assertThat(hasProjectProperty(pom, "micrometer.version"))
|
||||
.isTrue();
|
||||
assertThat(hasDependency(pom, "io.micrometer", "micrometer-bom"))
|
||||
.isTrue();
|
||||
assertThat(hasProjectProperty(pom, "otel.version"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(pom, "io.opentelemetry", "opentelemetry-bom"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(observabilityPom, "com.lingniu.ingest", "ingest-core"))
|
||||
.as("observability is an interceptor API module and should not depend on core implementation")
|
||||
.isFalse();
|
||||
assertThat(productionJavaContains("io.opentelemetry"))
|
||||
.isFalse();
|
||||
assertThat(readme)
|
||||
.contains("| 可观测 | Micrometer + Prometheus |")
|
||||
.doesNotContain("OpenTelemetry");
|
||||
assertThat(moduleDataFlow)
|
||||
.contains("observability<br/>metrics / health")
|
||||
.doesNotContain("observability<br/>metrics / tracing / health");
|
||||
assertThat(traceSpan)
|
||||
.doesNotContain("OpenTelemetry")
|
||||
.doesNotContain("span");
|
||||
}
|
||||
|
||||
@Test
|
||||
void activeAppsDependOnSharedObservabilityForHealthAndMetrics() throws Exception {
|
||||
for (String appPom : ACTIVE_APP_POMS) {
|
||||
assertThat(hasDependency(modulePom(appPom), "com.lingniu.ingest", "observability"))
|
||||
.as(appPom + " exposes actuator health/readiness and Prometheus metrics")
|
||||
.isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void springdocUiIsOnlyPackagedWithAppsThatExposeRestApis() throws Exception {
|
||||
for (String appPom : List.of(
|
||||
"modules/apps/gb32960-ingest-app/pom.xml",
|
||||
"modules/apps/jt808-ingest-app/pom.xml",
|
||||
"modules/apps/yutong-mqtt-app/pom.xml")) {
|
||||
assertThat(hasDependency(modulePom(appPom), "org.springdoc", "springdoc-openapi-starter-webmvc-ui"))
|
||||
.as(appPom + " exposes Actuator only; Swagger UI belongs to query/API apps")
|
||||
.isFalse();
|
||||
}
|
||||
for (String appPom : List.of(
|
||||
"modules/apps/vehicle-history-app/pom.xml",
|
||||
"modules/apps/vehicle-analytics-app/pom.xml")) {
|
||||
assertThat(hasDependency(modulePom(appPom), "org.springdoc", "springdoc-openapi-starter-webmvc-ui"))
|
||||
.as(appPom + " exposes REST APIs")
|
||||
.isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildSurfaceDoesNotManageLombokWhenProductionSourcesDoNotUseIt() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(hasProjectProperty(pom, "lombok.version"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(pom, "org.projectlombok", "lombok"))
|
||||
.isFalse();
|
||||
assertThat(productionJavaContains("lombok."))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildSurfaceDoesNotManageArchUnitWhenThereAreNoArchitectureTests() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(hasProjectProperty(pom, "archunit.version"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(pom, "com.tngtech.archunit", "archunit-junit5"))
|
||||
.isFalse();
|
||||
assertThat(javaSourcesContain("import com.tngtech." + "archunit"))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildSurfaceDoesNotManageTestcontainersWhenKafkaTestsUseInMemoryFakes() throws Exception {
|
||||
Document pom = rootPom();
|
||||
Document kafkaSinkPom = modulePom("modules/sinks/sink-kafka/pom.xml");
|
||||
|
||||
assertThat(hasProjectProperty(pom, "testcontainers.version"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(pom, "org.testcontainers", "testcontainers-bom"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(kafkaSinkPom, "org.testcontainers", "kafka"))
|
||||
.isFalse();
|
||||
assertThat(javaSourcesContain("import org." + "testcontainers"))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void productionDocsDoNotAdvertiseMemorySessionStore() throws Exception {
|
||||
String runbook = Files.readString(repositoryRoot()
|
||||
.resolve("docs/operations/gb32960-service-split-runbook.md"));
|
||||
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
|
||||
String redesignSpec = Files.readString(repositoryRoot()
|
||||
.resolve("docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md"));
|
||||
|
||||
assertThat(runbook)
|
||||
.contains("Session state requires Redis")
|
||||
.contains("REDIS_HOST")
|
||||
.contains("REDIS_DATABASE=50")
|
||||
.doesNotContain("SESSION_STORE=memory")
|
||||
.doesNotContain("without Redis");
|
||||
assertThat(moduleDataFlow)
|
||||
.contains("SessionStore 仅保留 Redis 后端")
|
||||
.doesNotContain("SessionStore 支持 <code>memory</code>")
|
||||
.doesNotContain("无 Redis 降级");
|
||||
assertThat(redesignSpec)
|
||||
.contains("Reads Redis latest state")
|
||||
.doesNotContain("Redis or in-memory")
|
||||
.doesNotContain("Redis/in-memory");
|
||||
}
|
||||
|
||||
@Test
|
||||
void sessionCorePomDescriptionDoesNotAdvertiseRedisAsFallback() throws Exception {
|
||||
Document sessionCorePom = modulePom("modules/core/session-core/pom.xml");
|
||||
String description = firstDirectChild(sessionCorePom.getDocumentElement(), "description").getTextContent();
|
||||
|
||||
assertThat(description)
|
||||
.contains("Redis SessionStore")
|
||||
.doesNotContain("兜底")
|
||||
.doesNotContain("fallback")
|
||||
.doesNotContain("memory");
|
||||
}
|
||||
|
||||
@Test
|
||||
void rootPomDescriptionMatchesCurrentRuntimeStack() throws Exception {
|
||||
Document pom = rootPom();
|
||||
String description = firstDirectChild(pom.getDocumentElement(), "description").getTextContent();
|
||||
|
||||
assertThat(description)
|
||||
.contains("只支持 Kafka")
|
||||
.contains("Java 25")
|
||||
.contains("Spring Boot " + projectProperty(pom, "spring-boot.version"))
|
||||
.contains("Netty " + projectProperty(pom, "netty.version"))
|
||||
.doesNotContain("Spring Boot 3.4");
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultProductionSurfaceOnlySupportsKafkaAsMessageBackbone() throws Exception {
|
||||
List<Path> productionSurface = List.of(
|
||||
repositoryRoot().resolve("README.md"),
|
||||
repositoryRoot().resolve("pom.xml"),
|
||||
repositoryRoot().resolve("deploy/portainer/docker-compose.yml"),
|
||||
repositoryRoot().resolve("docs/operations/gb32960-service-split-runbook.md"),
|
||||
repositoryRoot().resolve("docs/operations/vehicle-ingest-tdengine-verification.md"),
|
||||
repositoryRoot().resolve("docs/operations/jt808-daily-mileage-runbook.md"),
|
||||
repositoryRoot().resolve("docs/target-architecture.md"));
|
||||
|
||||
for (Path path : productionSurface) {
|
||||
String text = Files.readString(path);
|
||||
|
||||
assertThat(text)
|
||||
.as(path.toString())
|
||||
.doesNotContain("sink-mq")
|
||||
.doesNotContain("RabbitMQ")
|
||||
.doesNotContain("RocketMQ")
|
||||
.doesNotContain("Pulsar")
|
||||
.doesNotContain("AMQP")
|
||||
.doesNotContain("其他 MQ 后端");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void architectureDecisionsMatchCurrentDefaultProductionSurface() throws Exception {
|
||||
String decisions = Files.readString(repositoryRoot().resolve("DECISIONS.md"));
|
||||
|
||||
assertThat(decisions)
|
||||
.contains("## ADR-004 信达 Push:Removed")
|
||||
.contains("## ADR-006 部署形态:GB32960 三应用拆分\n- **Status**: Superseded by ADR-011")
|
||||
.contains("## ADR-011 默认生产面:三协议接入 + 历史 + 统计")
|
||||
.contains("## ADR-012 JSATL12 附件上传:Optional only")
|
||||
.contains("GB32960、JT808、Yutong MQTT、vehicle-history-app、vehicle-analytics-app")
|
||||
.contains("Xinda Push 源码、Maven profile、Woodpecker 镜像发布和历史消费绑定全部移除")
|
||||
.contains("JSATL12 仅保留在 `optional-attachments` profile")
|
||||
.contains("vehicle-state-service 仅保留在 `optional-latest-state` profile")
|
||||
.contains("raw-archive-store 原型已删除")
|
||||
.contains("协议接入应用不直接写业务库")
|
||||
.contains("`vehicle-analytics-app` 写入 MySQL `vehicle_stat_metric`")
|
||||
.doesNotContain("event-file-store")
|
||||
.doesNotContain("optional-event-file-store")
|
||||
.doesNotContain("optional-raw-archive-store")
|
||||
.doesNotContain("legacy-xinda")
|
||||
.doesNotContain("## ADR-004 信达 Push:保留模块但彻底重写\n- **Status**: Accepted")
|
||||
.doesNotContain("CI/CD 分别构建三个镜像")
|
||||
.doesNotContain("Spring Boot 3.4.x");
|
||||
}
|
||||
|
||||
@Test
|
||||
void productionDocsMatchCompactHistoryAndJt808OnlyAnalyticsSurface() throws Exception {
|
||||
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
|
||||
String targetArchitecture = Files.readString(repositoryRoot().resolve("docs/target-architecture.md"));
|
||||
String tdengineVerification = Files.readString(repositoryRoot()
|
||||
.resolve("docs/operations/vehicle-ingest-tdengine-verification.md"));
|
||||
String runbook = Files.readString(repositoryRoot()
|
||||
.resolve("docs/operations/gb32960-service-split-runbook.md"));
|
||||
|
||||
assertThat(moduleDataFlow)
|
||||
.contains("TDengine raw_frames + locations")
|
||||
.contains("TDengine <code>raw_frames</code> 和位置表")
|
||||
.doesNotContain("telemetry field 行映射")
|
||||
.doesNotContain("可选 telemetry field 表");
|
||||
assertThat(tdengineVerification)
|
||||
.contains("history 写入 raw、location 到 TDengine;不写 telemetry_fields")
|
||||
.doesNotContain("按需开启 telemetry field");
|
||||
assertThat(runbook)
|
||||
.contains("| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` |")
|
||||
.contains("| `vehicle.event.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` |")
|
||||
.doesNotContain("| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` |");
|
||||
assertThat(targetArchitecture)
|
||||
.contains("vehicle-state-service is absent from the default Maven reactor")
|
||||
.doesNotContain("kafka --> state[\"vehicle-state-service\"]")
|
||||
.doesNotContain("state --> redisState[\"Redis<br/>latest state\"]");
|
||||
}
|
||||
|
||||
@Test
|
||||
void supersededDesignDocsPointToCurrentArchitecture() throws Exception {
|
||||
for (String path : List.of(
|
||||
"docs/superpowers/specs/2026-06-23-gb32960-production-readiness-design.md",
|
||||
"docs/superpowers/specs/2026-06-23-gb32960-service-split-design.md")) {
|
||||
String doc = Files.readString(repositoryRoot().resolve(path));
|
||||
|
||||
assertThat(doc)
|
||||
.as(path)
|
||||
.startsWith("> **Superseded:**")
|
||||
.contains("docs/target-architecture.md")
|
||||
.contains("docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void supersededImplementationPlansPointToCurrentArchitecture() throws Exception {
|
||||
for (String path : List.of(
|
||||
"docs/superpowers/plans/2026-06-23-gb32960-history-duckdb-hot-store.md",
|
||||
"docs/superpowers/plans/2026-06-23-gb32960-service-split.md",
|
||||
"docs/superpowers/plans/2026-06-29-vehicle-ingest-redesign-phase1.md")) {
|
||||
String doc = Files.readString(repositoryRoot().resolve(path));
|
||||
|
||||
assertThat(doc)
|
||||
.as(path)
|
||||
.startsWith("> **Superseded:**")
|
||||
.contains("docs/target-architecture.md")
|
||||
.contains("docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void duckDbAndParquetAreNotPartOfTheCurrentBuildSurface() throws Exception {
|
||||
Document pom = rootPom();
|
||||
|
||||
assertThat(hasProjectProperty(pom, "duckdb.version"))
|
||||
.isFalse();
|
||||
assertThat(hasDependency(pom, "org.duckdb", "duckdb_jdbc"))
|
||||
.isFalse();
|
||||
assertThat(Files.exists(repositoryRoot().resolve("modules/sinks/event-file-store")))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyEventFileStoreContractsAreRemovedFromProductionSources() throws Exception {
|
||||
Path root = repositoryRoot();
|
||||
|
||||
assertThat(Files.exists(root.resolve("modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/history")))
|
||||
.isFalse();
|
||||
assertThat(Files.exists(root.resolve(
|
||||
"modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java")))
|
||||
.isFalse();
|
||||
assertThat(Files.exists(root.resolve(
|
||||
"modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java")))
|
||||
.isFalse();
|
||||
assertThat(productionJavaContains("EventFileStore"))
|
||||
.isFalse();
|
||||
assertThat(productionJavaContains("EventFileRecord"))
|
||||
.isFalse();
|
||||
assertThat(productionJavaContains("EventFileQuery"))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
private static Document rootPom() throws Exception {
|
||||
return modulePom("pom.xml");
|
||||
}
|
||||
|
||||
private static Document modulePom(String path) throws Exception {
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
|
||||
return factory.newDocumentBuilder().parse(Files.newInputStream(repositoryRoot().resolve("pom.xml")));
|
||||
return factory.newDocumentBuilder().parse(Files.newInputStream(repositoryRoot().resolve(path)));
|
||||
}
|
||||
|
||||
private static String projectProperty(Document pom, String name) {
|
||||
Element properties = firstDirectChild(pom.getDocumentElement(), "properties");
|
||||
Element property = firstDirectChild(properties, name);
|
||||
if (property == null) {
|
||||
throw new IllegalArgumentException("project property not found: " + name);
|
||||
}
|
||||
return property.getTextContent().trim();
|
||||
}
|
||||
|
||||
private static boolean hasProjectProperty(Document pom, String name) {
|
||||
Element properties = firstDirectChild(pom.getDocumentElement(), "properties");
|
||||
return properties != null && firstDirectChild(properties, name) != null;
|
||||
}
|
||||
|
||||
private static List<String> defaultModules(Document pom) {
|
||||
@@ -55,9 +647,56 @@ class MavenModuleProfileTest {
|
||||
}
|
||||
|
||||
private static List<String> profileModules(Document pom, String profileId) {
|
||||
Element profile = profile(pom, profileId);
|
||||
if (profile == null) {
|
||||
return List.of();
|
||||
}
|
||||
return directModules(firstDirectChild(profile, "modules"));
|
||||
}
|
||||
|
||||
private static List<String> rootDependencyManagementArtifacts(Document pom) {
|
||||
return dependencyManagementArtifacts(pom.getDocumentElement());
|
||||
}
|
||||
|
||||
private static List<String> profileDependencyManagementArtifacts(Document pom, String profileId) {
|
||||
Element profile = profile(pom, profileId);
|
||||
if (profile == null) {
|
||||
return List.of();
|
||||
}
|
||||
return dependencyManagementArtifacts(profile);
|
||||
}
|
||||
|
||||
private static List<String> profileRepositoryIds(Document pom, String profileId) {
|
||||
Element profile = profile(pom, profileId);
|
||||
if (profile == null) {
|
||||
return List.of();
|
||||
}
|
||||
return repositoryIds(profile);
|
||||
}
|
||||
|
||||
private static List<String> repositoryIds(Element owner) {
|
||||
Element repositories = firstDirectChild(owner, "repositories");
|
||||
if (repositories == null) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> ids = new ArrayList<>();
|
||||
NodeList children = repositories.getChildNodes();
|
||||
for (int i = 0; i < children.getLength(); i++) {
|
||||
if (!(children.item(i) instanceof Element repository) || !"repository".equals(repository.getTagName())) {
|
||||
continue;
|
||||
}
|
||||
Element id = firstDirectChild(repository, "id");
|
||||
if (id != null) {
|
||||
ids.add(id.getTextContent().trim());
|
||||
}
|
||||
}
|
||||
return List.copyOf(ids);
|
||||
}
|
||||
|
||||
private static Element profile(Document pom, String profileId) {
|
||||
Element profiles = firstDirectChild(pom.getDocumentElement(), "profiles");
|
||||
if (profiles == null) {
|
||||
return List.of();
|
||||
return null;
|
||||
}
|
||||
NodeList children = profiles.getChildNodes();
|
||||
for (int i = 0; i < children.getLength(); i++) {
|
||||
@@ -66,10 +705,10 @@ class MavenModuleProfileTest {
|
||||
}
|
||||
Element id = firstDirectChild(profile, "id");
|
||||
if (id != null && profileId.equals(id.getTextContent().trim())) {
|
||||
return directModules(firstDirectChild(profile, "modules"));
|
||||
return profile;
|
||||
}
|
||||
}
|
||||
return List.of();
|
||||
return null;
|
||||
}
|
||||
|
||||
private static List<String> directModules(Element modules) {
|
||||
@@ -86,6 +725,136 @@ class MavenModuleProfileTest {
|
||||
return List.copyOf(out);
|
||||
}
|
||||
|
||||
private static List<String> dependencyManagementArtifacts(Element owner) {
|
||||
Element dependencyManagement = firstDirectChild(owner, "dependencyManagement");
|
||||
if (dependencyManagement == null) {
|
||||
return List.of();
|
||||
}
|
||||
Element dependencies = firstDirectChild(dependencyManagement, "dependencies");
|
||||
if (dependencies == null) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> artifacts = new ArrayList<>();
|
||||
NodeList children = dependencies.getChildNodes();
|
||||
for (int i = 0; i < children.getLength(); i++) {
|
||||
if (!(children.item(i) instanceof Element dependency) || !"dependency".equals(dependency.getTagName())) {
|
||||
continue;
|
||||
}
|
||||
Element groupId = firstDirectChild(dependency, "groupId");
|
||||
Element artifactId = firstDirectChild(dependency, "artifactId");
|
||||
if (groupId != null
|
||||
&& artifactId != null
|
||||
&& "com.lingniu.ingest".equals(groupId.getTextContent().trim())) {
|
||||
artifacts.add(artifactId.getTextContent().trim());
|
||||
}
|
||||
}
|
||||
return List.copyOf(artifacts);
|
||||
}
|
||||
|
||||
private static Element managedPlugin(Document pom, String groupId, String artifactId) {
|
||||
Element build = firstDirectChild(pom.getDocumentElement(), "build");
|
||||
Element pluginManagement = build == null ? null : firstDirectChild(build, "pluginManagement");
|
||||
return plugin(pluginManagement, groupId, artifactId);
|
||||
}
|
||||
|
||||
private static Element buildPlugin(Document pom, String groupId, String artifactId) {
|
||||
Element build = firstDirectChild(pom.getDocumentElement(), "build");
|
||||
return plugin(build, groupId, artifactId);
|
||||
}
|
||||
|
||||
private static Element plugin(Element owner, String groupId, String artifactId) {
|
||||
Element plugins = owner == null ? null : firstDirectChild(owner, "plugins");
|
||||
if (plugins == null) {
|
||||
return null;
|
||||
}
|
||||
NodeList children = plugins.getChildNodes();
|
||||
for (int i = 0; i < children.getLength(); i++) {
|
||||
if (!(children.item(i) instanceof Element plugin) || !"plugin".equals(plugin.getTagName())) {
|
||||
continue;
|
||||
}
|
||||
Element currentGroupId = firstDirectChild(plugin, "groupId");
|
||||
Element currentArtifactId = firstDirectChild(plugin, "artifactId");
|
||||
if (currentGroupId != null
|
||||
&& currentArtifactId != null
|
||||
&& groupId.equals(currentGroupId.getTextContent().trim())
|
||||
&& artifactId.equals(currentArtifactId.getTextContent().trim())) {
|
||||
return plugin;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean hasDependency(Document pom, String groupId, String artifactId) {
|
||||
Element owner = pom.getDocumentElement();
|
||||
Element dependencyManagement = firstDirectChild(owner, "dependencyManagement");
|
||||
Element dependencies = firstDirectChild(owner, "dependencies");
|
||||
if (hasDependencyIn(dependencies, groupId, artifactId)) {
|
||||
return true;
|
||||
}
|
||||
if (dependencyManagement == null) {
|
||||
return false;
|
||||
}
|
||||
return hasDependencyIn(firstDirectChild(dependencyManagement, "dependencies"), groupId, artifactId);
|
||||
}
|
||||
|
||||
private static boolean hasDependencyIn(Element dependencies, String groupId, String artifactId) {
|
||||
if (dependencies == null) {
|
||||
return false;
|
||||
}
|
||||
NodeList children = dependencies.getChildNodes();
|
||||
for (int i = 0; i < children.getLength(); i++) {
|
||||
if (!(children.item(i) instanceof Element dependency) || !"dependency".equals(dependency.getTagName())) {
|
||||
continue;
|
||||
}
|
||||
Element currentGroupId = firstDirectChild(dependency, "groupId");
|
||||
Element currentArtifactId = firstDirectChild(dependency, "artifactId");
|
||||
if (currentGroupId != null
|
||||
&& currentArtifactId != null
|
||||
&& groupId.equals(currentGroupId.getTextContent().trim())
|
||||
&& artifactId.equals(currentArtifactId.getTextContent().trim())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static boolean productionJavaContains(String token) throws Exception {
|
||||
return javaSourcesContain(token, false);
|
||||
}
|
||||
|
||||
private static boolean javaSourcesContain(String token) throws Exception {
|
||||
return javaSourcesContain(token, true);
|
||||
}
|
||||
|
||||
private static boolean javaSourcesContain(String token, boolean includeTests) throws Exception {
|
||||
try (Stream<Path> files = Files.walk(repositoryRoot().resolve("modules"))) {
|
||||
return files
|
||||
.filter(Files::isRegularFile)
|
||||
.filter(path -> path.toString().endsWith(".java"))
|
||||
.filter(path -> includeTests || !path.toString().contains("/src/test/"))
|
||||
.anyMatch(path -> {
|
||||
try {
|
||||
return Files.readString(path).contains(token);
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException("failed to read " + path, e);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static List<String> javaFilesNamed(String fileName) throws Exception {
|
||||
Path root = repositoryRoot();
|
||||
try (Stream<Path> files = Files.walk(root.resolve("modules"))) {
|
||||
return files
|
||||
.filter(Files::isRegularFile)
|
||||
.filter(path -> fileName.equals(path.getFileName().toString()))
|
||||
.map(root::relativize)
|
||||
.map(Path::toString)
|
||||
.sorted()
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
private static Element firstDirectChild(Element parent, String tagName) {
|
||||
NodeList children = parent.getChildNodes();
|
||||
for (int i = 0; i < children.getLength(); i++) {
|
||||
|
||||
@@ -1,15 +1,34 @@
|
||||
package com.lingniu.ingest.historyapp;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class PortainerComposeResourceLimitsTest {
|
||||
|
||||
@Test
|
||||
void portainerStackDocumentsDefaultProductionSurfaceOnly() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
|
||||
assertThat(compose)
|
||||
.contains("Default production stack: GB32960, JT808, Yutong MQTT, history, analytics.")
|
||||
.contains("Legacy and optional services stay outside this compose file.")
|
||||
.doesNotContain("KAFKA_ENABLED")
|
||||
.doesNotContain("\n xinda-push-app:\n")
|
||||
.doesNotContain("\n command-gateway:\n")
|
||||
.doesNotContain("\n vehicle-state-service:\n")
|
||||
.doesNotContain("\n raw-archive-store:\n")
|
||||
.doesNotContain("\n event-file-store:\n");
|
||||
}
|
||||
|
||||
@Test
|
||||
void productionServicesHaveOverridableMemoryLimits() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
@@ -22,19 +41,243 @@ class PortainerComposeResourceLimitsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void vehicleHistoryUsesApplicationConsumerGroupEnvironmentNames() throws IOException {
|
||||
void productionImagesShareOverridableRegistryAndNamespace() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String registry = "${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}";
|
||||
String namespace = "${LINGNIU_IMAGE_NAMESPACE:-oneos}";
|
||||
|
||||
for (String serviceName : List.of(
|
||||
"gb32960-ingest-app",
|
||||
"jt808-ingest-app",
|
||||
"yutong-mqtt-app",
|
||||
"vehicle-history-app",
|
||||
"vehicle-analytics-app")) {
|
||||
assertThat(serviceBlock(compose, serviceName))
|
||||
.contains("image: " + registry + "/" + namespace + "/" + serviceName
|
||||
+ ":${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}");
|
||||
}
|
||||
assertThat(countOccurrences(compose, "image: crpi-85r4m0ackrm3qpje"))
|
||||
.isZero();
|
||||
assertThat(countOccurrences(compose, "/oneos/")).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void portainerServicesUseContainerReadinessHealthchecks() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String dockerfile = Files.readString(repositoryRoot().resolve("Dockerfile"));
|
||||
|
||||
assertThat(dockerfile).contains("apt-get install -y --no-install-recommends curl");
|
||||
assertThat(compose)
|
||||
.contains("x-readiness-healthcheck: &readiness-healthcheck\n"
|
||||
+ " interval: 30s\n"
|
||||
+ " timeout: 5s\n"
|
||||
+ " retries: 3\n"
|
||||
+ " start_period: 60s");
|
||||
assertServiceReadinessHealthcheck(compose, "gb32960-ingest-app", "20100");
|
||||
assertServiceReadinessHealthcheck(compose, "jt808-ingest-app", "20400");
|
||||
assertServiceReadinessHealthcheck(compose, "yutong-mqtt-app", "20500");
|
||||
assertServiceReadinessHealthcheck(compose, "vehicle-history-app", "20200");
|
||||
assertServiceReadinessHealthcheck(compose, "vehicle-analytics-app", "20310");
|
||||
}
|
||||
|
||||
@Test
|
||||
void containerDefaultsIncludeJava25UnsafeCompatibilityFlag() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String dockerfile = Files.readString(repositoryRoot().resolve("Dockerfile"));
|
||||
|
||||
assertThat(dockerfile)
|
||||
.contains("ENV JAVA_OPTS=\"--sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true\"");
|
||||
assertThat(commonEnvBlock(compose))
|
||||
.contains("JAVA_OPTS: ${JAVA_OPTS:---sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true -XX:+UseZGC -XX:MaxRAMPercentage=75}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void redisSessionDefaultsUseEcsPrivateEndpoint() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
|
||||
assertThat(compose)
|
||||
.contains("KAFKA_GROUP_HISTORY_GB32960_EVENT: ${KAFKA_GROUP_HISTORY_GB32960_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-gb32960-event}")
|
||||
.contains("KAFKA_GROUP_HISTORY_JT808_EVENT: ${KAFKA_GROUP_HISTORY_JT808_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-jt808-event}")
|
||||
.contains("KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-yutong-mqtt-event}")
|
||||
.contains("KAFKA_GROUP_HISTORY_GB32960_RAW: ${KAFKA_GROUP_HISTORY_GB32960_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-gb32960-raw}")
|
||||
.contains("KAFKA_GROUP_HISTORY_JT808_RAW: ${KAFKA_GROUP_HISTORY_JT808_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-jt808-raw}")
|
||||
.contains("KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-yutong-mqtt-raw}")
|
||||
.doesNotContain("\n KAFKA_GROUP_HISTORY_GB32960:")
|
||||
.doesNotContain("\n KAFKA_GROUP_HISTORY_JT808:")
|
||||
.doesNotContain("\n KAFKA_GROUP_HISTORY_MQTT_YUTONG:");
|
||||
.contains("REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com}")
|
||||
.doesNotContain("r-bp1u741kij7e51i481pd.redis.rds.aliyuncs.com");
|
||||
}
|
||||
|
||||
@Test
|
||||
void activeAppsExposeHealthProbeGroupsByDefault() {
|
||||
Path root = repositoryRoot();
|
||||
List<Path> activeAppConfigs = List.of(
|
||||
root.resolve("modules/apps/gb32960-ingest-app/src/main/resources/application.yml"),
|
||||
root.resolve("modules/apps/jt808-ingest-app/src/main/resources/application.yml"),
|
||||
root.resolve("modules/apps/yutong-mqtt-app/src/main/resources/application.yml"),
|
||||
root.resolve("modules/apps/vehicle-history-app/src/main/resources/application.yml"),
|
||||
root.resolve("modules/apps/vehicle-analytics-app/src/main/resources/application.yml"));
|
||||
|
||||
for (Path config : activeAppConfigs) {
|
||||
assertThat(yamlProperties(config))
|
||||
.as(config.toString())
|
||||
.containsEntry("management.endpoint.health.probes.enabled",
|
||||
"${MANAGEMENT_HEALTH_PROBES_ENABLED:true}");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void productionRunbookChecksLivenessAndReadinessForActiveApps() throws IOException {
|
||||
String runbook = Files.readString(repositoryRoot()
|
||||
.resolve("docs/operations/gb32960-service-split-runbook.md"));
|
||||
|
||||
for (String httpPort : List.of("20100", "20400", "20500", "20200", "20310")) {
|
||||
assertThat(runbook)
|
||||
.contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/liveness")
|
||||
.contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/readiness");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultProductionSurfaceDoesNotShipLocalLaunchctlRunners() throws IOException {
|
||||
Path root = repositoryRoot();
|
||||
String runbook = Files.readString(root.resolve("docs/operations/gb32960-service-split-runbook.md"));
|
||||
|
||||
assertThat(root.resolve("deploy/local/launchctl")).doesNotExist();
|
||||
assertThat(runbook)
|
||||
.contains("Production services run on ECS through Portainer/Docker.")
|
||||
.doesNotContain("Local launchctl Deployment")
|
||||
.doesNotContain("deploy/local/launchctl")
|
||||
.doesNotContain("launchctl bootstrap")
|
||||
.doesNotContain("launchctl bootout");
|
||||
}
|
||||
|
||||
@Test
|
||||
void tdengineVerificationRunbookChecksLivenessAndReadinessForManagedApps() throws IOException {
|
||||
String runbook = Files.readString(repositoryRoot()
|
||||
.resolve("docs/operations/vehicle-ingest-tdengine-verification.md"));
|
||||
|
||||
for (String httpPort : List.of("20100", "20400", "20500", "20200", "20310")) {
|
||||
assertThat(runbook)
|
||||
.contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/liveness")
|
||||
.contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/readiness");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void operationRunbooksUseBuildFlagThatSkipsTestCompilation() throws IOException {
|
||||
Path operations = repositoryRoot().resolve("docs/operations");
|
||||
String splitRunbook = Files.readString(operations.resolve("gb32960-service-split-runbook.md"));
|
||||
String tdengineVerification = Files.readString(operations.resolve("vehicle-ingest-tdengine-verification.md"));
|
||||
|
||||
assertThat(splitRunbook)
|
||||
.contains("-Dmaven.test.skip=true")
|
||||
.doesNotContain("-DskipTests");
|
||||
assertThat(tdengineVerification)
|
||||
.contains("-Dmaven.test.skip=true")
|
||||
.doesNotContain("-DskipTests");
|
||||
}
|
||||
|
||||
@Test
|
||||
void dockerEntrypointExecsJavaForSignalHandling() throws IOException {
|
||||
String dockerfile = Files.readString(repositoryRoot().resolve("Dockerfile"));
|
||||
|
||||
assertThat(dockerfile)
|
||||
.contains("ENTRYPOINT [\"sh\", \"-c\", \"exec java $JAVA_OPTS -jar /app/app.jar\"]")
|
||||
.doesNotContain("\"java $JAVA_OPTS -jar /app/app.jar\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void gb32960ComposeExposesBothPlatformAccountsWithoutSecrets() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String gb32960Service = serviceBlock(compose, "gb32960-ingest-app");
|
||||
|
||||
assertThat(gb32960Service)
|
||||
.contains("GB32960_PLATFORM_USER_HYUNDAI: ${GB32960_PLATFORM_USER_HYUNDAI:-Hyundai}")
|
||||
.contains("GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}")
|
||||
.contains("GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}")
|
||||
.contains("GB32960_PLATFORM_USER_YUEJIN: ${GB32960_PLATFORM_USER_YUEJIN:-YueJin}")
|
||||
.contains("GB32960_PLATFORM_PWD_YUEJIN: ${GB32960_PLATFORM_PWD_YUEJIN:-}")
|
||||
.contains("GB32960_PLATFORM_IP_YUEJIN: ${GB32960_PLATFORM_IP_YUEJIN:-}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void vehicleHistoryOnlyExposesBaseConsumerGroupEnvironmentName() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String historyService = serviceBlock(compose, "vehicle-history-app");
|
||||
|
||||
assertThat(historyService)
|
||||
.contains("KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}")
|
||||
.doesNotContain("KAFKA_GROUP_HISTORY_GB32960_EVENT")
|
||||
.doesNotContain("KAFKA_GROUP_HISTORY_JT808_EVENT")
|
||||
.doesNotContain("KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT")
|
||||
.doesNotContain("KAFKA_GROUP_HISTORY_GB32960_RAW")
|
||||
.doesNotContain("KAFKA_GROUP_HISTORY_JT808_RAW")
|
||||
.doesNotContain("KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW");
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaConsumerEnvironmentIsScopedToConsumerAppsOnly() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String commonEnv = commonEnvBlock(compose);
|
||||
String ingestServices = serviceBlock(compose, "gb32960-ingest-app")
|
||||
+ serviceBlock(compose, "jt808-ingest-app")
|
||||
+ serviceBlock(compose, "yutong-mqtt-app");
|
||||
String historyService = serviceBlock(compose, "vehicle-history-app");
|
||||
String analyticsService = serviceBlock(compose, "vehicle-analytics-app");
|
||||
|
||||
assertThat(commonEnv)
|
||||
.doesNotContain("KAFKA_CONSUMER_ENABLED")
|
||||
.doesNotContain("KAFKA_CONSUMER_MAX_POLL_INTERVAL");
|
||||
assertThat(ingestServices)
|
||||
.doesNotContain("KAFKA_CONSUMER_");
|
||||
assertThat(historyService)
|
||||
.contains("KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_HISTORY:-true}")
|
||||
.contains("KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_HISTORY:-1800000}")
|
||||
.doesNotContain("KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS");
|
||||
assertThat(analyticsService)
|
||||
.contains("KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_ANALYTICS:-true}")
|
||||
.contains("KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_ANALYTICS:-900000}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaTopicEnvironmentIsScopedToServicesThatUseIt() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String commonEnv = commonEnvBlock(compose);
|
||||
String gb32960Service = serviceBlock(compose, "gb32960-ingest-app");
|
||||
String jt808Service = serviceBlock(compose, "jt808-ingest-app");
|
||||
String mqttService = serviceBlock(compose, "yutong-mqtt-app");
|
||||
String historyService = serviceBlock(compose, "vehicle-history-app");
|
||||
String analyticsService = serviceBlock(compose, "vehicle-analytics-app");
|
||||
|
||||
assertThat(commonEnv).doesNotContain("KAFKA_TOPIC_");
|
||||
assertThat(gb32960Service)
|
||||
.contains("KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}")
|
||||
.contains("KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}")
|
||||
.contains("KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}")
|
||||
.doesNotContain("KAFKA_TOPIC_JT808")
|
||||
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT");
|
||||
assertThat(jt808Service)
|
||||
.contains("KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}")
|
||||
.contains("KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}")
|
||||
.contains("KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}")
|
||||
.doesNotContain("KAFKA_TOPIC_GB32960")
|
||||
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT");
|
||||
assertThat(mqttService)
|
||||
.contains("KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}")
|
||||
.contains("KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}")
|
||||
.contains("KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}")
|
||||
.doesNotContain("KAFKA_TOPIC_GB32960")
|
||||
.doesNotContain("KAFKA_TOPIC_JT808");
|
||||
assertThat(historyService)
|
||||
.contains("KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}")
|
||||
.contains("KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}")
|
||||
.contains("KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}")
|
||||
.contains("KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}")
|
||||
.contains("KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}")
|
||||
.contains("KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}")
|
||||
.contains("KAFKA_TOPIC_HISTORY_DLQ: ${KAFKA_TOPIC_HISTORY_DLQ:-vehicle.dlq.history.v1}")
|
||||
.doesNotContain("KAFKA_TOPIC_GB32960_DLQ")
|
||||
.doesNotContain("KAFKA_TOPIC_JT808_DLQ")
|
||||
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT_DLQ");
|
||||
assertThat(analyticsService)
|
||||
.contains("KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}")
|
||||
.contains("KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}")
|
||||
.doesNotContain("KAFKA_TOPIC_GB32960")
|
||||
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT")
|
||||
.doesNotContain("KAFKA_TOPIC_JT808_RAW");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -44,6 +287,7 @@ class PortainerComposeResourceLimitsTest {
|
||||
assertThat(compose)
|
||||
.contains("TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}")
|
||||
.contains("TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}")
|
||||
.doesNotContain("TDENGINE_TELEMETRY_FIELDS_ENABLED")
|
||||
.doesNotContain("\n TDENGINE_ENABLED:")
|
||||
.doesNotContain("\n TDENGINE_DATABASE:");
|
||||
}
|
||||
@@ -62,17 +306,22 @@ class PortainerComposeResourceLimitsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void localLaunchctlHistoryRuntimeDoesNotRequireSharedArchiveRoot() throws IOException {
|
||||
Path launchctl = repositoryRoot().resolve("deploy/local/launchctl");
|
||||
String historyTemplate = Files.readString(launchctl.resolve("com.lingniu.vehicle-history.plist.template"));
|
||||
String readme = Files.readString(launchctl.resolve("README.md"));
|
||||
void portainerIngestAppsWriteColdArchiveToMountedDataVolume() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String gb32960Service = serviceBlock(compose, "gb32960-ingest-app");
|
||||
String jt808Service = serviceBlock(compose, "jt808-ingest-app");
|
||||
String mqttService = serviceBlock(compose, "yutong-mqtt-app");
|
||||
|
||||
assertThat(historyTemplate)
|
||||
.doesNotContain("SINK_ARCHIVE_PATH")
|
||||
.doesNotContain("__SHARED_ARCHIVE_PATH__");
|
||||
assertThat(readme)
|
||||
.doesNotContain("三个服务必须使用同一个 `SINK_ARCHIVE_PATH`")
|
||||
.doesNotContain("history 服务按同一个 archive root 读取原始帧");
|
||||
assertThat(gb32960Service)
|
||||
.contains("SINK_ARCHIVE_PATH: ${GB32960_ARCHIVE_PATH:-/data/archive}")
|
||||
.contains("- gb32960-ingest-data:/data");
|
||||
assertThat(jt808Service)
|
||||
.contains("SINK_ARCHIVE_PATH: ${JT808_ARCHIVE_PATH:-/data/archive}")
|
||||
.contains("- jt808-ingest-data:/data");
|
||||
assertThat(mqttService)
|
||||
.contains("SINK_ARCHIVE_PATH: ${YUTONG_MQTT_ARCHIVE_PATH:-/data/archive}")
|
||||
.contains("- yutong-mqtt-data:/data");
|
||||
assertThat(compose).contains("\n yutong-mqtt-data:\n");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -83,9 +332,15 @@ class PortainerComposeResourceLimitsTest {
|
||||
|
||||
assertThat(tdengineVerification)
|
||||
.contains("TDengine `raw_frames`")
|
||||
.doesNotContain("20482")
|
||||
.doesNotContain("三个服务必须共用同一个 `SINK_ARCHIVE_PATH`")
|
||||
.doesNotContain("再回读共享 archive")
|
||||
.doesNotContain("读取共享 archive 中的原始 `.bin`");
|
||||
.doesNotContain("读取共享 archive 中的原始 `.bin`")
|
||||
.doesNotContain("EVENT_FILE_STORE_ENABLED")
|
||||
.doesNotContain("EVENT_FILE_STORE_PATH")
|
||||
.doesNotContain("TDENGINE_TELEMETRY_FIELDS_ENABLED")
|
||||
.doesNotContain("/api/event-history/telemetry/fields' 会存在")
|
||||
.doesNotContain("/api/event-history/records");
|
||||
assertThat(splitRunbook)
|
||||
.contains("TDengine `raw_frames`")
|
||||
.doesNotContain("must share the same `SINK_ARCHIVE_PATH`")
|
||||
@@ -93,6 +348,84 @@ class PortainerComposeResourceLimitsTest {
|
||||
.doesNotContain("shared volume mounted to all three services");
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitRunbookDocumentsAllActiveProductionAppsAndTopics() throws IOException {
|
||||
String splitRunbook = Files.readString(repositoryRoot()
|
||||
.resolve("docs/operations/gb32960-service-split-runbook.md"));
|
||||
String activeBuildCommand =
|
||||
"mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true";
|
||||
|
||||
assertThat(splitRunbook)
|
||||
.contains("| Yutong MQTT ingest | `:yutong-mqtt-app`")
|
||||
.contains("`vehicle.raw.mqtt-yutong.v1`")
|
||||
.contains("`vehicle.event.mqtt-yutong.v1`")
|
||||
.contains("`vehicle.dlq.mqtt-yutong.v1`")
|
||||
.contains("`vehicle.dlq.history.v1`")
|
||||
.contains(activeBuildCommand)
|
||||
.contains("modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar")
|
||||
.doesNotContain("20482")
|
||||
.doesNotContain("bootstrap-all")
|
||||
.doesNotContain("mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app")
|
||||
.doesNotContain("mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:vehicle-history-app,:vehicle-analytics-app")
|
||||
.doesNotContain("repackaged all three app jars");
|
||||
}
|
||||
|
||||
@Test
|
||||
void tdengineVerificationRunbookDocumentsAllActiveProductionApps() throws IOException {
|
||||
String runbook = Files.readString(repositoryRoot()
|
||||
.resolve("docs/operations/vehicle-ingest-tdengine-verification.md"));
|
||||
String activeBuildCommand =
|
||||
"mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true";
|
||||
|
||||
assertThat(runbook)
|
||||
.contains(activeBuildCommand)
|
||||
.contains("modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar")
|
||||
.contains("modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar")
|
||||
.contains("modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar")
|
||||
.contains("modules/apps/vehicle-history-app/target/vehicle-history-app.jar")
|
||||
.contains("modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar")
|
||||
.contains("vehicle.raw.mqtt-yutong.v1")
|
||||
.contains("vehicle.event.mqtt-yutong.v1")
|
||||
.contains("vehicle-history-yutong-mqtt-event")
|
||||
.contains("vehicle-history-yutong-mqtt-raw")
|
||||
.contains("vehicle-analytics-app")
|
||||
.doesNotContain("mvn -pl modules/apps/jt808-ingest-app,modules/apps/gb32960-ingest-app,modules/apps/vehicle-history-app")
|
||||
.doesNotContain("健康检查已确认三个服务");
|
||||
}
|
||||
|
||||
@Test
|
||||
void architectureDocsNameKafkaInsteadOfGenericMq() throws IOException {
|
||||
String decisions = Files.readString(repositoryRoot().resolve("DECISIONS.md"));
|
||||
String dataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
|
||||
|
||||
assertThat(decisions)
|
||||
.contains("生产链路只支持 Kafka")
|
||||
.doesNotContain("MQ 选型")
|
||||
.doesNotContain("RocketMQ");
|
||||
assertThat(dataFlow)
|
||||
.contains("participant Kafka as sink-kafka")
|
||||
.doesNotContain("participant MQ")
|
||||
.doesNotContain("->>MQ")
|
||||
.doesNotContain("MQ->>");
|
||||
}
|
||||
|
||||
@Test
|
||||
void activeSpringBootAppsDoNotScanWholeIngestPackage() throws IOException {
|
||||
Path root = repositoryRoot();
|
||||
List<Path> activeApps = List.of(
|
||||
root.resolve("modules/apps/gb32960-ingest-app/src/main/java/com/lingniu/ingest/gb32960app/Gb32960IngestApplication.java"),
|
||||
root.resolve("modules/apps/jt808-ingest-app/src/main/java/com/lingniu/ingest/jt808app/Jt808IngestApplication.java"),
|
||||
root.resolve("modules/apps/yutong-mqtt-app/src/main/java/com/lingniu/ingest/yutongmqttapp/YutongMqttApplication.java"),
|
||||
root.resolve("modules/apps/vehicle-history-app/src/main/java/com/lingniu/ingest/historyapp/VehicleHistoryApplication.java"),
|
||||
root.resolve("modules/apps/vehicle-analytics-app/src/main/java/com/lingniu/ingest/analyticsapp/VehicleAnalyticsApplication.java"));
|
||||
|
||||
for (Path app : activeApps) {
|
||||
assertThat(Files.readString(app))
|
||||
.contains("@SpringBootApplication")
|
||||
.doesNotContain("scanBasePackages = \"com.lingniu.ingest\"");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitRunbookKeepsAnalyticsAsJt808MetricRuntimeOnly() throws IOException {
|
||||
String splitRunbook = Files.readString(repositoryRoot()
|
||||
@@ -108,7 +441,7 @@ class PortainerComposeResourceLimitsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitRunbookUsesTdengineHistoryVerificationInsteadOfEventFileStore() throws IOException {
|
||||
void splitRunbookUsesTdengineHistoryVerificationInsteadOfLegacyRecordApi() throws IOException {
|
||||
String splitRunbook = Files.readString(repositoryRoot()
|
||||
.resolve("docs/operations/gb32960-service-split-runbook.md"));
|
||||
|
||||
@@ -135,6 +468,24 @@ class PortainerComposeResourceLimitsTest {
|
||||
.doesNotContain("VEHICLE_STATE_ENABLED");
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaConsumerAppsDoNotDependOnProtocolIngestContainers() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String historyService = serviceBlock(compose, "vehicle-history-app");
|
||||
String analyticsService = serviceBlock(compose, "vehicle-analytics-app");
|
||||
|
||||
assertThat(historyService)
|
||||
.doesNotContain(" depends_on:")
|
||||
.doesNotContain("- gb32960-ingest-app")
|
||||
.doesNotContain("- jt808-ingest-app")
|
||||
.doesNotContain("- yutong-mqtt-app");
|
||||
assertThat(analyticsService)
|
||||
.doesNotContain(" depends_on:")
|
||||
.doesNotContain("- gb32960-ingest-app")
|
||||
.doesNotContain("- jt808-ingest-app")
|
||||
.doesNotContain("- yutong-mqtt-app");
|
||||
}
|
||||
|
||||
@Test
|
||||
void vehicleAnalyticsDoesNotInheritRedisStateEnvironment() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
@@ -150,6 +501,80 @@ class PortainerComposeResourceLimitsTest {
|
||||
.doesNotContain("REDIS_PASSWORD");
|
||||
}
|
||||
|
||||
@Test
|
||||
void mysqlEnvironmentIsScopedToIdentityAndAnalyticsOnly() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String commonEnv = commonEnvBlock(compose);
|
||||
String identityEnv = blockBetween(compose, "x-identity-mysql-env: &identity-mysql-env\n",
|
||||
"\nx-redis-session-env:");
|
||||
String ingestServices = serviceBlock(compose, "gb32960-ingest-app")
|
||||
+ serviceBlock(compose, "jt808-ingest-app")
|
||||
+ serviceBlock(compose, "yutong-mqtt-app");
|
||||
String analyticsService = serviceBlock(compose, "vehicle-analytics-app");
|
||||
|
||||
assertThat(commonEnv)
|
||||
.doesNotContain("MYSQL_JDBC_URL")
|
||||
.doesNotContain("MYSQL_USERNAME")
|
||||
.doesNotContain("MYSQL_PASSWORD");
|
||||
assertThat(identityEnv)
|
||||
.contains("VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}")
|
||||
.contains("VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}")
|
||||
.contains("VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}");
|
||||
assertThat(ingestServices)
|
||||
.doesNotContain("\n MYSQL_JDBC_URL:")
|
||||
.doesNotContain("\n MYSQL_USERNAME:")
|
||||
.doesNotContain("\n MYSQL_PASSWORD:");
|
||||
assertThat(analyticsService)
|
||||
.contains("MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}")
|
||||
.contains("MYSQL_USERNAME: ${MYSQL_USERNAME:-}")
|
||||
.contains("MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void ingestAppsReuseSharedIdentityAndRedisEnvironmentAnchors() throws IOException {
|
||||
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
|
||||
String gb32960Service = serviceBlock(compose, "gb32960-ingest-app");
|
||||
String jt808Service = serviceBlock(compose, "jt808-ingest-app");
|
||||
String mqttService = serviceBlock(compose, "yutong-mqtt-app");
|
||||
|
||||
assertThat(compose)
|
||||
.contains("x-identity-mysql-env: &identity-mysql-env")
|
||||
.contains("x-redis-session-env: &redis-session-env")
|
||||
.doesNotContain("VEHICLE_IDENTITY_STORE");
|
||||
assertThat(gb32960Service)
|
||||
.contains("- *identity-mysql-env")
|
||||
.contains("- *redis-session-env");
|
||||
assertThat(jt808Service)
|
||||
.contains("- *identity-mysql-env")
|
||||
.contains("- *redis-session-env");
|
||||
assertThat(mqttService)
|
||||
.contains("- *identity-mysql-env")
|
||||
.doesNotContain("- *redis-session-env");
|
||||
|
||||
assertThat(gb32960Service + jt808Service + mqttService)
|
||||
.doesNotContain("VEHICLE_IDENTITY_MYSQL_JDBC_URL")
|
||||
.doesNotContain("VEHICLE_IDENTITY_MYSQL_USERNAME")
|
||||
.doesNotContain("VEHICLE_IDENTITY_MYSQL_PASSWORD")
|
||||
.doesNotContain("REDIS_HOST")
|
||||
.doesNotContain("REDIS_PORT")
|
||||
.doesNotContain("REDIS_DATABASE")
|
||||
.doesNotContain("REDIS_USERNAME")
|
||||
.doesNotContain("REDIS_PASSWORD");
|
||||
}
|
||||
|
||||
@Test
|
||||
void moduleDataFlowShowsSplitProductionAppsInsteadOfDeletedBootstrapAll() throws IOException {
|
||||
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
|
||||
|
||||
assertThat(moduleDataFlow)
|
||||
.contains("gb32960-ingest-app")
|
||||
.contains("jt808-ingest-app")
|
||||
.contains("yutong-mqtt-app")
|
||||
.contains("vehicle-history-app")
|
||||
.contains("vehicle-analytics-app")
|
||||
.doesNotContain("bootstrap-all");
|
||||
}
|
||||
|
||||
private static void assertServiceMemoryLimit(String compose,
|
||||
String serviceName,
|
||||
String envName,
|
||||
@@ -160,6 +585,18 @@ class PortainerComposeResourceLimitsTest {
|
||||
.contains(" mem_limit: ${" + envName + ":-" + defaultLimit + "}");
|
||||
}
|
||||
|
||||
private static void assertServiceReadinessHealthcheck(String compose, String serviceName, String httpPort) {
|
||||
assertThat(serviceBlock(compose, serviceName))
|
||||
.contains(" healthcheck:\n")
|
||||
.contains(" <<: *readiness-healthcheck\n")
|
||||
.contains(" test: [\"CMD-SHELL\", \"curl -fsS http://127.0.0.1:"
|
||||
+ httpPort + "/actuator/health/readiness >/dev/null\"]")
|
||||
.doesNotContain(" interval: 30s\n")
|
||||
.doesNotContain(" timeout: 5s\n")
|
||||
.doesNotContain(" retries: 3\n")
|
||||
.doesNotContain(" start_period: 60s\n");
|
||||
}
|
||||
|
||||
private static String serviceBlock(String compose, String serviceName) {
|
||||
String marker = "\n " + serviceName + ":\n";
|
||||
int start = compose.indexOf(marker);
|
||||
@@ -189,13 +626,46 @@ class PortainerComposeResourceLimitsTest {
|
||||
if (start < 0) {
|
||||
throw new IllegalArgumentException("common env block not found");
|
||||
}
|
||||
int end = compose.indexOf("\nservices:\n", start);
|
||||
int end = compose.indexOf("\nx-identity-mysql-env:", start);
|
||||
if (end < 0) {
|
||||
end = compose.indexOf("\nservices:\n", start);
|
||||
}
|
||||
if (end < 0) {
|
||||
throw new IllegalArgumentException("services block not found");
|
||||
}
|
||||
return compose.substring(start, end);
|
||||
}
|
||||
|
||||
private static String blockBetween(String text, String startMarker, String endMarker) {
|
||||
int start = text.indexOf(startMarker);
|
||||
if (start < 0) {
|
||||
throw new IllegalArgumentException("block start not found: " + startMarker);
|
||||
}
|
||||
int end = text.indexOf(endMarker, start + startMarker.length());
|
||||
if (end < 0) {
|
||||
throw new IllegalArgumentException("block end not found: " + endMarker);
|
||||
}
|
||||
return text.substring(start, end);
|
||||
}
|
||||
|
||||
private static int countOccurrences(String text, String needle) {
|
||||
int count = 0;
|
||||
int index = 0;
|
||||
while ((index = text.indexOf(needle, index)) >= 0) {
|
||||
count++;
|
||||
index += needle.length();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private static Properties yamlProperties(Path path) {
|
||||
YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean();
|
||||
factory.setResources(new FileSystemResource(path));
|
||||
Properties properties = factory.getObject();
|
||||
assertThat(properties).isNotNull();
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static Path repositoryRoot() {
|
||||
Path current = Path.of(System.getProperty("user.dir")).toAbsolutePath();
|
||||
while (current != null) {
|
||||
|
||||
@@ -3,7 +3,6 @@ package com.lingniu.ingest.historyapp;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||
import com.lingniu.ingest.eventfilestore.EventFileStore;
|
||||
import com.lingniu.ingest.eventhistory.EventHistoryEnvelopeIngestor;
|
||||
import com.lingniu.ingest.eventhistory.Gb32960DecodedFrameService;
|
||||
import com.lingniu.ingest.eventhistory.Gb32960FrameController;
|
||||
@@ -11,22 +10,16 @@ import com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration;
|
||||
import com.lingniu.ingest.eventhistory.Jt808LocationHistoryController;
|
||||
import com.lingniu.ingest.eventhistory.LocationHistoryController;
|
||||
import com.lingniu.ingest.eventhistory.RawFrameHistoryController;
|
||||
import com.lingniu.ingest.eventhistory.TelemetryFieldHistoryController;
|
||||
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
|
||||
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqProperties;
|
||||
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
|
||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
||||
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
|
||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;
|
||||
import com.lingniu.ingest.tdenginehistory.config.TdengineHistoryAutoConfiguration;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -37,16 +30,12 @@ import org.springframework.context.annotation.Bean;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
class VehicleHistoryAppCompositionTest {
|
||||
|
||||
@@ -75,8 +64,8 @@ class VehicleHistoryAppCompositionTest {
|
||||
() -> processor("event-history"))
|
||||
.withBean("eventHistoryRawEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
|
||||
() -> processor("event-history-raw"))
|
||||
.withBean(SinkMqProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
|
||||
.withPropertyValues("lingniu.ingest.sink.mq.consumer.enabled=true")
|
||||
.withBean(KafkaSinkProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
|
||||
.withPropertyValues("lingniu.ingest.sink.kafka.consumer.enabled=true")
|
||||
.run(context -> {
|
||||
assertThat(context).hasFailed();
|
||||
assertThat(context.getStartupFailure())
|
||||
@@ -85,12 +74,11 @@ class VehicleHistoryAppCompositionTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void createsTdengineHistoryStorageAndQueryBeansWithoutGb32960TcpServerEventFileStoreOrLocalArchiveSink() {
|
||||
void createsTdengineHistoryStorageAndQueryBeansWithoutGb32960TcpServerOrLocalArchiveSink() {
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
TdengineHistoryAutoConfiguration.class,
|
||||
SinkMqAutoConfiguration.class,
|
||||
Gb32960AutoConfiguration.class,
|
||||
KafkaSinkAutoConfiguration.class,
|
||||
EventHistoryAutoConfiguration.class))
|
||||
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
|
||||
.withAllowBeanDefinitionOverriding(true)
|
||||
@@ -99,12 +87,9 @@ class VehicleHistoryAppCompositionTest {
|
||||
"lingniu.ingest.tdengine-history.enabled=true",
|
||||
"lingniu.ingest.tdengine-history.database=vehicle_history_test",
|
||||
"lingniu.ingest.event-history.enabled=true",
|
||||
"lingniu.ingest.gb32960.enabled=true",
|
||||
"lingniu.ingest.gb32960.server.enabled=false",
|
||||
"lingniu.ingest.sink.mq.enabled=true",
|
||||
"lingniu.ingest.sink.mq.type=kafka",
|
||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=false")
|
||||
.run(context -> {
|
||||
@@ -121,7 +106,7 @@ class VehicleHistoryAppCompositionTest {
|
||||
assertThat(context).hasSingleBean(Gb32960DecodedFrameService.class);
|
||||
assertThat(context).hasSingleBean(LocationHistoryController.class);
|
||||
assertThat(context).hasSingleBean(RawFrameHistoryController.class);
|
||||
assertThat(context).hasSingleBean(TelemetryFieldHistoryController.class);
|
||||
assertThat(context).doesNotHaveBean("telemetryFieldHistoryController");
|
||||
assertThat(context).doesNotHaveBean(Gb32960FrameController.class);
|
||||
assertThat(context).doesNotHaveBean(Jt808LocationHistoryController.class);
|
||||
assertThat(context).doesNotHaveBean(Gb32960NettyServer.class);
|
||||
@@ -129,7 +114,7 @@ class VehicleHistoryAppCompositionTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void createsHistoryIngestorWithoutEventFileStoreForTdengineOnlyRuntime() {
|
||||
void createsHistoryIngestorForTdengineOnlyRuntime() {
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(EventHistoryAutoConfiguration.class))
|
||||
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
|
||||
@@ -137,19 +122,38 @@ class VehicleHistoryAppCompositionTest {
|
||||
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.event-history.enabled=true",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false")
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
||||
assertThat(context).doesNotHaveBean("eventFileStore");
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void applicationScanDoesNotExposeTelemetryFieldControllerByDefault() {
|
||||
new ApplicationContextRunner()
|
||||
.withUserConfiguration(VehicleHistoryApplication.class)
|
||||
.withBean(TdengineHistoryReader.class, () -> mock(TdengineHistoryReader.class))
|
||||
.withPropertyValues(
|
||||
"spring.cloud.nacos.config.enabled=false",
|
||||
"spring.config.import=",
|
||||
"lingniu.ingest.event-history.enabled=true",
|
||||
"lingniu.ingest.tdengine-history.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(LocationHistoryController.class);
|
||||
assertThat(context).hasSingleBean(RawFrameHistoryController.class);
|
||||
assertThat(context).doesNotHaveBean("telemetryFieldHistoryController");
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void createsKafkaConsumerRunnerWhenTdengineHistoryRuntimeConsumesKafka() {
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
TdengineHistoryAutoConfiguration.class,
|
||||
SinkMqAutoConfiguration.class,
|
||||
KafkaSinkAutoConfiguration.class,
|
||||
EventHistoryAutoConfiguration.class))
|
||||
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
|
||||
.withAllowBeanDefinitionOverriding(true)
|
||||
@@ -158,13 +162,12 @@ class VehicleHistoryAppCompositionTest {
|
||||
"lingniu.ingest.tdengine-history.enabled=true",
|
||||
"lingniu.ingest.tdengine-history.database=vehicle_history_test",
|
||||
"lingniu.ingest.event-history.enabled=true",
|
||||
"lingniu.ingest.sink.mq.enabled=true",
|
||||
"lingniu.ingest.sink.mq.type=kafka",
|
||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=true",
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
|
||||
"lingniu.ingest.sink.kafka.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
|
||||
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
||||
assertThat(context.getBeanNamesForType(EnvelopeConsumerProcessor.class))
|
||||
@@ -173,32 +176,6 @@ class VehicleHistoryAppCompositionTest {
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyIngestorIgnoresEventFileStoreBeanAndWritesTdengineOnly() {
|
||||
EventFileStore eventFileStore = mock(EventFileStore.class);
|
||||
CapturingTdengineWriter writer = new CapturingTdengineWriter();
|
||||
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(EventHistoryAutoConfiguration.class))
|
||||
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
|
||||
.withBean(EventFileStore.class, () -> eventFileStore)
|
||||
.withBean(TdengineHistoryWriter.class, () -> writer)
|
||||
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.event-history.enabled=true",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false")
|
||||
.run(context -> {
|
||||
EventHistoryEnvelopeIngestor ingestor = context.getBean(EventHistoryEnvelopeIngestor.class);
|
||||
|
||||
ingestor.tryIngest(rawFrameEnvelope().toByteArray());
|
||||
|
||||
verify(eventFileStore, never()).appendAll(any());
|
||||
assertThat(writer.rawFrames)
|
||||
.extracting(TdengineRawFrameRow::frameId)
|
||||
.containsExactly("raw-frame-1");
|
||||
});
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static KafkaProducer<String, byte[]> kafkaProducer() {
|
||||
return mock(KafkaProducer.class);
|
||||
@@ -211,8 +188,8 @@ class VehicleHistoryAppCompositionTest {
|
||||
ignored -> {});
|
||||
}
|
||||
|
||||
private static SinkMqProperties genericOnlyHistoryConsumerProps() {
|
||||
SinkMqProperties props = new SinkMqProperties();
|
||||
private static KafkaSinkProperties genericOnlyHistoryConsumerProps() {
|
||||
KafkaSinkProperties props = new KafkaSinkProperties();
|
||||
props.setBootstrapServers("localhost:9092");
|
||||
props.getConsumer().setEnabled(true);
|
||||
props.getConsumer().setBindings(Map.of(
|
||||
@@ -221,50 +198,11 @@ class VehicleHistoryAppCompositionTest {
|
||||
return props;
|
||||
}
|
||||
|
||||
private static SinkMqProperties.Binding binding(String groupId, String topic) {
|
||||
SinkMqProperties.Binding binding = new SinkMqProperties.Binding();
|
||||
private static KafkaSinkProperties.Binding binding(String groupId, String topic) {
|
||||
KafkaSinkProperties.Binding binding = new KafkaSinkProperties.Binding();
|
||||
binding.setGroupId(groupId);
|
||||
binding.setTopics(List.of(topic));
|
||||
return binding;
|
||||
}
|
||||
|
||||
private static VehicleEnvelope rawFrameEnvelope() {
|
||||
return VehicleEnvelope.newBuilder()
|
||||
.setSchemaVersion("1.0")
|
||||
.setEventId("raw-event-1")
|
||||
.setVin("VINRAW001")
|
||||
.setSource("JT808")
|
||||
.setEventTimeMs(1_782_112_400_000L)
|
||||
.setIngestTimeMs(1_782_112_401_000L)
|
||||
.setRawArchive(RawArchiveRef.newBuilder()
|
||||
.setUri("archive://jt808/2026/06/29/raw-frame-1.bin")
|
||||
.setSizeBytes(68))
|
||||
.setRawFrameFact(RawFrameFactPayload.newBuilder()
|
||||
.setFrameId("raw-frame-1")
|
||||
.setVehicleKey("jt808:013800000000")
|
||||
.setVin("VINRAW001")
|
||||
.setPhone("013800000000")
|
||||
.setMessageId(0x0200)
|
||||
.setRawUri("archive://jt808/2026/06/29/raw-frame-1.bin")
|
||||
.setRawSizeBytes(68)
|
||||
.setParseStatus(ParseStatusProto.PARSE_STATUS_SUCCEEDED))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static final class CapturingTdengineWriter implements TdengineHistoryWriter {
|
||||
private final List<TdengineRawFrameRow> rawFrames = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public void appendRawFrames(List<TdengineRawFrameRow> rows) {
|
||||
rawFrames.addAll(rows);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void appendLocations(List<com.lingniu.ingest.tdenginehistory.TdengineLocationRow> rows) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void appendTelemetryFields(List<com.lingniu.ingest.tdenginehistory.TdengineTelemetryFieldRow> rows) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user