Compare commits
353 Commits
ec555cd67a
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbb6f3b741 | ||
|
|
42de422935 | ||
|
|
318486b1e9 | ||
|
|
0a024f9a76 | ||
|
|
ea9d9f5d35 | ||
|
|
409f55b5ae | ||
|
|
8def635bf5 | ||
|
|
c211447747 | ||
|
|
1741aacaaa | ||
|
|
27f4ad3468 | ||
|
|
601150b364 | ||
|
|
5b7f386c5a | ||
|
|
f4d9c9ef5a | ||
|
|
96514898eb | ||
|
|
0ef88e1a4c | ||
|
|
0fd4d319f9 | ||
|
|
4e8434c511 | ||
|
|
576644eb65 | ||
|
|
9bd48c5781 | ||
|
|
2214abe9c5 | ||
|
|
1477b997c9 | ||
|
|
4ea8d18972 | ||
|
|
947e645090 | ||
|
|
78b805e3a2 | ||
|
|
a7b24f4c0b | ||
|
|
eb124d929b | ||
|
|
4711287655 | ||
|
|
2c224396c9 | ||
|
|
4f2cded760 | ||
|
|
7ad1a25e8e | ||
|
|
3ee1d6f52b | ||
|
|
6d1d0aa85e | ||
|
|
055373c405 | ||
|
|
e37dd39c89 | ||
|
|
dd73fdbb5b | ||
|
|
fb9ab8525a | ||
|
|
cc89e0d537 | ||
|
|
f3ecf430cc | ||
|
|
bcf7cdae2e | ||
|
|
9045b871d6 | ||
|
|
b0150a5503 | ||
|
|
8f3b337fd6 | ||
|
|
594ab2d9c7 | ||
|
|
229ffcf61f | ||
|
|
593713283c | ||
|
|
6bee0499ae | ||
|
|
59c1c9df0f | ||
|
|
56f4811c68 | ||
|
|
1a53011408 | ||
|
|
c2d70dad86 | ||
|
|
91db6c959e | ||
|
|
7b44f97ddb | ||
|
|
a72977cd79 | ||
|
|
dd95101499 | ||
|
|
d347525a75 | ||
|
|
19008e840c | ||
|
|
c803a19bf5 | ||
|
|
243de5c2b2 | ||
|
|
4a32e2181a | ||
|
|
7e7066dd86 | ||
|
|
e7c8c57296 | ||
|
|
6b55694c43 | ||
|
|
eb48d16181 | ||
|
|
a15de91912 | ||
|
|
796d79dd30 | ||
|
|
5f4f4fd124 | ||
|
|
e0068a750d | ||
|
|
665122b23b | ||
|
|
60ca42e5d5 | ||
|
|
80474eeccb | ||
|
|
17024651c8 | ||
|
|
d8c77d7882 | ||
|
|
b55b075a97 | ||
|
|
bac8864e79 | ||
|
|
5d844701ff | ||
|
|
967981c040 | ||
|
|
1fdbc29e27 | ||
|
|
d148a4e3af | ||
|
|
cde55e1ae6 | ||
|
|
e06534d6a5 | ||
|
|
4eaaa88e9c | ||
|
|
f89ac1c23f | ||
|
|
c8abc721e1 | ||
|
|
c8dac638b2 | ||
|
|
4591919a31 | ||
|
|
6e24bf615d | ||
|
|
96b8474182 | ||
|
|
fe4ed5e88f | ||
|
|
71353a3cb4 | ||
|
|
37844e4425 | ||
|
|
ec4aed3ca9 | ||
|
|
6ac77b2f7d | ||
|
|
5deadd06af | ||
|
|
ccf77b4596 | ||
|
|
f4ce4557c0 | ||
|
|
2e6a982af5 | ||
|
|
68253901cf | ||
|
|
fc81693285 | ||
|
|
61cc15ab07 | ||
|
|
b7bb93de8e | ||
|
|
5971d1a8c3 | ||
|
|
c359a0683e | ||
|
|
bf6f94afbb | ||
|
|
4d9779e088 | ||
|
|
80c051e056 | ||
|
|
53f8f79c3e | ||
|
|
dc56a664f4 | ||
|
|
9cd5c5a35d | ||
|
|
192c5eff00 | ||
|
|
329f106e3c | ||
|
|
15c57c33b9 | ||
|
|
664a0c801b | ||
|
|
4ea4469640 | ||
|
|
a6d0236fc4 | ||
|
|
3161e98eff | ||
|
|
5910d23dd9 | ||
|
|
faeee0c2a8 | ||
|
|
90791d28bd | ||
|
|
3795e43a6a | ||
|
|
aa30a413c5 | ||
|
|
0d77f7777f | ||
|
|
b979d7a020 | ||
|
|
f7bb5b1554 | ||
|
|
cd12a84e51 | ||
|
|
1eed7623d1 | ||
|
|
1cca142550 | ||
|
|
a6784bc8c3 | ||
|
|
eeba8fb754 | ||
|
|
476d5a7a14 | ||
|
|
c3d67e4339 | ||
|
|
68f45ee79c | ||
|
|
a5cb0c9705 | ||
|
|
932b386c69 | ||
|
|
981879136f | ||
|
|
27f9cbf0d2 | ||
|
|
cea8ceeabe | ||
|
|
97ea77ed27 | ||
|
|
85013a0ebe | ||
|
|
57ab57c42b | ||
|
|
28bde233a1 | ||
|
|
4134824caf | ||
|
|
422a24684d | ||
|
|
78c39036cd | ||
|
|
495ded0547 | ||
|
|
252aff984c | ||
|
|
dc481e8653 | ||
|
|
9fb77a1aa2 | ||
|
|
afed6cd4f3 | ||
|
|
fd76bfdf61 | ||
|
|
3185696b6a | ||
|
|
641114696c | ||
|
|
2deea2d072 | ||
|
|
3d9a520c3c | ||
|
|
e81d8dcab5 | ||
|
|
a5b39e22b6 | ||
|
|
bc60017c30 | ||
|
|
5ff7cfb9bb | ||
|
|
59edc7a8d4 | ||
|
|
a76cd3a8d6 | ||
|
|
2d65a2555f | ||
|
|
5ede13e6d4 | ||
|
|
5ec3be97aa | ||
|
|
12423efd64 | ||
|
|
5beb81f8cd | ||
|
|
77d5aa4ef7 | ||
|
|
c6347c50a4 | ||
|
|
d4c54b5d0f | ||
|
|
ebbf2ca8fb | ||
|
|
61ad4a2439 | ||
|
|
986fc2fac3 | ||
|
|
7ce64e9963 | ||
|
|
e80fa6f38c | ||
|
|
849e8f8a8b | ||
|
|
303b2eb47e | ||
|
|
e2b6ab63d6 | ||
|
|
87830fee09 | ||
|
|
53a9b3b3b7 | ||
|
|
163f313066 | ||
|
|
7b0b69295f | ||
|
|
74e01acc7a | ||
|
|
418aefc189 | ||
|
|
da17acb09a | ||
|
|
136b8a040c | ||
|
|
0236884f81 | ||
|
|
bb051391c2 | ||
|
|
ce93236a5a | ||
|
|
ba1ec9a012 | ||
|
|
072fa00ea9 | ||
|
|
74fbdd2b1a | ||
|
|
d9329f4364 | ||
|
|
bdc035185e | ||
|
|
d491f543b5 | ||
|
|
5d082d50d5 | ||
|
|
6faf3533a5 | ||
|
|
ce3f2ffb90 | ||
|
|
42e8742b10 | ||
|
|
a321b6351d | ||
|
|
7c51c83220 | ||
|
|
9db8b15ae3 | ||
|
|
2ab1eeb1f8 | ||
|
|
dcd5f2a1a2 | ||
|
|
625d09c856 | ||
|
|
f80c47df0f | ||
|
|
a08a132a3a | ||
|
|
bb6ffa86fd | ||
|
|
62ef3f5bc1 | ||
|
|
8c61ae7139 | ||
|
|
37902be034 | ||
|
|
e745419f76 | ||
|
|
80d3afa80a | ||
|
|
59e8315fe6 | ||
|
|
56a6c01f95 | ||
|
|
51b1dd4174 | ||
|
|
ba4bb51a07 | ||
|
|
0987bb4718 | ||
|
|
4f83cd2792 | ||
|
|
e47e3eebb4 | ||
|
|
cf4d369335 | ||
|
|
9bd61ac91b | ||
|
|
18aa32a721 | ||
|
|
40dea58278 | ||
|
|
e03d1d7629 | ||
|
|
6ccf8fdc26 | ||
|
|
442898cc0b | ||
|
|
ef1fd6e4f5 | ||
|
|
7c3f25eaf6 | ||
|
|
d8f8235544 | ||
|
|
be2daf986a | ||
|
|
3e19c72915 | ||
|
|
3d4676adf3 | ||
|
|
6bc36eb5a8 | ||
|
|
f91fa21463 | ||
|
|
56149c884e | ||
|
|
de97d072d4 | ||
|
|
25d762b52d | ||
|
|
e67c3e8b71 | ||
|
|
aecccd30c6 | ||
|
|
e269c0ad55 | ||
|
|
6c9459bb64 | ||
|
|
6016e3f99c | ||
|
|
b841141fed | ||
|
|
84a098d84c | ||
|
|
f1f29ddd4e | ||
|
|
848a1647df | ||
|
|
a64b7c5081 | ||
|
|
b8ee90a996 | ||
|
|
02504de021 | ||
|
|
2e54607bd6 | ||
|
|
d5ef061cca | ||
|
|
f276b6f6f0 | ||
|
|
b36f993059 | ||
|
|
e97116202b | ||
|
|
371b89aeb0 | ||
|
|
6ab2f25387 | ||
|
|
99ad73e4c9 | ||
|
|
6bcf46034d | ||
|
|
3908c1e752 | ||
|
|
140b3f3995 | ||
|
|
a3367f1886 | ||
|
|
293d75dc96 | ||
|
|
f889f1400d | ||
|
|
7a245215b8 | ||
|
|
1ce4f80161 | ||
|
|
ee549f5ba2 | ||
|
|
e3dfa9d1c0 | ||
|
|
fe5bc1f532 | ||
|
|
b94780b48c | ||
|
|
a5f5cc94ee | ||
|
|
990c675e16 | ||
|
|
356be9fc98 | ||
|
|
f0c1983867 | ||
|
|
18ebd6c84f | ||
|
|
7081800b84 | ||
|
|
0fc91f512c | ||
|
|
b1190cd7c7 | ||
|
|
c6a647ce36 | ||
|
|
af899d03b7 | ||
|
|
83067eacc3 | ||
|
|
1ffa95c924 | ||
|
|
519f5591e9 | ||
|
|
235cc46bf8 | ||
|
|
cbba617801 | ||
|
|
3cc7ac9669 | ||
|
|
e47e341034 | ||
|
|
b6cb661869 | ||
|
|
68cd3d0d06 | ||
|
|
b977de5519 | ||
|
|
4a6c986bf1 | ||
|
|
f3ec20c26c | ||
|
|
bf6728041b | ||
|
|
1e2e59bddc | ||
|
|
4d683809ad | ||
|
|
1682b3f83f | ||
|
|
6a17134942 | ||
|
|
2a4150adc9 | ||
|
|
dfd58632a7 | ||
|
|
8b7a1ae76b | ||
|
|
b1bd67d4d2 | ||
|
|
33ae76fc1c | ||
|
|
c99fa6cfb9 | ||
|
|
bdc455dcf7 | ||
|
|
12ac1dab3b | ||
|
|
f56de96cec | ||
|
|
272ecef194 | ||
|
|
80e96d8075 | ||
|
|
12de83e37f | ||
|
|
d1748fcc2f | ||
|
|
8804c01d99 | ||
|
|
5c1c61f29b | ||
|
|
9dc667f8da | ||
|
|
bf09f4e9b2 | ||
|
|
705b507f88 | ||
|
|
1d569b7bf3 | ||
|
|
d003757308 | ||
|
|
acf087a3f5 | ||
|
|
9372256940 | ||
|
|
f00d851f2a | ||
|
|
114e707a5b | ||
|
|
7b346420af | ||
|
|
ef9ac327fe | ||
|
|
d2bc4a0d0a | ||
|
|
98b37e71e9 | ||
|
|
57ef82f563 | ||
|
|
6646b9b1e1 | ||
|
|
b244984fcb | ||
|
|
8fe0adcac9 | ||
|
|
00eee9a769 | ||
|
|
06423db8d4 | ||
|
|
7abac30ea4 | ||
|
|
b631d5d998 | ||
|
|
a66cfe1532 | ||
|
|
2273d72e95 | ||
|
|
57ba7dca90 | ||
|
|
192f115176 | ||
|
|
90342fe5bf | ||
|
|
84de9b2dfa | ||
|
|
3904dcf869 | ||
|
|
ab9c669e76 | ||
|
|
8466d622f6 | ||
|
|
e708091f2d | ||
|
|
4de723d0b4 | ||
|
|
f58fe51cf5 | ||
|
|
5a60b8c43c | ||
|
|
feb9552234 | ||
|
|
9c2779f82b | ||
|
|
0ec29a732c | ||
|
|
969a245454 | ||
|
|
b5453ff888 | ||
|
|
aed023043c | ||
|
|
4feb1ec829 | ||
|
|
310ef94a6e | ||
|
|
8d69296b09 | ||
|
|
cd953e7cd7 |
@@ -54,13 +54,16 @@
|
||||
V2016/V2025 双版本信息体 parser、平台登入鉴权、VIN 白名单、读空闲告警。
|
||||
- **JT/T 808 / JT/T 1078 / JSATL12 接入**(`protocol-jt808` / `protocol-jt1078` /
|
||||
`protocol-jsatl12`)。
|
||||
- **MQTT 入站**与**信达平台 push 接入**(`inbound-mqtt` / `inbound-xinda-push`)。
|
||||
- **MQTT 入站**(`inbound-mqtt`)。
|
||||
- **Disruptor 事件总线**(`ingest-core`):高吞吐解耦解码与下游 sink。
|
||||
- **Sink**:本地归档(`sink-archive`)、Kafka(`sink-mq`,含 protobuf envelope)。
|
||||
- **Sink**:本地归档(`sink-archive`)、Kafka(`sink-kafka`,含 protobuf envelope)。
|
||||
- **会话状态**(`session-core`)。
|
||||
- **可观测性**:Micrometer/Actuator 装配(`observability`)。
|
||||
- **终端控制命令网关**(`command-gateway`)。
|
||||
|
||||
### Removed
|
||||
- 旧文件型事件索引模块和 DuckDB 依赖;历史查询统一收敛到 TDengine 与 `archive://...` 原始报文引用。
|
||||
|
||||
### Fixed —— GB/T 32960 应答帧时间字段
|
||||
- `Gb32960MessageDecoder` 对 `VEHICLE_LOGIN` / `VEHICLE_LOGOUT` /
|
||||
`PLATFORM_LOGIN` / `PLATFORM_LOGOUT` 命令预解析 body 首 6B 采集时间,写入
|
||||
|
||||
80
DECISIONS.md
80
DECISIONS.md
@@ -2,11 +2,11 @@
|
||||
|
||||
> 本文件记录 lingniu-vehicle-ingest v2 的关键架构决策,后续每次重大调整追加条目(不删除旧条目)。
|
||||
|
||||
## ADR-001 MQ 选型:Kafka
|
||||
## ADR-001 消息通道:Kafka
|
||||
- **Status**: Accepted
|
||||
- **Context**: 需要高吞吐、严格单车有序、成熟生态
|
||||
- **Decision**: 使用 Kafka,分区 key = VIN
|
||||
- **Consequences**: 下游消费者需使用消费组 + 分区内顺序消费;不使用 RocketMQ 的事务/顺序特性
|
||||
- **Consequences**: 下游消费者需使用消费组 + 分区内顺序消费;生产链路只支持 Kafka。
|
||||
|
||||
## ADR-002 线上消息格式:Protobuf
|
||||
- **Status**: Accepted
|
||||
@@ -20,27 +20,22 @@
|
||||
- **Decision**: 拆出独立 `command-gateway` 模块,复用 `session-core`;本次重构同步交付
|
||||
- **Consequences**: HTTP 对外接口(原 JT808Controller / JT1078Controller)迁移到此模块
|
||||
|
||||
## ADR-004 信达 Push:保留模块但彻底重写
|
||||
- **Status**: Accepted
|
||||
## ADR-004 信达 Push:Removed
|
||||
- **Status**: Superseded by ADR-011
|
||||
- **Context**: 旧实现硬编码凭证、无重连、0300/0401 空实现
|
||||
- **Decision**:
|
||||
1. 新模块 `inbound-xinda-push` 隔离第三方库 `gps-push-client`
|
||||
2. 配置外置(yml + env)
|
||||
3. 自动重连 + Resilience4j 熔断
|
||||
4. 0200/0300/0401 **仅做协议解析**,转换为 `VehicleEvent` 后直接投 Kafka
|
||||
5. **业务处理全部下沉到下游消费者**,本服务不再实现报警/透传业务逻辑
|
||||
- **Consequences**: 旧代码 messageReceived 里的 TODO 不再回到本仓库;需要协调下游消费者团队认领
|
||||
- **Decision**: 删除信达 Push 源码、Maven profile、私仓依赖和部署入口。
|
||||
- **Consequences**: 信达 Push 后续已从源码和构建面删除,优化优先投向 GB32960、JT808、宇通 MQTT、历史和统计链路。
|
||||
|
||||
## ADR-005 JT1078 / JSATL12 进第一批
|
||||
- **Status**: Accepted
|
||||
- **Status**: Superseded by ADR-012
|
||||
- **Decision**: 第一批迁移即覆盖 JT1078 信令 + JSATL12 报警附件上传
|
||||
- **Consequences**: Phase 2 工期相应拉长;JSATL12 需要对象存储后端(本地 / S3 / OSS)
|
||||
|
||||
## ADR-006 部署形态:GB32960 三应用拆分
|
||||
- **Status**: Superseded
|
||||
- **Status**: Superseded by ADR-011
|
||||
- **Context**: 原 `bootstrap-all` 一体化启动已被 GB32960 拆分部署替代
|
||||
- **Decision**: 生产默认部署 `gb32960-ingest-app`、`vehicle-history-app`、`vehicle-analytics-app` 三个应用;旧 `bootstrap-all` 模块删除
|
||||
- **Consequences**: CI/CD 分别构建三个镜像;协议接入、历史查询、统计消费可以独立发布和回滚
|
||||
- **Decision**: 当时生产默认部署 `gb32960-ingest-app`、`vehicle-history-app`、`vehicle-analytics-app`;旧 `bootstrap-all` 模块删除
|
||||
- **Consequences**: 该三应用边界已由 ADR-011 的三协议接入 + 历史 + 统计生产面取代;协议接入、历史查询、统计消费仍保持独立发布和回滚。
|
||||
|
||||
## ADR-007 Java 25 + 虚拟线程 + Disruptor
|
||||
- **Status**: Accepted
|
||||
@@ -51,15 +46,60 @@
|
||||
- 禁用 `synchronized`,使用 `ReentrantLock` 避免虚拟线程 pinning
|
||||
- **Consequences**: 需开启 `jdk.VirtualThreadPinned` JFR 监控
|
||||
|
||||
## ADR-008 本服务不写业务库
|
||||
## ADR-008 协议接入应用不直接写业务库
|
||||
- **Status**: Accepted
|
||||
- **Decision**: 所有业务表(vehicle_data_actual / history / travel / gps_data 等)不再由本服务写入
|
||||
- **Consequences**: 需要新建消费服务 `vehicle-analytics-service` 承接;迁移期采用双跑对账
|
||||
- **Decision**: GB32960、JT808、Yutong MQTT 等协议接入应用只负责收、解析、校验、规整和投递 Kafka,不直接写业务表。
|
||||
- **Consequences**: 业务落库由独立 Kafka 消费应用承接:`vehicle-history-app` 写入 TDengine 历史与 RAW JSON,`vehicle-analytics-app` 写入 MySQL `vehicle_stat_metric`。
|
||||
|
||||
## ADR-009 构建工具:Maven
|
||||
- **Status**: Accepted
|
||||
- **Decision**: 沿用 Maven,统一 BOM 管理版本,Spotless 格式化,ArchUnit 守护分层
|
||||
|
||||
## ADR-010 框架:Spring Boot 3.4
|
||||
## ADR-010 框架:Spring Boot 3.5
|
||||
- **Status**: Accepted
|
||||
- **Decision**: Spring Boot 3.4.x,支持 JDK 25;AutoConfiguration 用于按需启停
|
||||
- **Decision**: Spring Boot 3.5.x,支持 JDK 25;AutoConfiguration 用于按需启停
|
||||
|
||||
## ADR-011 默认生产面:三协议接入 + 历史 + 统计
|
||||
- **Status**: Accepted
|
||||
- **Context**: 生产接入范围已经从 GB32960 三应用拆分,扩展为三条活跃接入链路和两个消费应用;信达 Push 已废弃并删除,不应再出现在构建、部署或优化目标里。
|
||||
- **Decision**:
|
||||
1. 默认生产面包含 GB32960、JT808、Yutong MQTT、vehicle-history-app、vehicle-analytics-app。
|
||||
2. Xinda Push 源码、Maven profile、Woodpecker 镜像发布和历史消费绑定全部移除。
|
||||
3. `command-gateway` 和 JT1078 继续作为可选能力,通过 `optional-command-gateway` profile 显式启用。
|
||||
- **Consequences**: 默认构建和部署保持精简;后续性能、可靠性、字段解析和存储优化都服务三条活跃协议链路。
|
||||
|
||||
## ADR-012 JSATL12 附件上传:Optional only
|
||||
- **Status**: Accepted
|
||||
- **Context**: 当前默认生产面只部署 GB32960、JT808、Yutong MQTT、vehicle-history-app、vehicle-analytics-app;JSATL12 附件上传没有独立生产 app,也不在 Portainer 和 Woodpecker 活跃镜像列表中。
|
||||
- **Decision**:
|
||||
1. `protocol-jsatl12` 不进入默认 Maven reactor。
|
||||
2. JSATL12 仅保留在 `optional-attachments` profile,需要附件上传能力时显式构建。
|
||||
3. 默认优化和验证优先覆盖活跃接入链路,附件上传能力保持源码可用但不增加默认构建面。
|
||||
- **Consequences**: 默认构建更轻,生产部署边界更清晰;附件上传相关测试需要通过 `-Poptional-attachments` 显式运行。
|
||||
|
||||
## ADR-013 最新状态服务:Optional only
|
||||
- **Status**: Accepted
|
||||
- **Context**: Redis 最新状态查询是独立消费能力,但当前默认 Portainer 部署和 Woodpecker 镜像发布只包含三条接入链路、history 和 analytics;`vehicle-state-service` 没有独立 app,也不应增加默认构建面。
|
||||
- **Decision**:
|
||||
1. `vehicle-state-service` 不进入默认 Maven reactor。
|
||||
2. vehicle-state-service 仅保留在 `optional-latest-state` profile,需要 Redis 最新状态查询能力时显式构建。
|
||||
3. 默认优化和验证优先覆盖活跃接入、TDengine 历史和 MySQL 指标链路。
|
||||
- **Consequences**: 默认构建和部署边界继续收窄;最新状态能力保持源码可用,但其测试需要通过 `-Poptional-latest-state` 显式运行。
|
||||
|
||||
## ADR-014 raw-archive-store 原型已删除
|
||||
- **Status**: Accepted
|
||||
- **Context**: 默认生产 raw bytes 写入由 `sink-archive` 负责,历史查询通过 TDengine `raw_frames` 和 `archive://...` 引用追溯;`raw-archive-store` 只是未部署的独立读写原型,继续保留会造成 raw archive 路径歧义。
|
||||
- **Decision**:
|
||||
1. 删除 `raw-archive-store` 模块和对应 optional profile。
|
||||
2. 默认生产链路只保留 `sink-archive`、Kafka raw topic、TDengine raw_frames 这条 raw 路径。
|
||||
3. 如需新的 archive store 能力,先以生产链路需求重新设计,不恢复旧原型。
|
||||
- **Consequences**: 默认构建面和可选构建面继续收窄;raw bytes 写入职责集中在 `sink-archive`。
|
||||
|
||||
## ADR-015 文件型事件索引:Removed
|
||||
- **Status**: Accepted
|
||||
- **Context**: 默认历史查询已收敛到 TDengine `raw_frames`、`vehicle_locations` 和按需解码;旧文件型索引会增加一套无生产部署的查询和依赖边界。
|
||||
- **Decision**:
|
||||
1. 删除旧文件型事件索引模块和 profile。
|
||||
2. 父 POM 不再管理旧索引驱动依赖。
|
||||
3. 历史查询和 RAW 回放统一以 TDengine + `archive://...` 引用为准。
|
||||
- **Consequences**: 默认构建面和可选构建面都更小;需要历史查询时只维护 TDengine 一条路径。
|
||||
|
||||
10
Dockerfile
10
Dockerfile
@@ -5,11 +5,15 @@ ARG APP_VERSION
|
||||
|
||||
ENV APP_NAME=${APP_NAME}
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
ENV JAVA_OPTS=""
|
||||
ENV JAVA_OPTS="--sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true"
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY modules/apps/${APP_NAME}/target/${APP_NAME}.jar /app/app.jar
|
||||
|
||||
EXPOSE 20100 20200 20300 20400 32960 10808
|
||||
EXPOSE 20100 20200 20310 20400 20500 32960 808
|
||||
|
||||
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app/app.jar"]
|
||||
ENTRYPOINT ["sh", "-c", "exec java $JAVA_OPTS -jar /app/app.jar"]
|
||||
|
||||
78
README.md
78
README.md
@@ -4,9 +4,9 @@
|
||||
|
||||
## 设计目标
|
||||
|
||||
- **协议接入统一抽象**:GB/T 32960、JT/T 808、JT/T 1078、JSATL12、MQTT、信达 Push 全部变成可插拔 Inbound Adapter
|
||||
- **协议接入统一抽象**:GB/T 32960、JT/T 808、Yutong MQTT 是默认生产接入;JT/T 1078、JSATL12 作为显式 profile 的可选能力;信达 Push 已废弃并删除源码
|
||||
- **原子能力化**:每个协议一个独立 Maven 模块 + 独立 AutoConfiguration + 独立配置开关
|
||||
- **业务 / 实时彻底解耦**:本服务只负责 **收 → 解析 → 校验 → 规整 → 投递 Kafka**,不碰业务库
|
||||
- **业务 / 实时彻底解耦**:协议接入应用只负责收、解析、校验、规整和投递 Kafka;历史与统计由独立 Kafka 消费应用落库
|
||||
- **高并发低延迟**:Netty + Disruptor + Java 25 虚拟线程;目标单节点 ≥ 5 万 msg/s,P99 < 50 ms
|
||||
- **可观测、可回放、可灰度**:统一 traceId、Envelope、幂等键、DLQ、原始报文冷存
|
||||
|
||||
@@ -15,17 +15,17 @@
|
||||
| 类别 | 选型 |
|
||||
|---|---|
|
||||
| 语言 | Java **25** |
|
||||
| 框架 | Spring Boot 3.4.x |
|
||||
| 网络 | Netty 4.1 |
|
||||
| 并发 | Disruptor 4 + 虚拟线程 + `StructuredTaskScope` |
|
||||
| 消息队列 | **Kafka**(vin 分区,保证单车有序) |
|
||||
| 框架 | Spring Boot 3.5.3 |
|
||||
| 网络 | Netty 4.2.9.Final |
|
||||
| 并发 | Disruptor 4 + 虚拟线程 |
|
||||
| 事件流 | **Kafka**(唯一生产消息通道,vin 分区,保证单车有序) |
|
||||
| 线上消息格式 | **Protobuf**,调试走 JSON |
|
||||
| MQTT 客户端 | HiveMQ MQTT Client |
|
||||
| 会话缓存 | Redis 生产索引 + Caffeine 内存降级 |
|
||||
| MQTT 客户端 | Eclipse Paho 1.2.5 |
|
||||
| 会话索引 | Redis |
|
||||
| 熔断/限流 | Resilience4j 2 |
|
||||
| 可观测 | Micrometer + Prometheus + OpenTelemetry |
|
||||
| 冷存 | S3 / OSS / 本地 |
|
||||
| 构建 | Maven + Spotless + ArchUnit |
|
||||
| 可观测 | Micrometer + Prometheus |
|
||||
| 冷存 | 本地文件系统 |
|
||||
| 构建 | Maven |
|
||||
|
||||
## 模块划分
|
||||
|
||||
@@ -36,31 +36,32 @@ lingniu-vehicle-ingest/
|
||||
│ │ ├── ingest-api/ SPI + sealed 领域事件 + 注解
|
||||
│ │ ├── ingest-codec-common/ 公共编解码工具(BCD/CRC/BCC/bit utils)
|
||||
│ │ ├── ingest-core/ Pipeline / Dispatcher / Disruptor / Session 桥
|
||||
│ │ ├── session-core/ 设备会话 + 鉴权 + Token + Redis/Memory SessionStore
|
||||
│ │ ├── vehicle-identity/ 跨协议车辆身份解析 + 外部标识绑定(memory/file)
|
||||
│ │ └── observability/ metrics / tracing / health
|
||||
│ │ ├── session-core/ 设备会话 + 鉴权 + Token + Redis SessionStore
|
||||
│ │ ├── vehicle-identity/ 跨协议车辆身份解析 + MySQL 外部标识绑定
|
||||
│ │ └── observability/ metrics / health
|
||||
│ ├── protocols/
|
||||
│ │ ├── protocol-gb32960/ GB/T 32960
|
||||
│ │ ├── protocol-jt808/ JT/T 808(统一身份映射 + 事件 metadata 内部 VIN + 注册/鉴权/心跳/注销/位置/批量位置/参数/属性/媒体/透传/未知上行和坏帧兜底/断链清理会话/下行分包)
|
||||
│ │ ├── protocol-jt1078/ JT/T 1078(808 信令按需桥接 + 常用下行信令编码 + TCP/UDP RTP 媒体流分段归档 + 事件 metadata 内部 VIN + 坏 RTP 统一 RawArchive/Passthrough + 归档失败兜底 + SIM 身份映射)
|
||||
│ │ └── protocol-jsatl12/ 苏标主动安全报警附件(真实 DataPacket 分帧 + 附件流归档 + T1210 文件清单身份继承 + 9212 补传应答 + MediaMeta 引用事件 + 坏帧/坏信令/归档失败兜底)
|
||||
│ │ ├── protocol-jt1078/ JT/T 1078(optional-command-gateway profile;808 信令按需桥接 + 常用下行信令编码 + TCP/UDP RTP 媒体流分段归档 + 事件 metadata 内部 VIN + 坏 RTP 统一 RawArchive/Passthrough + 归档失败兜底 + SIM 身份映射)
|
||||
│ │ └── protocol-jsatl12/ 苏标主动安全报警附件(optional-attachments profile)
|
||||
│ ├── inbound/
|
||||
│ │ ├── inbound-mqtt/ MQTT 接入(endpoint 生命周期 + profile 注册扩展 + 统一身份映射 + PEM 双向 TLS + 未知 profile/解析失败/profile异常/连接订阅失败兜底 + 统一 UNKNOWN 身份 metadata)
|
||||
│ │ └── inbound-xinda-push/ 信达 Push 接入(统一身份映射 + 事件 metadata 内部 VIN + 未知业务 cmd/解析失败/启动配置失败/业务处理异常兜底)
|
||||
│ │ └── inbound-mqtt/ MQTT 接入(endpoint 生命周期 + profile 注册扩展 + 统一身份映射 + PEM 双向 TLS + 未知 profile/解析失败/profile异常/连接订阅失败兜底 + 统一 UNKNOWN 身份 metadata)
|
||||
│ ├── sinks/
|
||||
│ │ ├── sink-mq/ Kafka producer + Protobuf Envelope
|
||||
│ │ ├── sink-archive/ 原始报文冷存
|
||||
│ │ └── event-file-store/ Parquet + DuckDB 文件型明细库
|
||||
│ │ ├── sink-kafka/ Kafka producer + Protobuf Envelope
|
||||
│ │ └── sink-archive/ 原始报文冷存
|
||||
│ ├── services/
|
||||
│ │ ├── event-history-service/ Kafka 全字段事件消费 + 历史查询/导出
|
||||
│ │ ├── vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询
|
||||
│ │ ├── event-history-service/ Kafka 全字段事件消费 + 历史查询
|
||||
│ │ ├── vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询(optional-latest-state profile)
|
||||
│ │ └── vehicle-stat-service/ Kafka 全字段事件消费 + 可配置日统计
|
||||
│ ├── testing/
|
||||
│ │ └── vehicle-identity-test-support/ 协议测试共享身份解析夹具
|
||||
│ └── apps/
|
||||
│ ├── command-gateway/ HTTP → 设备下行命令(808 位置/参数/属性/控制/区域删除/报警确认 + 1078 音视频控制)
|
||||
│ ├── command-gateway/ 可选 HTTP → 设备下行命令(optional-command-gateway profile)
|
||||
│ ├── gb32960-ingest-app/ GB32960 TCP 接入 + Kafka 投递
|
||||
│ ├── jt808-ingest-app/ JT808 TCP 接入 + Kafka 投递
|
||||
│ ├── vehicle-history-app/ 历史查询 + 原始归档索引消费
|
||||
│ └── vehicle-analytics-app/ 车辆状态/日统计消费
|
||||
│ ├── yutong-mqtt-app/ 宇通 MQTT 接入 + Kafka 投递
|
||||
│ ├── vehicle-history-app/ TDengine 历史查询 + RAW JSON
|
||||
│ └── vehicle-analytics-app/ JT808 每日里程指标消费
|
||||
├── docs/ 架构文档、模块图、实施计划
|
||||
└── reference/ 参考资料
|
||||
```
|
||||
@@ -70,11 +71,19 @@ lingniu-vehicle-ingest/
|
||||
```bash
|
||||
# 要求:JDK 25, Maven 3.9+
|
||||
mvn -v
|
||||
mvn clean install -DskipTests
|
||||
mvn -pl :gb32960-ingest-app -am spring-boot:run
|
||||
mvn -pl :jt808-ingest-app -am spring-boot:run
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
生产服务只在 ECS/Portainer 上运行;本机只用于源码检查、Maven 构建和单元测试。部署、健康检查和真实流量验证步骤见 `docs/operations/gb32960-service-split-runbook.md` 与 `docs/operations/vehicle-ingest-tdengine-verification.md`。
|
||||
|
||||
信达 Push 已废弃并删除源码,不参与 Maven reactor、CI、Portainer 部署和历史消费链路。
|
||||
|
||||
JSATL12 附件上传不在默认 Maven reactor 中;需要时使用 `-Poptional-attachments` 构建。
|
||||
|
||||
Command Gateway/JT1078 下行与音视频信令不在默认 Maven reactor 中;需要时使用 `-Poptional-command-gateway` 构建。
|
||||
|
||||
Redis 最新状态查询不在默认 Maven reactor 中;需要时使用 `-Poptional-latest-state` 构建 `vehicle-state-service`。
|
||||
|
||||
## 迁移说明
|
||||
|
||||
本项目是 `lingniu-vehicle-data-reception` 的 v2 重构,采用 strangler fig 渐进式迁移,旧项目保留只读参考。迁移路径与决策参见 `../REFRACTOR_PLAN.md`。
|
||||
@@ -87,8 +96,9 @@ mvn -pl :jt808-ingest-app -am spring-boot:run
|
||||
|
||||
## 核心原则
|
||||
|
||||
1. **本服务不写业务库**:历史、行程、在线检测、里程统计全部由 Kafka 下游消费者实现
|
||||
2. **协议即插拔**:每个 `protocol-*` / `inbound-*` 都可独立开关(`lingniu.ingest.<name>.enabled`)
|
||||
3. **顺序保证**:同一 VIN 严格有序(Disruptor hash + Kafka 分区 key)
|
||||
4. **幂等消费**:Envelope 带 `eventId`,下游去重
|
||||
5. **原始可回放**:Dispatcher 会先发布 `RawArchive`,并把同一份 `rawArchiveKey/rawArchiveUri` 追加到业务事件 metadata;下游明细库、Kafka Envelope 和导出都使用 `archive://...` 逻辑 URI 追溯原始 bytes,实际文件由 `sink-archive`/`ArchiveStore` 管理;`event-file-store` 只额外保存 RawArchive 查询索引,不嵌入原始 bytes
|
||||
1. **接入与业务落库分离**:协议接入应用只负责收、解析、校验、规整和投递 Kafka;`vehicle-history-app` 写入 TDengine 历史与 RAW JSON,`vehicle-analytics-app` 将 JT808 `daily_mileage_km` 写入 MySQL `vehicle_stat_metric`
|
||||
2. **Kafka 唯一消息通道**:生产链路只支持 Kafka;接入、历史、统计统一通过 Kafka Envelope 解耦
|
||||
3. **协议即插拔**:每个 `protocol-*` / `inbound-*` 都可独立开关(`lingniu.ingest.<name>.enabled`)
|
||||
4. **顺序保证**:同一 VIN 严格有序(Disruptor hash + Kafka 分区 key)
|
||||
5. **幂等消费**:Envelope 带 `eventId`,下游去重
|
||||
6. **原始可回放**:协议接入会把 `rawArchiveKey/rawArchiveUri` 追加到事件 metadata;Kafka Envelope、TDengine `raw_frames` 和导出都使用 `archive://...` 逻辑 URI 追溯原始 bytes,实际文件由 `sink-archive` 管理
|
||||
|
||||
117
deploy/portainer/docker-compose-go.yml
Normal file
117
deploy/portainer/docker-compose-go.yml
Normal file
@@ -0,0 +1,117 @@
|
||||
version: "3.8"
|
||||
|
||||
# Go sidecar stack for the vehicle ingest redesign.
|
||||
# It is intentionally separate from deploy/portainer/docker-compose.yml so the
|
||||
# current Java production stack can keep running until Go evidence is captured.
|
||||
|
||||
x-common-env: &common-env
|
||||
TZ: Asia/Shanghai
|
||||
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.yutong-mqtt.v1}
|
||||
KAFKA_TOPIC_UNIFIED: ${KAFKA_TOPIC_UNIFIED:-vehicle.event.unified.v1}
|
||||
|
||||
x-restart: &restart-policy
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "100m"
|
||||
max-file: "5"
|
||||
|
||||
services:
|
||||
go-vehicle-gateway:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-vehicle-gateway
|
||||
command: ["/app/gateway"]
|
||||
mem_limit: ${GO_GATEWAY_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
GB32960_TCP_ADDR: ":32960"
|
||||
JT808_TCP_ADDR: ":808"
|
||||
TCP_READ_BUFFER_BYTES: ${TCP_READ_BUFFER_BYTES:-65536}
|
||||
TCP_IDLE_TIMEOUT_SECONDS: ${TCP_IDLE_TIMEOUT_SECONDS:-180}
|
||||
TCP_MAX_CONNECTIONS: ${TCP_MAX_CONNECTIONS:-20000}
|
||||
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
|
||||
YUTONG_MQTT_ENDPOINT: ${YUTONG_MQTT_ENDPOINT:-yutong}
|
||||
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
|
||||
YUTONG_MQTT_TOPICS: ${YUTONG_MQTT_TOPICS:-/ytforward/shln/+}
|
||||
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-2}
|
||||
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt}
|
||||
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
|
||||
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
|
||||
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
|
||||
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
|
||||
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
|
||||
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
|
||||
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
|
||||
IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-}
|
||||
VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding}
|
||||
ports:
|
||||
- "${GO_GB32960_TCP_PORT:-32960}:32960"
|
||||
- "${GO_JT808_TCP_PORT:-808}:808"
|
||||
volumes:
|
||||
- "${KAFKA_SPOOL_HOST_DIR:-/opt/lingniu-go/spool/gateway}:${KAFKA_SPOOL_DIR:-/data/spool/gateway}"
|
||||
- "${YUTONG_MQTT_CERT_HOST_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:${YUTONG_MQTT_CERT_CONTAINER_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:ro"
|
||||
|
||||
go-history-writer:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-history-writer
|
||||
command: ["/app/history-writer"]
|
||||
mem_limit: ${GO_HISTORY_WRITER_MEM_LIMIT:-768m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_HISTORY:-go-history-writer}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_HISTORY:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1}
|
||||
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
|
||||
TDENGINE_DSN: ${TDENGINE_DSN:?set TDENGINE_DSN, example root:password@ws(172.17.111.57:6041)/lingniu_vehicle_ts}
|
||||
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
|
||||
TDENGINE_ENSURE_SCHEMA: ${TDENGINE_ENSURE_SCHEMA:-true}
|
||||
|
||||
go-stat-writer:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-stat-writer
|
||||
command: ["/app/stat-writer"]
|
||||
mem_limit: ${GO_STAT_WRITER_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_STAT:-go-stat-writer}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_STAT:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1}
|
||||
MYSQL_DSN: ${MYSQL_DSN:?set MYSQL_DSN}
|
||||
MYSQL_ENSURE_SCHEMA: ${MYSQL_ENSURE_SCHEMA:-true}
|
||||
LOCAL_TZ: ${LOCAL_TZ:-Asia/Shanghai}
|
||||
|
||||
go-realtime-api:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-realtime-api
|
||||
command: ["/app/realtime-api"]
|
||||
mem_limit: ${GO_REALTIME_API_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
HTTP_ADDR: ":20210"
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_REALTIME:-go-realtime-api}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_REALTIME:-vehicle.event.unified.v1}
|
||||
REDIS_ADDR: ${REDIS_ADDR:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379}
|
||||
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
|
||||
REDIS_DB: ${REDIS_DB:-50}
|
||||
ONLINE_TTL_SECONDS: ${ONLINE_TTL_SECONDS:-600}
|
||||
MYSQL_DSN: ${MYSQL_DSN:-}
|
||||
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
|
||||
TDENGINE_DSN: ${TDENGINE_DSN:-}
|
||||
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
|
||||
ports:
|
||||
- "${GO_REALTIME_HTTP_PORT:-20210}:20210"
|
||||
|
||||
networks:
|
||||
vehicle-ingest:
|
||||
name: vehicle-ingest
|
||||
@@ -1,10 +1,13 @@
|
||||
version: "3.8"
|
||||
|
||||
# Default production stack: GB32960, JT808, Yutong MQTT, history, analytics.
|
||||
# Legacy and optional services stay outside this compose file.
|
||||
# Set LINGNIU_IMAGE_VERSION in Portainer Stack env, for example:
|
||||
# main-0.1.0-SNAPSHOT-22468e7
|
||||
# Override LINGNIU_IMAGE_REGISTRY / LINGNIU_IMAGE_NAMESPACE only when publishing to another registry.
|
||||
x-common-env: &common-env
|
||||
TZ: Asia/Shanghai
|
||||
JAVA_OPTS: ${JAVA_OPTS:--XX:+UseZGC -XX:MaxRAMPercentage=75}
|
||||
JAVA_OPTS: ${JAVA_OPTS:---sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true -XX:+UseZGC -XX:MaxRAMPercentage=75}
|
||||
NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true}
|
||||
NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848}
|
||||
NACOS_NAMESPACE: ${NACOS_NAMESPACE:-}
|
||||
@@ -13,33 +16,52 @@ x-common-env: &common-env
|
||||
NACOS_REFRESH_ENABLED: ${NACOS_REFRESH_ENABLED:-true}
|
||||
NACOS_USERNAME: ${NACOS_USERNAME:-}
|
||||
NACOS_PASSWORD: ${NACOS_PASSWORD:-}
|
||||
KAFKA_BROKERS: ${KAFKA_BROKERS:-114.55.58.251:9092}
|
||||
KAFKA_ENABLED: ${KAFKA_ENABLED:-true}
|
||||
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true}
|
||||
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
|
||||
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
|
||||
|
||||
x-identity-mysql-env: &identity-mysql-env
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
|
||||
x-redis-session-env: &redis-session-env
|
||||
REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DATABASE: ${REDIS_DATABASE:-50}
|
||||
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
|
||||
|
||||
x-readiness-healthcheck: &readiness-healthcheck
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
services:
|
||||
gb32960-ingest-app:
|
||||
image: crpi-85r4m0ackrm3qpje-vpc.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: gb32960-ingest-app
|
||||
restart: unless-stopped
|
||||
mem_limit: ${GB32960_MEM_LIMIT:-768m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
<<:
|
||||
- *common-env
|
||||
- *identity-mysql-env
|
||||
- *redis-session-env
|
||||
HTTP_PORT: 20100
|
||||
GB32960_PORT: 32960
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID:-gb32960-ingest-portainer}
|
||||
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
|
||||
GB32960_AUTH_ENABLED: ${GB32960_AUTH_ENABLED:-false}
|
||||
GB32960_PLATFORM_USER_HYUNDAI: ${GB32960_PLATFORM_USER_HYUNDAI:-Hyundai}
|
||||
GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}
|
||||
GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}
|
||||
GB32960_PLATFORM_USER_YUEJIN: ${GB32960_PLATFORM_USER_YUEJIN:-YueJin}
|
||||
GB32960_PLATFORM_PWD_YUEJIN: ${GB32960_PLATFORM_PWD_YUEJIN:-}
|
||||
GB32960_PLATFORM_IP_YUEJIN: ${GB32960_PLATFORM_IP_YUEJIN:-}
|
||||
GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false}
|
||||
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
|
||||
SINK_ARCHIVE_PATH: ${GB32960_ARCHIVE_PATH:-/data/archive}
|
||||
ports:
|
||||
- "${GB32960_HTTP_PORT:-20100}:20100"
|
||||
- "${GB32960_TCP_PORT:-32960}:32960"
|
||||
@@ -47,74 +69,142 @@ services:
|
||||
- gb32960-ingest-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20100/actuator/health/readiness >/dev/null"]
|
||||
|
||||
jt808-ingest-app:
|
||||
image: crpi-85r4m0ackrm3qpje-vpc.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: jt808-ingest-app
|
||||
restart: unless-stopped
|
||||
mem_limit: ${JT808_MEM_LIMIT:-768m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
<<:
|
||||
- *common-env
|
||||
- *identity-mysql-env
|
||||
- *redis-session-env
|
||||
HTTP_PORT: 20400
|
||||
JT808_PORT: 10808
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
JT808_PORT: 808
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer}
|
||||
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
|
||||
SINK_ARCHIVE_PATH: ${JT808_ARCHIVE_PATH:-/data/archive}
|
||||
ports:
|
||||
- "${JT808_HTTP_PORT:-20400}:20400"
|
||||
- "${JT808_TCP_PORT:-10808}:10808"
|
||||
- "${JT808_TCP_PORT:-808}:808"
|
||||
volumes:
|
||||
- jt808-ingest-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20400/actuator/health/readiness >/dev/null"]
|
||||
|
||||
yutong-mqtt-app:
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: yutong-mqtt-app
|
||||
restart: unless-stopped
|
||||
mem_limit: ${YUTONG_MQTT_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<:
|
||||
- *common-env
|
||||
- *identity-mysql-env
|
||||
HTTP_PORT: 20500
|
||||
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
|
||||
YUTONG_MQTT_AUTO_STARTUP: ${YUTONG_MQTT_AUTO_STARTUP:-true}
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_MQTT:-yutong-mqtt-portainer}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
|
||||
YUTONG_MQTT_ENDPOINT_NAME: ${YUTONG_MQTT_ENDPOINT_NAME:-yutong}
|
||||
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
|
||||
YUTONG_MQTT_TOPIC: "${YUTONG_MQTT_TOPIC:-#}"
|
||||
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-1}
|
||||
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-yutong-mqtt}
|
||||
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
|
||||
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
|
||||
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
|
||||
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
|
||||
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
|
||||
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
|
||||
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
|
||||
SINK_ARCHIVE_PATH: ${YUTONG_MQTT_ARCHIVE_PATH:-/data/archive}
|
||||
ports:
|
||||
- "${YUTONG_MQTT_HTTP_PORT:-20500}:20500"
|
||||
volumes:
|
||||
- yutong-mqtt-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20500/actuator/health/readiness >/dev/null"]
|
||||
|
||||
vehicle-history-app:
|
||||
image: crpi-85r4m0ackrm3qpje-vpc.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: vehicle-history-app
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- gb32960-ingest-app
|
||||
- jt808-ingest-app
|
||||
mem_limit: ${VEHICLE_HISTORY_MEM_LIMIT:-1536m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
HTTP_PORT: 20200
|
||||
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_HISTORY:-true}
|
||||
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history}
|
||||
KAFKA_CONSUMER_MAX_POLL_RECORDS: ${KAFKA_CONSUMER_MAX_POLL_RECORDS_HISTORY:-2000}
|
||||
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_HISTORY:-1800000}
|
||||
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_HISTORY_DLQ: ${KAFKA_TOPIC_HISTORY_DLQ:-vehicle.dlq.history.v1}
|
||||
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
|
||||
SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true}
|
||||
SINK_ARCHIVE_PATH: /archive/
|
||||
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true}
|
||||
EVENT_FILE_STORE_PATH: /event-store/
|
||||
EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai}
|
||||
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}
|
||||
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts}
|
||||
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}
|
||||
TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root}
|
||||
TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata}
|
||||
TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-30000}
|
||||
TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-1}
|
||||
TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32}
|
||||
ports:
|
||||
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
|
||||
volumes:
|
||||
- vehicle-history-archive:/archive
|
||||
- vehicle-history-event-store:/event-store
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20200/actuator/health/readiness >/dev/null"]
|
||||
|
||||
vehicle-analytics-app:
|
||||
image: crpi-85r4m0ackrm3qpje-vpc.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: vehicle-analytics-app
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- gb32960-ingest-app
|
||||
- jt808-ingest-app
|
||||
mem_limit: ${VEHICLE_ANALYTICS_MEM_LIMIT:-1024m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
HTTP_PORT: 20300
|
||||
HTTP_PORT: 20310
|
||||
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_ANALYTICS:-true}
|
||||
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics}
|
||||
KAFKA_GROUP_STATE: ${KAFKA_GROUP_STATE:-vehicle-state}
|
||||
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_ANALYTICS:-900000}
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
|
||||
KAFKA_GROUP_STAT: ${KAFKA_GROUP_STAT:-vehicle-stat}
|
||||
VEHICLE_STATE_ENABLED: ${VEHICLE_STATE_ENABLED:-false}
|
||||
VEHICLE_STAT_ENABLED: ${VEHICLE_STAT_ENABLED:-true}
|
||||
VEHICLE_STAT_FILE_PATH: /vehicle-stat/
|
||||
VEHICLE_STAT_ZONE_ID: ${VEHICLE_STAT_ZONE_ID:-Asia/Shanghai}
|
||||
VEHICLE_STAT_JT808_MILEAGE_ENABLED: ${VEHICLE_STAT_JT808_MILEAGE_ENABLED:-true}
|
||||
MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
ports:
|
||||
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300"
|
||||
volumes:
|
||||
- vehicle-analytics-stat:/vehicle-stat
|
||||
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20310}:20310"
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
healthcheck:
|
||||
<<: *readiness-healthcheck
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20310/actuator/health/readiness >/dev/null"]
|
||||
|
||||
networks:
|
||||
vehicle-ingest:
|
||||
@@ -123,6 +213,4 @@ networks:
|
||||
volumes:
|
||||
gb32960-ingest-data:
|
||||
jt808-ingest-data:
|
||||
vehicle-history-archive:
|
||||
vehicle-history-event-store:
|
||||
vehicle-analytics-stat:
|
||||
yutong-mqtt-data:
|
||||
|
||||
66
deploy/systemd/README.md
Normal file
66
deploy/systemd/README.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Go Vehicle Gateway Native Deployment
|
||||
|
||||
本目录用于 ECS 原生部署。当前 goal 后续不再使用 Docker/Portainer 作为 Go 接入链路的部署方式。
|
||||
|
||||
## Runtime Layout
|
||||
|
||||
```text
|
||||
/opt/lingniu-go-native/
|
||||
current -> /opt/lingniu-go-native/releases/<git-short-sha>
|
||||
releases/<git-short-sha>/
|
||||
gateway
|
||||
history-writer
|
||||
stat-writer
|
||||
realtime-api
|
||||
env/
|
||||
gateway.env
|
||||
history-writer.env
|
||||
stat-writer.env
|
||||
realtime-api.env
|
||||
spool/gateway/
|
||||
```
|
||||
|
||||
## Services
|
||||
|
||||
| systemd unit | Binary | Purpose |
|
||||
|---|---|---|
|
||||
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP `32960`、JT808 TCP `808`、宇通 MQTT 接入,写 Kafka RAW/unified |
|
||||
| `lingniu-go-history-writer.service` | `history-writer` | 消费 RAW topic,写 TDengine `raw_frames` 和核心时序表 |
|
||||
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 RAW topic,写 MySQL `vehicle_daily_metric` |
|
||||
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费 unified topic,写 Redis,并提供 realtime/raw/stat 查询 API |
|
||||
|
||||
## Build
|
||||
|
||||
在开发机或 CI 上构建 Linux amd64 二进制:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/gateway ./cmd/gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/history-writer ./cmd/history-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/stat-writer ./cmd/stat-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/realtime-api ./cmd/realtime-api
|
||||
```
|
||||
|
||||
## Cutover Notes
|
||||
|
||||
1. 先上传二进制到新 release 目录并更新 `current` symlink。
|
||||
2. 从现有生产 env 生成四个 systemd 专用 env 文件;不要把密钥写入仓库。
|
||||
3. 停止旧 Docker Go 容器或 Java 容器,释放 `808`、`32960`、`20210`。
|
||||
4. 执行:
|
||||
|
||||
```bash
|
||||
systemctl daemon-reload
|
||||
systemctl enable lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=JT808&limit=1'
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=GB32960&limit=1'
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=YUTONG_MQTT&limit=1'
|
||||
```
|
||||
18
deploy/systemd/lingniu-go-gateway.service
Normal file
18
deploy/systemd/lingniu-go-gateway.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Vehicle Gateway
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/gateway.env
|
||||
ExecStart=/opt/lingniu-go-native/current/gateway
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-history-writer.service
Normal file
18
deploy/systemd/lingniu-go-history-writer.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go History Writer
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/history-writer.env
|
||||
ExecStart=/opt/lingniu-go-native/current/history-writer
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-realtime-api.service
Normal file
18
deploy/systemd/lingniu-go-realtime-api.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Realtime API
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/realtime-api.env
|
||||
ExecStart=/opt/lingniu-go-native/current/realtime-api
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-stat-writer.service
Normal file
18
deploy/systemd/lingniu-go-stat-writer.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Stat Writer
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/stat-writer.env
|
||||
ExecStart=/opt/lingniu-go-native/current/stat-writer
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
86
deploy/tools/EnsureKafkaTopics.java
Normal file
86
deploy/tools/EnsureKafkaTopics.java
Normal file
@@ -0,0 +1,86 @@
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.clients.admin.AdminClientConfig;
|
||||
import org.apache.kafka.clients.admin.NewPartitions;
|
||||
import org.apache.kafka.clients.admin.NewTopic;
|
||||
import org.apache.kafka.clients.admin.TopicDescription;
|
||||
|
||||
public final class EnsureKafkaTopics {
|
||||
private static final short REPLICATION_FACTOR = 1;
|
||||
|
||||
private record TopicSpec(String name, int partitions) {}
|
||||
|
||||
private EnsureKafkaTopics() {}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String brokers = args.length > 0 ? args[0] : "172.17.111.56:9092";
|
||||
List<TopicSpec> specs =
|
||||
List.of(
|
||||
topic("vehicle.event.gb32960.v1", 12),
|
||||
topic("vehicle.raw.gb32960.v1", 12),
|
||||
topic("vehicle.dlq.gb32960.v1", 3),
|
||||
topic("vehicle.event.jt808.v1", 12),
|
||||
topic("vehicle.raw.jt808.v1", 12),
|
||||
topic("vehicle.dlq.jt808.v1", 3),
|
||||
topic("vehicle.event.mqtt-yutong.v1", 12),
|
||||
topic("vehicle.raw.mqtt-yutong.v1", 12),
|
||||
topic("vehicle.dlq.mqtt-yutong.v1", 3));
|
||||
|
||||
Properties props = new Properties();
|
||||
props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);
|
||||
props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "15000");
|
||||
props.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "30000");
|
||||
|
||||
try (AdminClient admin = AdminClient.create(props)) {
|
||||
Set<String> existing = admin.listTopics().names().get();
|
||||
List<NewTopic> missing =
|
||||
specs.stream()
|
||||
.filter(spec -> !existing.contains(spec.name()))
|
||||
.map(spec -> new NewTopic(spec.name(), spec.partitions(), REPLICATION_FACTOR))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
try {
|
||||
admin.createTopics(missing).all().get();
|
||||
} catch (ExecutionException e) {
|
||||
System.out.println("createTopics result=" + e.getCause());
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, TopicSpec> byName =
|
||||
specs.stream().collect(Collectors.toMap(TopicSpec::name, spec -> spec));
|
||||
Map<String, TopicDescription> descriptions =
|
||||
admin.describeTopics(byName.keySet()).allTopicNames().get();
|
||||
|
||||
Map<String, NewPartitions> increases =
|
||||
descriptions.entrySet().stream()
|
||||
.filter(entry -> entry.getValue().partitions().size() < byName.get(entry.getKey()).partitions())
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Map.Entry::getKey,
|
||||
entry -> NewPartitions.increaseTo(byName.get(entry.getKey()).partitions())));
|
||||
if (!increases.isEmpty()) {
|
||||
admin.createPartitions(increases).all().get();
|
||||
Thread.sleep(Duration.ofSeconds(2).toMillis());
|
||||
descriptions = admin.describeTopics(byName.keySet()).allTopicNames().get();
|
||||
}
|
||||
|
||||
List<String> lines = new ArrayList<>();
|
||||
descriptions.entrySet().stream()
|
||||
.sorted(Map.Entry.comparingByKey())
|
||||
.forEach(entry -> lines.add(entry.getKey() + " partitions=" + entry.getValue().partitions().size()));
|
||||
System.out.println("bootstrap=" + brokers);
|
||||
lines.forEach(System.out::println);
|
||||
}
|
||||
}
|
||||
|
||||
private static TopicSpec topic(String name, int partitions) {
|
||||
return new TopicSpec(name, partitions);
|
||||
}
|
||||
}
|
||||
@@ -257,14 +257,15 @@
|
||||
<section>
|
||||
<h2>一、系统定位</h2>
|
||||
<p>
|
||||
当前项目不是业务库写入服务。它的核心职责是把 GB/T 32960、JT/T 808、JT/T 1078、JSATL12、
|
||||
MQTT、信达 Push 等不同来源的车辆数据统一转换为 <code>VehicleEvent</code> 和全字段 <code>TelemetrySnapshot</code>,
|
||||
再通过 Kafka、原始报文归档、Parquet 明细文件库、Redis 热状态和独立统计模块交给下游业务。
|
||||
当前项目不是业务库写入服务。它的核心职责是把 GB/T 32960、JT/T 808、Yutong MQTT
|
||||
等默认生产来源的车辆数据统一转换为 <code>VehicleEvent</code> 和全字段 <code>TelemetrySnapshot</code>;
|
||||
JT/T 1078、JSATL12 作为显式 profile 的可选能力保留,
|
||||
再通过 Kafka、原始报文归档、TDengine 历史索引、Redis 热状态和独立统计模块交给下游业务。
|
||||
</p>
|
||||
<div class="summary-grid">
|
||||
<div class="summary-card">
|
||||
<strong>接入方式</strong>
|
||||
<span>Netty TCP、MQTT、HTTP Push、后续可扩展更多 Inbound Adapter。</span>
|
||||
<span>Netty TCP、MQTT、后续可扩展更多 Inbound Adapter;信达 Push 源码已删除。</span>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<strong>统一模型</strong>
|
||||
@@ -272,7 +273,7 @@
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<strong>输出方式</strong>
|
||||
<span>Kafka Protobuf Envelope + 原始 bytes 冷存 + Parquet/DuckDB + Redis 热状态。</span>
|
||||
<span>Kafka Protobuf Envelope + 原始 bytes 冷存 + TDengine raw_frames/locations + Redis 热状态。</span>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<strong>业务边界</strong>
|
||||
@@ -289,7 +290,6 @@ flowchart TB
|
||||
subgraph external["外部来源"]
|
||||
vehicle["车辆终端<br/>GB/T 32960 / JT808 / JT1078"]
|
||||
mqttSource["车企或平台 MQTT"]
|
||||
xindaSource["信达 Push"]
|
||||
commandClient["业务系统 / 运维平台<br/>HTTP 下行命令"]
|
||||
end
|
||||
|
||||
@@ -303,14 +303,13 @@ flowchart TB
|
||||
subgraph inbound["入口适配 modules/inbound"]
|
||||
mqtt["inbound-mqtt<br/>MQTT endpoint 生命周期、PEM TLS"]
|
||||
mqttProfile["MqttProfileRegistry<br/>按 endpoint profile 解析/映射"]
|
||||
xinda["inbound-xinda-push<br/>HTTP Push 适配"]
|
||||
end
|
||||
|
||||
subgraph protocolBase["核心与公共能力 modules/core"]
|
||||
codec["ingest-codec-common<br/>BCD / CRC / BCC / bit 工具"]
|
||||
session["session-core<br/>设备会话、命令分发接口、会话存储"]
|
||||
identity["vehicle-identity<br/>跨协议身份解析、外部标识绑定、文件持久化"]
|
||||
obs["observability<br/>metrics / tracing / health"]
|
||||
identity["vehicle-identity<br/>跨协议身份解析、MySQL 绑定表"]
|
||||
obs["observability<br/>metrics / health"]
|
||||
api["ingest-api<br/>ProtocolId / RawFrame / VehicleEvent / 注解 SPI"]
|
||||
registry["HandlerRegistry<br/>按协议、命令、infoType 路由"]
|
||||
dispatcher["Dispatcher<br/>RawFrame 到 Handler 到 EventBus"]
|
||||
@@ -318,30 +317,36 @@ flowchart TB
|
||||
end
|
||||
|
||||
subgraph sink["输出与明细存储 modules/sinks"]
|
||||
mq["sink-mq<br/>VehicleEvent 到 Protobuf Envelope 到 Kafka"]
|
||||
archive["sink-archive<br/>RawArchive 到本地/S3/OSS 冷存"]
|
||||
fileStore["event-file-store<br/>VehicleEvent/RawArchive 索引到 Parquet,DuckDB 按日查询"]
|
||||
kafka["sink-kafka<br/>VehicleEvent 到 Protobuf Envelope 到 Kafka"]
|
||||
archive["sink-archive<br/>RawArchive 到本地文件系统冷存"]
|
||||
tdengineStore["tdengine-history-store<br/>TDengine raw_frames + locations"]
|
||||
end
|
||||
|
||||
subgraph services["消费服务 modules/services"]
|
||||
history["event-history-service<br/>Kafka 全字段事件到历史查询/导出"]
|
||||
state["vehicle-state-service<br/>Kafka 全字段事件到 Redis 热状态"]
|
||||
stat["vehicle-stat-service<br/>Kafka 全字段事件到可配置日统计"]
|
||||
history["event-history-service<br/>Kafka event/raw 到 TDengine 历史查询"]
|
||||
state["vehicle-state-service<br/>可选 Redis 热状态<br/>optional-latest-state"]
|
||||
stat["vehicle-stat-service<br/>Kafka 全字段事件到指标表"]
|
||||
end
|
||||
|
||||
subgraph apps["应用入口 modules/apps"]
|
||||
gateway["command-gateway<br/>HTTP 到设备命令入口"]
|
||||
bootall["bootstrap-all<br/>聚合所有 AutoConfiguration 一体启动"]
|
||||
gbApp["gb32960-ingest-app<br/>GB32960 TCP 接入"]
|
||||
jtApp["jt808-ingest-app<br/>JT808 TCP 接入"]
|
||||
yutongApp["yutong-mqtt-app<br/>宇通 MQTT 接入"]
|
||||
historyApp["vehicle-history-app<br/>历史查询与 TDengine 写入"]
|
||||
analyticsApp["vehicle-analytics-app<br/>808 日里程指标消费"]
|
||||
gateway["command-gateway<br/>可选 HTTP 设备命令入口"]
|
||||
end
|
||||
|
||||
vehicle --> gb
|
||||
vehicle --> jt808
|
||||
vehicle --> gbApp
|
||||
vehicle --> jtApp
|
||||
vehicle --> jt1078
|
||||
vehicle --> jsatl12
|
||||
mqttSource --> mqtt
|
||||
mqttSource --> yutongApp
|
||||
gbApp --> gb
|
||||
jtApp --> jt808
|
||||
yutongApp --> mqtt
|
||||
mqtt --> mqttProfile
|
||||
mqttProfile --> api
|
||||
xindaSource --> xinda
|
||||
commandClient --> gateway
|
||||
|
||||
gb --> codec
|
||||
@@ -351,41 +356,30 @@ flowchart TB
|
||||
|
||||
gb --> api
|
||||
jt808 --> api
|
||||
xinda --> api
|
||||
xinda --> dispatcher
|
||||
jsatl12 --> archive
|
||||
jsatl12 --> jt808
|
||||
|
||||
api --> registry
|
||||
registry --> dispatcher
|
||||
dispatcher --> bus
|
||||
bus --> mq
|
||||
bus --> kafka
|
||||
bus --> archive
|
||||
bus --> fileStore
|
||||
mq --> history
|
||||
mq --> state
|
||||
mq --> stat
|
||||
kafka --> historyApp
|
||||
kafka --> analyticsApp
|
||||
historyApp --> history
|
||||
analyticsApp --> stat
|
||||
kafka -.可选热状态.-> state
|
||||
history --> tdengineStore
|
||||
|
||||
gateway --> session
|
||||
session --> jt808
|
||||
session --> gb
|
||||
identity --> jt808
|
||||
identity --> mqttProfile
|
||||
identity --> xinda
|
||||
|
||||
obs -.监控.-> dispatcher
|
||||
obs -.监控.-> bus
|
||||
obs -.监控.-> mq
|
||||
bootall -.装配.-> gb
|
||||
bootall -.装配.-> jt808
|
||||
bootall -.装配.-> mqtt
|
||||
bootall -.装配.-> xinda
|
||||
bootall -.装配.-> mq
|
||||
bootall -.装配.-> archive
|
||||
bootall -.装配.-> fileStore
|
||||
bootall -.装配.-> history
|
||||
bootall -.装配.-> state
|
||||
bootall -.装配.-> stat
|
||||
obs -.监控.-> kafka
|
||||
|
||||
classDef entry fill:#eff6ff,stroke:#2563eb,color:#1e3a8a;
|
||||
classDef core fill:#ecfdf3,stroke:#0f766e,color:#134e48;
|
||||
@@ -393,11 +387,11 @@ flowchart TB
|
||||
classDef command fill:#f5f3ff,stroke:#6d28d9,color:#4c1d95;
|
||||
classDef support fill:#f8fafc,stroke:#64748b,color:#334155;
|
||||
|
||||
class gb,jt808,jt1078,jsatl12,mqtt,mqttProfile,xinda entry;
|
||||
class gb,jt808,jt1078,jsatl12,mqtt,mqttProfile,gbApp,jtApp,yutongApp,historyApp,analyticsApp entry;
|
||||
class api,registry,dispatcher,bus,identity core;
|
||||
class mq,archive,fileStore sink;
|
||||
class kafka,archive,tdengineStore sink;
|
||||
class gateway,session command;
|
||||
class codec,obs,bootall support;
|
||||
class codec,obs support;
|
||||
</pre>
|
||||
</div>
|
||||
<div class="legend">
|
||||
@@ -415,17 +409,18 @@ flowchart TB
|
||||
sequenceDiagram
|
||||
autonumber
|
||||
participant Device as 车辆/平台
|
||||
participant Inbound as Inbound Adapter<br/>Netty/MQTT/Push
|
||||
participant Inbound as Inbound Adapter<br/>Netty/MQTT
|
||||
participant Decoder as FrameDecoder<br/>MessageDecoder
|
||||
participant Dispatcher as Dispatcher
|
||||
participant Handler as Protocol Handler<br/>Mapper
|
||||
participant EventBus as DisruptorEventBus
|
||||
participant MQ as sink-mq<br/>Kafka Envelope
|
||||
participant Kafka as sink-kafka<br/>Kafka Envelope
|
||||
participant Archive as sink-archive<br/>ArchiveStore
|
||||
participant FileStore as event-file-store<br/>Parquet + DuckDB
|
||||
participant History as vehicle-history-app
|
||||
participant TDengine as tdengine-history-store<br/>raw_frames + locations
|
||||
participant Consumer as 下游业务消费者
|
||||
|
||||
Device->>Inbound: 原始报文 bytes / MQTT message / HTTP payload
|
||||
Device->>Inbound: 原始报文 bytes / MQTT message
|
||||
Inbound->>Decoder: 粘包拆帧、校验、协议体解析
|
||||
Inbound->>Inbound: MQTT 按 endpoint profile 选择厂商解析器
|
||||
Decoder->>Dispatcher: RawFrame(protocol, command, infoType, payload, rawBytes)
|
||||
@@ -435,12 +430,12 @@ sequenceDiagram
|
||||
Handler->>Handler: 协议字段映射为内部字段
|
||||
Dispatcher->>Handler: 绑定 rawArchiveKey / rawArchiveUri
|
||||
Handler->>EventBus: VehicleEvent(Realtime / Location / Alarm / Login ...)
|
||||
EventBus->>MQ: 投递规整后的业务事件
|
||||
EventBus->>FileStore: 写入解析后明细事件
|
||||
MQ->>MQ: EnvelopeMapper 转 Protobuf
|
||||
MQ->>Consumer: Kafka topic,key=vin,单车有序
|
||||
FileStore->>Consumer: 按日期范围正序/倒序查询和导出
|
||||
Consumer->>Consumer: 历史明细查询、Redis 热状态、可配置日统计、告警工单
|
||||
EventBus->>Kafka: 投递规整后的业务事件
|
||||
Kafka->>Kafka: EnvelopeMapper 转 Protobuf
|
||||
Kafka->>History: event/raw topic,key=vin,单车有序
|
||||
History->>TDengine: 写 raw_frames、locations 和 raw parsed JSON
|
||||
TDengine->>Consumer: 分页历史查询、原始帧追溯、位置查询
|
||||
Kafka->>Consumer: Redis 热状态、指标统计、告警工单等下游消费
|
||||
</pre>
|
||||
</div>
|
||||
<div class="callout">
|
||||
@@ -449,7 +444,7 @@ sequenceDiagram
|
||||
</div>
|
||||
<div class="callout">
|
||||
<strong>原始追溯:</strong>
|
||||
Dispatcher 为所有带 rawBytes 的 RawFrame 生成统一的 <code>rawArchiveKey</code>,业务事件 metadata 使用 <code>rawArchiveUri=archive://...</code> 指向同一份原始 bytes。归档文件的真实本地/S3/OSS 地址只属于 ArchiveStore,查询、导出和 Kafka Envelope 使用逻辑 URI 解耦存储实现。
|
||||
Dispatcher 为所有带 rawBytes 的 RawFrame 生成统一的 <code>rawArchiveKey</code>,业务事件 metadata 使用 <code>rawArchiveUri=archive://...</code> 指向同一份原始 bytes。归档文件的真实本地路径只属于 ArchiveStore,查询、导出和 Kafka Envelope 使用逻辑 URI 解耦存储实现。
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -526,21 +521,21 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/session-core</td>
|
||||
<td>设备会话、命令下发抽象、SessionStore、CommandDispatcher 默认实现。SessionStore 支持 <code>memory</code> 和 <code>redis</code> 两种后端;生产默认使用 Redis 保存 <code>sessionId</code>、VIN 和 phone 三索引及 TTL,内存实现作为本地开发/无 Redis 降级,协议模块只依赖 SPI,不感知存储后端。</td>
|
||||
<td>设备会话、命令下发抽象、SessionStore、CommandDispatcher 默认实现。SessionStore 仅保留 Redis 后端;协议进程本地只保存真实 Netty Channel,Redis 保存 <code>sessionId</code>、VIN 和 phone 三索引及 TTL,协议模块只依赖 SPI,不感知存储细节。</td>
|
||||
<td>设备连接、命令请求。</td>
|
||||
<td>会话状态、命令分发结果、Redis 会话索引。</td>
|
||||
<td><span class="tag">在线状态</span><span class="tag">下行控制</span><span class="tag green">多实例会话索引</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/vehicle-identity</td>
|
||||
<td>跨协议车辆身份解析,维护 phone、deviceId、plate 到 VIN 的绑定关系;支持 memory 和 append-only JSONL file 后端,避免重启丢失绑定。</td>
|
||||
<td>跨协议车辆身份解析,维护 phone、deviceId、plate 到 VIN 的绑定关系;生产运行使用 MySQL 绑定表,避免重启丢失绑定并保证多实例一致。</td>
|
||||
<td>协议模块传入的外部标识。</td>
|
||||
<td>稳定 VIN、解析来源、是否已命中绑定。</td>
|
||||
<td>稳定 VIN、解析来源、是否已命中绑定、MySQL 绑定记录。</td>
|
||||
<td><span class="tag red">统计主键</span><span class="tag">多协议归一</span><span class="tag green">持久化绑定</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/observability</td>
|
||||
<td>监控、指标、追踪、健康检查等横切能力。</td>
|
||||
<td>监控、指标、健康检查等横切能力。</td>
|
||||
<td>核心管线和 Sink 运行状态。</td>
|
||||
<td>Micrometer 指标、健康信息。</td>
|
||||
<td><span class="tag green">可用性</span><span class="tag">延迟监控</span></td>
|
||||
@@ -568,7 +563,7 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/protocols/protocol-jsatl12</td>
|
||||
<td>苏标主动安全报警附件接入。依赖 ArchiveStore 和 JT808 decoder;生产默认端口按旧接收服务对齐为 <code>7612</code>;附件 DataPacket 按旧服务真实布局 <code>01cd + 50B文件名 + offset + length + data</code> 分帧解析,不再把文件名前 4 字节误判为帧长;附件数据写 ArchiveStore 后通过 Dispatcher 产出 MediaMeta 引用事件,事件 metadata 保留 fileName、fileOffset、declaredChunkSizeBytes、archiveKey 和 archiveRef,便于附件明细查询、导出和补传诊断;内置附件状态机处理 T1210 文件清单、DataPacket 分块区间和 T1212 上传完成消息,数据分块会按 T1210 文件清单中的文件名继承手机号并通过共享身份解析器得到内部 VIN,入口 RawFrame、ArchivedChunk 和 MediaMeta 均保留 phone、vin、identityResolved、identitySource,只有找不到文件归属时才标记 UNKNOWN;按文件名合并无身份数据分块并返回 0x9212 完成/补传应答,补传应答包含缺失 offset/length 区间;附件归档失败会产出 JSATL12 Passthrough 错误事件并保留原始分块;JT_MESSAGE 信令复用 JT808 decoder 后进入 Dispatcher,桥接 RawFrame 使用共享身份解析后的内部 VIN并保留 phone、identityResolved、identitySource,坏 JT 信令入口 RawFrame 同样标记 UNKNOWN 身份;超长/畸形附件帧、未闭合超长 JT 信令和坏 JT 信令都以 JSATL12 Passthrough 兜底,附件帧异常同时标记 frameError 与统一 parseError/parseErrorMessage,保留原始字节、peer、长度和错误元数据,帧长和 worker 线程支持部署配置。</td>
|
||||
<td>苏标主动安全报警附件接入(仅 <code>optional-attachments</code> profile 显式构建,不属于默认生产面)。依赖 ArchiveStore 和 JT808 decoder;端口按旧接收服务对齐为 <code>7612</code>;附件 DataPacket 按旧服务真实布局 <code>01cd + 50B文件名 + offset + length + data</code> 分帧解析,不再把文件名前 4 字节误判为帧长;附件数据写 ArchiveStore 后通过 Dispatcher 产出 MediaMeta 引用事件,事件 metadata 保留 fileName、fileOffset、declaredChunkSizeBytes、archiveKey 和 archiveRef,便于附件明细查询、导出和补传诊断;内置附件状态机处理 T1210 文件清单、DataPacket 分块区间和 T1212 上传完成消息,数据分块会按 T1210 文件清单中的文件名继承手机号并通过共享身份解析器得到内部 VIN,入口 RawFrame、ArchivedChunk 和 MediaMeta 均保留 phone、vin、identityResolved、identitySource,只有找不到文件归属时才标记 UNKNOWN;按文件名合并无身份数据分块并返回 0x9212 完成/补传应答,补传应答包含缺失 offset/length 区间;附件归档失败会产出 JSATL12 Passthrough 错误事件并保留原始分块;JT_MESSAGE 信令复用 JT808 decoder 后进入 Dispatcher,桥接 RawFrame 使用共享身份解析后的内部 VIN并保留 phone、identityResolved、identitySource,坏 JT 信令入口 RawFrame 同样标记 UNKNOWN 身份;超长/畸形附件帧、未闭合超长 JT 信令和坏 JT 信令都以 JSATL12 Passthrough 兜底,附件帧异常同时标记 frameError 与统一 parseError/parseErrorMessage,保留原始字节、peer、长度和错误元数据,帧长和 worker 线程支持部署配置。</td>
|
||||
<td>主动安全附件流、JT808 信令帧。</td>
|
||||
<td>归档文件、MediaMeta、JT808 统一事件、坏帧/坏信令/归档失败 Passthrough。</td>
|
||||
<td><span class="tag amber">附件归档</span><span class="tag">报警证据</span><span class="tag">信令复用</span><span class="tag">错误隔离</span></td>
|
||||
@@ -581,14 +576,7 @@ flowchart LR
|
||||
<td><span class="tag">车企平台接入</span><span class="tag green">profile 扩展</span><span class="tag green">身份映射</span><span class="tag green">双向 TLS</span><span class="tag green">错误隔离</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/inbound/inbound-xinda-push</td>
|
||||
<td>信达 Push 接入适配,基于 PushClient 接收真实协议帧并统一转换为 RawFrame;入站 RawFrame 和事件映射均接入统一车辆身份解析,冷存和事件 metadata 均携带内部 vin,并保留 externalVin、phone、deviceId、plateNo、identityResolved、identitySource;0200 映射 Location,并将 totalMileage、mileage 等里程别名归一为内部字段 <code>total_mileage_km</code>;0300 映射标准 AlarmPayload,氢气泄漏类报警统一提升为 HYDROGEN_LEAK/CRITICAL 并标记 hydrogenLeakDetected、hydrogenLeakActionRequired,报警元数据保留 alarmType、alarmName、alarmLevel;未知业务 cmd、非法十六进制 cmd、JSON 异常、身份解析/业务处理异常、启动配置缺失、启动异常、登录拒绝和坏登录应答均兜底为 Passthrough,运行类错误按 vin=unknown、identityResolved=false、identitySource=UNKNOWN 归档,非法 cmd 命令码归零并标记 malformedCommand,业务消息 JSON 异常会在 RawArchive metadata 和最终 Passthrough 事件同时保留 parseErrorMessage,业务处理异常会保留原始 payload 并标记 operationalError、phase=message 和 reason,便于归档追溯。</td>
|
||||
<td>信达 Push payload。</td>
|
||||
<td>RawFrame、Location、Passthrough、operational error。</td>
|
||||
<td><span class="tag">第三方平台接入</span><span class="tag green">统一 Dispatcher</span><span class="tag green">身份映射</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/sink-mq</td>
|
||||
<td>modules/sinks/sink-kafka</td>
|
||||
<td>Kafka Sink。把 VehicleEvent 转成 Protobuf Envelope,按 VIN 分区投递;业务事件携带全字段 TelemetrySnapshot 和 rawArchiveUri,显式 RawArchive Envelope 会填充 archive:// 逻辑 URI 与 size,默认 Kafka Sink 仍不发送原始 bytes 本体;同时提供 KafkaEnvelopeConsumerFactory、KafkaEnvelopeConsumerRunner、KafkaEnvelopeConsumerWorker 和 KafkaEnvelopeDeadLetterSink,统一服务侧 Kafka 消费启动、独立 group 绑定、死信发布和 topic 到 EnvelopeConsumerProcessor 的分发。</td>
|
||||
<td>VehicleEvent。</td>
|
||||
<td>Kafka Protobuf 消息、消费侧 DLQ 记录。</td>
|
||||
@@ -596,52 +584,80 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/sink-archive</td>
|
||||
<td>原始报文冷存,当前有本地文件系统实现,后续可替换 S3/OSS;写入键与业务事件 metadata 中的 <code>rawArchiveKey</code> 保持一致。</td>
|
||||
<td>原始报文冷存,当前为本地文件系统实现;写入键与业务事件 metadata 中的 <code>rawArchiveKey</code> 保持一致。</td>
|
||||
<td>RawArchive 事件、附件流。</td>
|
||||
<td>可回放原始文件、<code>archive://...</code> 逻辑引用。</td>
|
||||
<td><span class="tag amber">问题排查</span><span class="tag amber">合规留痕</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/event-file-store</td>
|
||||
<td>文件型明细库,使用 Parquet 存储解析后的事件明细和 RawArchive 可查询索引,使用 DuckDB 按日期范围正序/倒序查询;RawArchive 只存 archive:// 逻辑 URI、命令码、长度和 metadata,不嵌入原始 bytes,原始 bytes 本体仍由 sink-archive 管理;读取历史文件时未知 protocol 会降级为 UNKNOWN,避免查询中断。</td>
|
||||
<td>VehicleEvent 或 EventFileRecord。</td>
|
||||
<td>protocol/date 分区的 Parquet 文件和查询结果。</td>
|
||||
<td><span class="tag green">按日查询</span><span class="tag green">导出</span><span class="tag">字段配置</span></td>
|
||||
<td>modules/sinks/tdengine-history-store</td>
|
||||
<td>生产历史库边界,负责 TDengine schema、raw_frames/location 行映射、批量写入和分页查询语句;raw_frames 保留完整 payloadJson.parsed,位置表保持轻量字段并通过 rawUri 关联原始帧。</td>
|
||||
<td>Kafka Protobuf Envelope、Raw Envelope。</td>
|
||||
<td>TDengine <code>raw_frames</code> 和位置表。</td>
|
||||
<td><span class="tag green">生产历史</span><span class="tag green">分页查询</span><span class="tag amber">raw 追溯</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/services/event-history-service</td>
|
||||
<td>消费 Kafka 全字段事件,写入文件型明细库,并提供历史查询和 CSV 导出边界;导出支持通过 <code>fields</code> 参数选择内部字段并平铺为 CSV 列,也支持 <code>metadata.xxx</code> 导出协议侧诊断、媒体归档、报警类型等 metadata 字段;未知 Envelope source 会进入 UNKNOWN 协议分区,并在 metadata 保留 <code>originalSource</code> 便于回溯;生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 将坏 protobuf、缺快照和存储异常收敛为结构化结果,并通过 <code>EnvelopeDeadLetterSink</code> 发布死信,避免阻塞 Kafka 分区。</td>
|
||||
<td>消费 32960、808、宇通 MQTT 的 Kafka event/raw topic,生产默认写入 TDengine raw_frames 和位置表,并提供 raw 帧、位置历史等分页查询边界;Raw 查询返回完整 parsed JSON,位置历史只保留核心字段并通过 rawUri 关联原始帧;生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 将坏 protobuf、缺快照和存储异常收敛为结构化结果,并通过 <code>EnvelopeDeadLetterSink</code> 发布死信,避免阻塞 Kafka 分区。</td>
|
||||
<td>Kafka Protobuf Envelope。</td>
|
||||
<td>EventFileRecord、查询结果、可配置字段 CSV。</td>
|
||||
<td><span class="tag green">历史明细</span><span class="tag green">多日导出</span><span class="tag">字段配置</span></td>
|
||||
<td>TDengine raw_frames、位置分页结果、raw archive 逻辑引用。</td>
|
||||
<td><span class="tag green">历史明细</span><span class="tag green">分页查询</span><span class="tag amber">raw JSON</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/services/vehicle-state-service</td>
|
||||
<td>消费 Kafka 全字段事件,更新车辆最新状态、位置、安全和最后事件到 Redis;生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf 和缺快照 Envelope,并把失败记录交给死信出口。</td>
|
||||
<td>消费 Kafka 全字段事件,更新车辆最新状态、位置、安全和最后事件到 Redis;仅通过 <code>optional-latest-state</code> profile 显式构建,不属于默认生产 reactor;生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf 和缺快照 Envelope,并把失败记录交给死信出口。</td>
|
||||
<td>Kafka Protobuf Envelope。</td>
|
||||
<td><code>vehicle:state:{vin}</code>、<code>vehicle:location:{vin}</code>、<code>vehicle:safety:{vin}</code>。</td>
|
||||
<td><span class="tag green">毫秒级热查询</span><span class="tag red">氢泄露安全</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/services/vehicle-stat-service</td>
|
||||
<td>消费 Kafka 全字段事件,抽取统计源点,按车辆规则计算每日里程等日统计;生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf,缺少 telemetry_snapshot 的消息会标记为 SKIPPED 并进入死信出口。</td>
|
||||
<td>Kafka Protobuf Envelope、车辆统计规则。</td>
|
||||
<td>按 <code>vin + stat_date</code> 保存的日统计结果。</td>
|
||||
<td><span class="tag green">每日里程</span><span class="tag">规则配置</span></td>
|
||||
<td>消费 Kafka 全字段事件,808 每日里程只使用 0x0200 附加项 0x01 上报的总里程做当日首末差值;生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf,缺少 telemetry_snapshot 的消息会标记为 SKIPPED 并进入死信出口。</td>
|
||||
<td>Kafka Protobuf Envelope、808 <code>total_mileage_km</code>。</td>
|
||||
<td><code>vehicle_stat_metric</code>:<code>metric_key=daily_mileage_km</code>、<code>calculation_method=JT808_TOTAL_MILEAGE_DIFF</code>。</td>
|
||||
<td><span class="tag green">每日里程</span><span class="tag">指标表</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/command-gateway</td>
|
||||
<td>HTTP 到设备下行命令入口。支持会话查询、808 位置查询、参数查询/设置、终端控制、平台通用应答、终端属性查询 0x8107、区域删除 0x8601、人工报警确认 0x8203,以及 1078 音视频属性查询、实时预览/控制、历史回放/控制、资源列表查询、文件上传/控制;通过 CommandDispatcher 复用 protocol-jt808 的在线通道、流水号和同步应答等待能力。</td>
|
||||
<td>可选 HTTP 到设备下行命令入口。支持会话查询、808 位置查询、参数查询/设置、终端控制、平台通用应答、终端属性查询 0x8107、区域删除 0x8601、人工报警确认 0x8203,以及 1078 音视频属性查询、实时预览/控制、历史回放/控制、资源列表查询、文件上传/控制;通过 CommandDispatcher 复用 protocol-jt808 的在线通道、流水号和同步应答等待能力。</td>
|
||||
<td>业务系统命令请求。</td>
|
||||
<td>CommandDispatcher 调用、设备应答摘要。</td>
|
||||
<td><span class="tag">远程控制</span><span class="tag">参数设置</span><span class="tag">属性查询</span><span class="tag">区域删除</span><span class="tag">报警确认</span><span class="tag">视频控制</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/bootstrap-all</td>
|
||||
<td>一体化启动模块,聚合各模块 AutoConfiguration 和运行配置。</td>
|
||||
<td>application.yml、Spring Boot 启动。</td>
|
||||
<td>完整接入服务实例。</td>
|
||||
<td><span class="tag green">部署</span><span class="tag green">开关配置</span></td>
|
||||
<td>modules/apps/gb32960-ingest-app</td>
|
||||
<td>生产 GB32960 TCP 接入应用,只负责接收、鉴权、解析、冷存引用和 Kafka 投递。</td>
|
||||
<td>GB/T 32960 TCP 32960。</td>
|
||||
<td><code>vehicle.raw.gb32960.v1</code>、<code>vehicle.event.gb32960.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">32960</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/jt808-ingest-app</td>
|
||||
<td>生产 JT808 TCP 接入应用,只负责 808 注册/鉴权/位置等上行解析、冷存引用和 Kafka 投递。</td>
|
||||
<td>JT/T 808 TCP 808。</td>
|
||||
<td><code>vehicle.raw.jt808.v1</code>、<code>vehicle.event.jt808.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">808</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/yutong-mqtt-app</td>
|
||||
<td>生产宇通 MQTT 接入应用,按 endpoint profile 解析 MQTT 报文并投递 Kafka。</td>
|
||||
<td>宇通 MQTT topic。</td>
|
||||
<td><code>vehicle.raw.mqtt-yutong.v1</code>、<code>vehicle.event.mqtt-yutong.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">MQTT</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/vehicle-history-app</td>
|
||||
<td>生产历史应用,消费 32960、808、宇通 MQTT 的 raw/event Kafka Envelope,写 TDengine 并提供历史查询。</td>
|
||||
<td>Kafka Protobuf Envelope。</td>
|
||||
<td>TDengine <code>raw_frames</code>、位置表和历史查询 API。</td>
|
||||
<td><span class="tag green">历史查询</span><span class="tag amber">raw JSON</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/vehicle-analytics-app</td>
|
||||
<td>生产指标应用,当前只消费 JT808 事件并用 GPS 总里程首末差值写每日里程指标。</td>
|
||||
<td><code>vehicle.event.jt808.v1</code>。</td>
|
||||
<td>MySQL <code>vehicle_stat_metric</code>。</td>
|
||||
<td><span class="tag green">每日里程</span><span class="tag">指标表</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -695,8 +711,8 @@ flowchart LR
|
||||
</tr>
|
||||
<tr>
|
||||
<td>明细展示和导出</td>
|
||||
<td><code>event-file-store</code></td>
|
||||
<td>解析后的事件明细和 RawArchive 索引按 protocol/date 写 Parquet。DuckDB 查询时按日期范围读取,并按 event_time 正序或倒序返回。</td>
|
||||
<td><code>vehicle-history-app</code> + <code>tdengine-history-store</code></td>
|
||||
<td>生产查询通过 TDengine raw_frames、位置表和 rawUri 关联完成;不再维护文件型事件索引旁路。</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
227
docs/operations/current-ecs-deployment.md
Normal file
227
docs/operations/current-ecs-deployment.md
Normal file
@@ -0,0 +1,227 @@
|
||||
# 当前 ECS Go 原生部署说明
|
||||
|
||||
更新时间:2026-07-02 01:30 CST
|
||||
|
||||
本文档记录当前 `lingniu-vehicle-ingest` 的生产运行面。当前 goal 的接入链路已经切到 Go 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。密钥只维护在 ECS 环境文件或受控凭据中,不写入 Git。
|
||||
|
||||
## 部署范围
|
||||
|
||||
| systemd 服务 | 二进制 | 说明 | 对外端口 |
|
||||
| --- | --- | --- | --- |
|
||||
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP、JT808 TCP、宇通 MQTT 接入,解析后写 Kafka RAW 和统一事件 | TCP `32960`、TCP `808` |
|
||||
| `lingniu-go-history-writer.service` | `history-writer` | 消费 Kafka RAW,写 TDengine RAW、位置点、里程点核心表 | 无 HTTP |
|
||||
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 Kafka RAW,按总里程差值法写 MySQL 每日指标 | 无 HTTP |
|
||||
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费统一事件写 Redis,并提供实时、历史、统计查询 API | HTTP `20210` |
|
||||
|
||||
信达 Push 已废弃,不参与当前 Go 生产链路。旧 Java/Docker 服务不应占用 `808`、`32960`、`20210`。
|
||||
|
||||
## 应用 ECS
|
||||
|
||||
| 项 | 值 |
|
||||
| --- | --- |
|
||||
| 公网 IP | `115.29.187.205` |
|
||||
| 登录用户 | `root` |
|
||||
| 部署目录 | `/opt/lingniu-go-native` |
|
||||
| 当前 release | `/opt/lingniu-go-native/current` |
|
||||
| 环境文件目录 | `/opt/lingniu-go-native/env` |
|
||||
| systemd unit 目录 | `/etc/systemd/system` |
|
||||
| 对外服务 | `32960`、`808`、`20210` |
|
||||
|
||||
运行目录结构:
|
||||
|
||||
```text
|
||||
/opt/lingniu-go-native/
|
||||
current -> /opt/lingniu-go-native/releases/<git-short-sha>
|
||||
releases/<git-short-sha>/
|
||||
gateway
|
||||
history-writer
|
||||
stat-writer
|
||||
realtime-api
|
||||
env/
|
||||
gateway.env
|
||||
history-writer.env
|
||||
stat-writer.env
|
||||
realtime-api.env
|
||||
spool/gateway/
|
||||
```
|
||||
|
||||
## 中间件
|
||||
|
||||
| 组件 | 内网地址 | 公网地址 | 用途 |
|
||||
| --- | --- | --- | --- |
|
||||
| Kafka | `172.17.111.56:9092` | `114.55.58.251:9092` | RAW 和统一事件消息总线 |
|
||||
| TDengine | `172.17.111.57:6041` | `115.29.185.82:6041` | RAW、位置、里程点时序热存储 |
|
||||
| MySQL RDS | `rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306` | `rm-bp179zbv481rnw3e2no.mysql.rds.aliyuncs.com:3306` | 身份映射、JT808 注册、每日指标 |
|
||||
| Redis RDS | `r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379` | 无 | 准实时车辆状态缓存 |
|
||||
|
||||
生产应用之间访问中间件优先使用内网地址。RDS 白名单需要允许应用 ECS 私网访问。
|
||||
|
||||
## Kafka Topic
|
||||
|
||||
| Topic | 生产者 | 消费者 | 内容 |
|
||||
| --- | --- | --- | --- |
|
||||
| `vehicle.raw.go.gb32960.v1` | `gateway` | `history-writer`、`stat-writer` | GB32960 完整 RAW 记录 |
|
||||
| `vehicle.raw.go.jt808.v1` | `gateway` | `history-writer`、`stat-writer` | JT808 完整 RAW 记录 |
|
||||
| `vehicle.raw.go.yutong-mqtt.v1` | `gateway` | `history-writer` | 宇通 MQTT 完整 RAW 记录 |
|
||||
| `vehicle.event.go.unified.v1` | `gateway` | `realtime-api` | 三类协议统一事件,用于 Redis 实时状态 |
|
||||
|
||||
`gateway` 配置了 Kafka retry;生产 env 中启用 `KAFKA_SPOOL_DIR` 时,本地 spool 可在 Kafka 短暂不可用后回放。
|
||||
|
||||
Kafka 消费组:
|
||||
|
||||
| Consumer Group | Topic | 当前验证 |
|
||||
| --- | --- | --- |
|
||||
| `go-history-writer` | 三个 `vehicle.raw.go.*` topic | 2026-07-02 验证 lag 为 `0` |
|
||||
| `go-stat-writer` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1` | 2026-07-02 验证 lag 为 `0` |
|
||||
| `go-realtime-api` | `vehicle.event.go.unified.v1` | 2026-07-02 验证 lag 为 `0` |
|
||||
|
||||
可重复 smoke:
|
||||
|
||||
```bash
|
||||
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
|
||||
```
|
||||
|
||||
该脚本通过 SSH 登录 Kafka ECS,检查 Go 生产 topic 是否存在,并校验 `go-history-writer`、`go-stat-writer`、`go-realtime-api` 的消费组 lag。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
## TDengine 数据层
|
||||
|
||||
默认数据库:`lingniu_vehicle_ts`
|
||||
|
||||
| 表 | 类型 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `raw_frames` | stable | 完整 RAW 帧,包含 `raw_hex`、`parsed_json`、`fields_json`、`source_endpoint` 和协议/车辆 tags |
|
||||
| `vehicle_locations` | stable | 最小化位置历史,保留 `frame_id` 回链 RAW |
|
||||
| `vehicle_mileage_points` | stable | 最小化总里程点,供历史查询和统计复核 |
|
||||
|
||||
设计原则:
|
||||
|
||||
- 完整结构化 payload 只落在 `raw_frames.parsed_json` / `fields_json`。
|
||||
- `vehicle_locations` 和 `vehicle_mileage_points` 只保留核心查询字段,不重复保存完整 JSON。
|
||||
- `telemetry_fields` 不由当前服务维护,字段配置解析由独立子服务处理。
|
||||
- API 传入东八区时间时,服务会转换为 TDengine 当前存储使用的 UTC 字面量再查询。
|
||||
|
||||
## MySQL 数据层
|
||||
|
||||
默认业务库:`lingniu_vehicle_data`
|
||||
|
||||
| 表 | 维护方 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `vehicle_identity_binding` | 人工/外部主数据 | VIN 映射主表,用 `phone`、`device_id`、`plate` 降级定位 VIN |
|
||||
| `jt808_registration` | `gateway` 自动写入 | 仅 JT808 注册和鉴权记录,以 `phone` 作为主键,记录设备、车牌、厂家、鉴权、来源端点 |
|
||||
| `vehicle_daily_metric` | `stat-writer` 自动写入 | 每日指标表,保存 `daily_mileage_km` 和 `daily_total_mileage_km` |
|
||||
|
||||
每日里程算法:
|
||||
|
||||
```text
|
||||
daily_mileage_km = 当日最大 total_mileage_km - 当日最小 total_mileage_km
|
||||
daily_total_mileage_km = 当日最大 total_mileage_km
|
||||
```
|
||||
|
||||
统计按 `Asia/Shanghai` 自然日归档,当前支持 GB32960、JT808、宇通 MQTT 中能解析出 `total_mileage_km` 的数据。JT808 的总里程来自位置附加信息 `0x01`,单位按协议转换为 km;宇通 MQTT 的 `TOTAL_MILEAGE` 原始单位为米,入库前转换为 km。
|
||||
|
||||
## Redis 实时层
|
||||
|
||||
`realtime-api` 消费 `vehicle.event.go.unified.v1`,按 VIN/vehicle key 维护准实时快照。主要能力:
|
||||
|
||||
| API | 说明 |
|
||||
| --- | --- |
|
||||
| `/api/realtime/vehicles/{vin}` | 查询车辆合并实时快照 |
|
||||
| `/api/realtime/vehicles/{vin}/online` | 查询车辆是否在线 |
|
||||
| `/api/realtime/vehicles/{vin}/protocols/{protocol}` | 查询指定协议的实时快照 |
|
||||
|
||||
Redis 只作为实时缓存;历史和统计以 TDengine/MySQL 为准。
|
||||
|
||||
## API 入口
|
||||
|
||||
公网基础地址:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210
|
||||
```
|
||||
|
||||
常用查询:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=YUTONG_MQTT&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/locations?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/mileage-points?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/stats/daily-metrics?dateFrom=2026-07-02&dateTo=2026-07-02&limit=20'
|
||||
```
|
||||
|
||||
分页参数统一使用 `limit` 和 `offset`。
|
||||
|
||||
部署后推荐直接运行 Go 原生生产 smoke:
|
||||
|
||||
```bash
|
||||
python3 tools/go_prod_acceptance.py --date 2026-07-02
|
||||
```
|
||||
|
||||
该聚合脚本会串行执行 systemd/端口检查、Kafka topic/consumer lag 检查、HTTP 数据链路检查,任一子检查失败时整体退出码为非 0。需要 SSH 可登录应用 ECS 和 Kafka ECS。
|
||||
|
||||
也可以单独运行子检查:
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
|
||||
```
|
||||
|
||||
`go_systemd_prod_smoke.py` 通过 SSH 检查四个 Go systemd 服务是否 `active` 且 `enabled`,并确认 `808`、`32960` 由 `gateway` 监听、`20210` 由 `realtime-api` 监听,同时要求 `/opt/lingniu-go-native/current` 指向有效 release、四个生产二进制均可执行、`/opt/lingniu-go-native/spool/gateway` 没有待回放文件,避免旧 Java/Docker 进程占用生产端口、部署目录错乱或 Kafka spool 静默堆积。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
`go_native_prod_smoke.py` 通过 `20210` HTTP API 验证 GB32960、JT808、宇通 MQTT 的 RAW 查询及结构化 `parsed_json`,三类协议的位置和里程点查询及 `frame_id` RAW 回链,GB32960/JT808 的 `daily_mileage_km`、`daily_total_mileage_km` 统计公式,以及三类协议的 Redis realtime snapshot、online、protocol 查询。实时 snapshot/protocol 查询会校验 `updated_at_ms` 新鲜度;默认检查今天东八区数据时,还会要求三类 RAW 最新样本不超过 15 分钟,避免旧数据误判为接收正常;任一检查不达标时退出码为非 0。
|
||||
|
||||
## 部署命令
|
||||
|
||||
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS,切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收:
|
||||
|
||||
```bash
|
||||
python3 tools/go_native_deploy.py --date 2026-07-02
|
||||
```
|
||||
|
||||
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout` 和 `--spool-poll-interval` 调整。
|
||||
|
||||
脚本执行的发布动作等价于:
|
||||
|
||||
```bash
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/history-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/stat-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/realtime-api
|
||||
scp release.tar.gz root@115.29.187.205:/tmp/
|
||||
ln -sfn /opt/lingniu-go-native/releases/<git-short-sha> /opt/lingniu-go-native/current
|
||||
systemctl daemon-reload
|
||||
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
```
|
||||
|
||||
只更新查询 API 时,可以仅替换 `/opt/lingniu-go-native/current/realtime-api` 并重启:
|
||||
|
||||
```bash
|
||||
systemctl restart lingniu-go-realtime-api.service
|
||||
```
|
||||
|
||||
## 运行检查
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-history-writer --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-stat-writer --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-realtime-api --since '5 minutes ago' --no-pager
|
||||
```
|
||||
|
||||
2026-07-02 01:30 CST 已验证:
|
||||
|
||||
- 四个 Go systemd 服务均为 `active` 且 `enabled`。
|
||||
- `/opt/lingniu-go-native/current` 指向有效 release,四个生产二进制均存在且可执行。
|
||||
- `gateway` 正在监听 `32960` 和 `808`。
|
||||
- `realtime-api` 正在监听 `20210`。
|
||||
- gateway Kafka spool 当前无待回放文件。
|
||||
- GB32960、JT808、YUTONG_MQTT 的东八区 RAW 查询均可命中生产数据,且最新 RAW 样本包含结构化 `parsed_json`。
|
||||
- GB32960、JT808、YUTONG_MQTT 的最新 RAW 样本均在 15 分钟内。
|
||||
- GB32960、JT808、YUTONG_MQTT 的 `vehicle_locations`、`vehicle_mileage_points` 查询均可命中生产数据,且样本包含 `frame_id` 回链 RAW。
|
||||
- `vehicle_daily_metric` 可查到 `daily_mileage_km` 和 `daily_total_mileage_km`,且公式满足 `daily_mileage_km = latest_total_mileage_km - first_total_mileage_km`、`daily_total_mileage_km = latest_total_mileage_km`。
|
||||
- Redis realtime 可查到 GB32960、JT808、YUTONG_MQTT 的在线状态和协议快照,且 snapshot/protocol 快照最近刷新。
|
||||
@@ -1,27 +1,35 @@
|
||||
# GB32960 Split Service Runbook
|
||||
|
||||
This runbook covers the split GB32960 ingest, history, and analytics runtimes. Kafka is the durability boundary between the protocol service and downstream business services.
|
||||
This runbook covers the default production runtimes: GB32960 ingest, JT808 ingest, Yutong MQTT ingest, history, and analytics. Kafka is the durability boundary between protocol ingest services and downstream business services.
|
||||
|
||||
## Service Roles
|
||||
|
||||
| Service | Module | Role | Local ports |
|
||||
| Service | Module | Role | Production ports |
|
||||
| --- | --- | --- | --- |
|
||||
| GB32960 ingest | `:gb32960-ingest-app` | Accept GB32960 TCP connections, decode/authenticate frames, produce raw and normalized records to Kafka, and send protocol ACKs after required Kafka production succeeds. | TCP `32960`, HTTP `20100` |
|
||||
| Vehicle history | `:vehicle-history-app` | Consume GB32960 raw/event Kafka records, store event-history records and raw archive references, and expose event/history/GB32960 query APIs. Kafka raw envelopes carry archive URI/size metadata, not raw frame bytes. | HTTP `20200` |
|
||||
| Vehicle analytics | `:vehicle-analytics-app` | Consume GB32960 event Kafka records and update analytics outputs such as daily vehicle statistics. Vehicle state can also be enabled from this runtime. | HTTP `20300` |
|
||||
| JT808 ingest | `:jt808-ingest-app` | Accept JT/T 808 TCP connections on the production receive port, parse raw frames, archive frame bytes, and produce raw/event envelopes to Kafka. | TCP `808`, HTTP `20400` |
|
||||
| Yutong MQTT ingest | `:yutong-mqtt-app` | Connect to Yutong MQTT, parse subscribed messages, archive payloads, and produce raw/event envelopes to Kafka. | HTTP `20500` |
|
||||
| Vehicle history | `:vehicle-history-app` | Consume GB32960/JT808/Yutong MQTT raw/event Kafka records, store compact history rows plus TDengine `raw_frames` `parsedJson`/`metadataJson`/`rawUri`, and expose TDengine-backed history APIs. | HTTP `20200` |
|
||||
| Vehicle analytics | `:vehicle-analytics-app` | Consume JT808 event Kafka records, calculate daily mileage from reported GPS total mileage, and write metrics to `vehicle_stat_metric`. | HTTP `20310` |
|
||||
|
||||
## Kafka Contract
|
||||
|
||||
| Topic | Producer | Consumers | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `vehicle.raw.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Raw archive reference records. The envelope contains URI/size metadata, not the raw `.bin` payload. |
|
||||
| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` | Normalized vehicle events keyed by VIN when available. |
|
||||
| `vehicle.raw.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Raw frame records. History writes the parsed JSON, metadata JSON, and raw URI to TDengine `raw_frames`. |
|
||||
| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Normalized vehicle events keyed by VIN when available. |
|
||||
| `vehicle.dlq.gb32960.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed production or consumer processing. |
|
||||
| `vehicle.raw.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app` | JT808 raw frame records for TDengine `raw_frames` and raw-frame troubleshooting APIs. |
|
||||
| `vehicle.event.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` | JT808 parsed location/session/alarm events keyed by phone or mapped VIN; analytics uses location total mileage for daily mileage metrics. |
|
||||
| `vehicle.dlq.jt808.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed JT808 production or consumer processing. |
|
||||
| `vehicle.raw.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT raw payload records for TDengine `raw_frames` and troubleshooting APIs. |
|
||||
| `vehicle.event.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT parsed telemetry events keyed by mapped VIN or device identity. |
|
||||
| `vehicle.dlq.mqtt-yutong.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed Yutong MQTT production or consumer processing. |
|
||||
| `vehicle.dlq.history.v1` | History app consumer error paths | Operators/replay tooling | Dead-letter records for failed history storage processing. |
|
||||
|
||||
Default local consumer groups:
|
||||
Default consumer groups:
|
||||
|
||||
- History: `vehicle-history`
|
||||
- Analytics state: `vehicle-state` when `VEHICLE_STATE_ENABLED=true`
|
||||
- Analytics statistics: `vehicle-stat`
|
||||
|
||||
## ACK Semantics
|
||||
@@ -32,11 +40,13 @@ History and analytics failures do not block GB32960 ACKs. They are downstream Ka
|
||||
|
||||
Authorization failures are rejected before the Kafka durability boundary and can be ACKed/rejected immediately according to protocol handling.
|
||||
|
||||
## Local Prerequisites
|
||||
## Build Prerequisites
|
||||
|
||||
- Java 25 and Maven available on PATH.
|
||||
- Local Kafka reachable at `127.0.0.1:9092`, plus Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer`.
|
||||
- No repository-local Kafka bootstrap script or compose file was found during Task 8 verification on 2026-06-23.
|
||||
- Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer` available on the ECS/operator host.
|
||||
- Session state requires Redis. GB32960/JT808 ingest services keep live Netty
|
||||
channels in-process, but session indexes and TTL metadata use Redis only.
|
||||
- Redis is configured through Portainer environment variables such as `REDIS_HOST` and `REDIS_DATABASE=50`; there is no memory session-store fallback.
|
||||
|
||||
If your shell needs an explicit JDK:
|
||||
|
||||
@@ -50,78 +60,82 @@ export PATH="$JAVA_HOME/bin:/opt/homebrew/bin:$PATH"
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
Expected result: `BUILD SUCCESS` and these runnable jars:
|
||||
|
||||
- `modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar`
|
||||
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
|
||||
- `modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar`
|
||||
- `modules/apps/vehicle-history-app/target/vehicle-history-app.jar`
|
||||
- `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar`
|
||||
|
||||
## Create Topics
|
||||
|
||||
Run these only after Kafka is reachable at `127.0.0.1:9092`:
|
||||
Run these only on the ECS/operator host after Kafka is reachable. The default bootstrap address is the ECS private Kafka address used by the production stack:
|
||||
|
||||
```bash
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1
|
||||
KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:-172.17.111.56:9092}"
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.jt808.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.jt808.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.jt808.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.mqtt-yutong.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.mqtt-yutong.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.mqtt-yutong.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.history.v1 --partitions 3 --replication-factor 1
|
||||
```
|
||||
|
||||
Optional sanity check:
|
||||
|
||||
```bash
|
||||
kafka-topics --bootstrap-server 127.0.0.1:9092 --list | grep -E 'vehicle\.(raw|event|dlq)\.gb32960\.v1'
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808|mqtt-yutong|history)\.v1'
|
||||
```
|
||||
|
||||
## Start Split Services Locally
|
||||
## Production Runtime
|
||||
|
||||
Run each command in a separate terminal from the repository root.
|
||||
Production services run on ECS through Portainer/Docker. Do not start GB32960, JT808, Yutong MQTT, history, or analytics services on local developer machines; local machines are for code build, unit tests, and source inspection only.
|
||||
|
||||
GB32960 ingest:
|
||||
Use `deploy/portainer/docker-compose.yml` as the production runtime manifest. The default stack is:
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=127.0.0.1:9092 \
|
||||
GB32960_PORT=32960 \
|
||||
HTTP_PORT=20100 \
|
||||
java --sun-misc-unsafe-memory-access=allow \
|
||||
-jar modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar
|
||||
```
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
- `vehicle-history-app`
|
||||
- `vehicle-analytics-app`
|
||||
|
||||
Vehicle history:
|
||||
`vehicle-history-app` intentionally has no raw archive volume. The history hot path is Kafka to TDengine: raw envelopes are stored in TDengine `raw_frames` with `parsedJson`, `metadataJson`, and `rawUri`. Ingest services may still write original `.bin` files as cold backup, but history API queries must not depend on a shared local archive mount.
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=127.0.0.1:9092 \
|
||||
HTTP_PORT=20200 \
|
||||
EVENT_FILE_STORE_PATH=./target/split-event-store \
|
||||
java --sun-misc-unsafe-memory-access=allow \
|
||||
-jar modules/apps/vehicle-history-app/target/vehicle-history-app.jar
|
||||
```
|
||||
Keep `TDENGINE_HISTORY_ENABLED=true` in the `vehicle-history-app` environment so history reads and writes use TDengine in production.
|
||||
|
||||
`SINK_ARCHIVE_PATH` is intentionally omitted in the split-service Kafka verification path. Current Kafka raw envelopes contain `RawArchiveRef` URI/size metadata only; they do not contain raw bytes that `RawArchiveEventSink` could materialize into `.bin` files in the history JVM.
|
||||
|
||||
Vehicle analytics:
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=127.0.0.1:9092 \
|
||||
HTTP_PORT=20300 \
|
||||
VEHICLE_STAT_FILE_PATH=./target/split-vehicle-stat \
|
||||
java --sun-misc-unsafe-memory-access=allow \
|
||||
-jar modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar
|
||||
```
|
||||
|
||||
To enable the latest-state API in analytics, also set `VEHICLE_STATE_ENABLED=true` and configure the state repository requirements used by `vehicle-state-service`.
|
||||
`vehicle-analytics-app` is intentionally a metric runtime only. Latest-state APIs belong to `vehicle-state-service`, which stays outside the default reactor and should be built with `-Poptional-latest-state` and deployed as a separate consumer if that surface is needed.
|
||||
|
||||
## Health Verification
|
||||
|
||||
Run these from the ECS host or inside the matching container:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20300/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
Expected: each endpoint returns `{"status":"UP"}` or an equivalent Spring Boot health JSON with top-level status `UP`.
|
||||
Expected: each aggregate, liveness, and readiness endpoint returns `{"status":"UP"}` or an equivalent Spring Boot health JSON with top-level status `UP`.
|
||||
|
||||
## End-to-End Verification
|
||||
|
||||
@@ -147,32 +161,57 @@ Expected: the ACK file is non-empty and the decoded bytes begin with the GB32960
|
||||
Verify only what was actually run in your environment:
|
||||
|
||||
```bash
|
||||
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
find target/split-event-store -type f | head
|
||||
find target/split-vehicle-stat -type f | head
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
```
|
||||
|
||||
Use TDengine CLI or a JDBC client to verify history writes:
|
||||
|
||||
```sql
|
||||
USE vehicle_ts;
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'GB32960';
|
||||
SELECT COUNT(*) FROM vehicle_locations WHERE protocol = 'GB32960';
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM jt808_locations WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'MQTT_YUTONG';
|
||||
SELECT COUNT(*) FROM vehicle_locations WHERE protocol = 'MQTT_YUTONG';
|
||||
```
|
||||
|
||||
Useful HTTP query checks after events are consumed. For `realtime_001.hex`, the fixture VIN is `LTEST000000000001`; replace `<date-from>`, `<date-to>`, and `<stat-date>` with values that cover the consumed record's event time:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/records?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/locations?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/raw-frames?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/dictionary'
|
||||
curl -sS 'http://127.0.0.1:20300/api/vehicle-stat/LTEST000000000001/daily?date=<stat-date>'
|
||||
curl -sS 'http://127.0.0.1:20310/api/vehicle-stat/LTEST000000000001/daily?date=<stat-date>'
|
||||
```
|
||||
|
||||
Expected E2E result when Kafka and all services are running:
|
||||
|
||||
- A GB32960 client receives a binary success ACK only after required Kafka production succeeds, and the captured ACK bytes are inspected.
|
||||
- `vehicle.raw.gb32960.v1` receives a raw archive reference envelope.
|
||||
- `vehicle.raw.gb32960.v1` receives a raw frame envelope.
|
||||
- `vehicle.event.gb32960.v1` receives one or more normalized records.
|
||||
- `target/split-event-store` receives event-store files after the history service flushes consumed Kafka records, including raw archive reference records.
|
||||
- `target/split-vehicle-stat` receives stat output after analytics consumes applicable event records.
|
||||
- TDengine `raw_frames` receives GB32960/JT808 RAW rows with parsed JSON and metadata.
|
||||
- TDengine `raw_frames` receives Yutong MQTT RAW rows with parsed JSON and metadata when MQTT payloads are consumed.
|
||||
- TDengine location tables receive compact GB32960/JT808/Yutong MQTT location rows when applicable telemetry is present.
|
||||
- JT808 analytics writes daily mileage rows to MySQL `vehicle_stat_metric` after consuming applicable `vehicle.event.jt808.v1` records.
|
||||
|
||||
Do not expect `vehicle-history-app` to create raw `.bin` archive files from Kafka raw records in the current implementation. `RawArchiveEventSink` can write archive files only when it receives `VehicleEvent.RawArchive.rawBytes()` inside the same JVM; the Kafka envelope carries only `RawArchiveRef` metadata.
|
||||
|
||||
Do not mark any of these as verified unless the matching command was run and the output was inspected.
|
||||
|
||||
## Runtime Location Policy
|
||||
|
||||
Production services run on ECS through Portainer/Docker.
|
||||
|
||||
- Do not create local service runners for GB32960, JT808, Yutong MQTT, history, or analytics.
|
||||
- Do not run the service jars directly on developer machines for production verification.
|
||||
- Keep local work limited to source edits, builds, and automated tests.
|
||||
|
||||
For Portainer deployments, keep history without an archive volume: history consumes raw/event Kafka topics and writes TDengine history rows.
|
||||
|
||||
## Rollback Guidance
|
||||
|
||||
If the split deployment is unhealthy:
|
||||
@@ -183,37 +222,37 @@ If the split deployment is unhealthy:
|
||||
4. If Kafka itself is unavailable, stop `gb32960-ingest-app` or route GB32960 traffic to a previously verified release that preserves the Kafka durability boundary.
|
||||
5. After rollback, compare Kafka consumer group lag and DLQ contents before resuming the split services.
|
||||
|
||||
Rollback commands depend on the deployment environment. For local testing, stop the three Java processes and restart the previous verified app jars or images.
|
||||
Rollback commands depend on the deployment environment. For the default production stack, roll back the Portainer stack image version and let Docker restart the affected containers.
|
||||
|
||||
## Task 8 Verification Notes
|
||||
|
||||
Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`:
|
||||
|
||||
- Package build ran with `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true` and ended with `BUILD SUCCESS`.
|
||||
- Package build should use the current active-app command shown in the Build section.
|
||||
- Repository-local Kafka setup inspection found no Kafka script, no Docker Compose file, and no compose YAML within the searched repository paths.
|
||||
- `nc -z -w 2 127.0.0.1 9092` exited `1`, so no local Kafka broker was reachable at `127.0.0.1:9092`.
|
||||
- `kafka-topics`, `kafka-topics.sh`, `docker`, and `docker-compose` were not found on PATH.
|
||||
- Kafka topic creation, service startup, health checks, Kafka record checks, archive checks, event-store checks, stat output checks, and ACK observation were not run because the local Kafka prerequisite was absent.
|
||||
- Kafka topic creation, service startup, health checks, Kafka record checks, archive checks, TDengine checks, stat output checks, and ACK observation were not run because the local Kafka prerequisite was absent.
|
||||
|
||||
## Latest Local Verification
|
||||
## Latest Build Verification
|
||||
|
||||
Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`:
|
||||
|
||||
- Targeted module tests: `mvn -pl :sink-mq,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test` ended with `BUILD SUCCESS`; Maven reported 55 protocol/sink/core tests, 30 event-history tests, 14 vehicle-state tests, and 19 vehicle-stat tests with 0 failures/errors/skips.
|
||||
- Split app tests: `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app test` ended with `BUILD SUCCESS`; Maven reported 6 app composition/default tests with 0 failures/errors/skips.
|
||||
- Package verification: `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true` ended with `BUILD SUCCESS` and repackaged all three app jars.
|
||||
- Split app startup: not run in this verification pass because local Kafka was still absent.
|
||||
- Kafka raw records: not verified; no local Kafka broker or Kafka CLI tools were available.
|
||||
- Kafka event records: not verified; no local Kafka broker or Kafka CLI tools were available.
|
||||
- Event-store files: not verified; downstream services were not started without Kafka.
|
||||
- Targeted module tests: `mvn -pl :sink-kafka,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test` ended with `BUILD SUCCESS`; Maven reported 55 protocol/sink/core tests, 30 event-history tests, 14 vehicle-state tests, and 19 vehicle-stat tests with 0 failures/errors/skips.
|
||||
- Split app tests should cover the current active apps listed in the Service Roles section.
|
||||
- Package verification should use the active-app package command from the Build section.
|
||||
- Split app startup: not run in this verification pass because an ECS/Portainer runtime was outside that build-only check.
|
||||
- Kafka raw records: not verified in that build-only check.
|
||||
- Kafka event records: not verified in that build-only check.
|
||||
- TDengine history rows: not verified; downstream services were not started without Kafka.
|
||||
- Analytics output: not verified; downstream services were not started without Kafka.
|
||||
- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above in a Kafka-backed local or staging environment.
|
||||
- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above on ECS or staging.
|
||||
|
||||
## Remote Kafka Live Verification
|
||||
|
||||
Observed on 2026-06-23 with Kafka `114.55.58.251:9092`:
|
||||
|
||||
- Replaced the local `bootstrap-all` listener on TCP `32960` with `gb32960-ingest-app` on TCP `32960` and HTTP `20100`.
|
||||
- `gb32960-ingest-app` handled TCP `32960` and HTTP `20100`.
|
||||
- Ingest app health check `curl -sS http://127.0.0.1:20100/actuator/health` returned `{"status":"UP"}`.
|
||||
- Kafka AdminClient confirmed topics `vehicle.raw.gb32960.v1` and `vehicle.event.gb32960.v1` existed, and created `vehicle.dlq.gb32960.v1`.
|
||||
- The app accepted the live Hyundai platform connection from `8.134.95.166`, authenticated platform login, and received real GB32960 realtime/report/login/logout frames.
|
||||
@@ -222,3 +261,13 @@ Observed on 2026-06-23 with Kafka `114.55.58.251:9092`:
|
||||
- Runtime logs showed successful GB32960 ACK flushes for platform login, vehicle login, and vehicle logout after the Kafka-backed split ingest service was running.
|
||||
|
||||
This verifies the live path `GB32960 TCP 32960 -> gb32960-ingest-app -> Kafka raw/event topics` against the remote Kafka broker. Downstream `vehicle-history-app` and `vehicle-analytics-app` were not started in this pass.
|
||||
|
||||
## 2026-06-29 Superseded Local Verification Record
|
||||
|
||||
Observed on 2026-06-29 in worktree `.worktrees/vehicle-ingest-redesign-phase1`. This record is retained only as historical evidence of parser/storage behavior; do not repeat it as a deployment procedure because production services now run on ECS through Portainer/Docker.
|
||||
|
||||
- External JT808 traffic on TCP `808` was parsed and written through Kafka into TDengine. Direct TDengine checks showed more than `6000` JT808 `raw_frames` and more than `1900` JT808 `vehicle_locations`; current production history does not maintain a `telemetry_fields` table.
|
||||
- A GB32960 synthetic realtime frame sent to TCP `32960` received a binary ACK beginning with `2323`. The same VIN `LTEST202606290001` was queryable from TDengine-backed history APIs with speed, mileage, longitude, and latitude fields.
|
||||
- History raw and event Kafka consumption used separate bindings so raw-frame writes remained current while location/field event ingestion continued.
|
||||
|
||||
The verified behavior was `GB32960/JT808 TCP -> protocol app -> Kafka raw/event topics -> vehicle-history-app -> TDengine raw_frames/location tables -> Swagger/API query`; the deployment method from that historical run is superseded.
|
||||
|
||||
@@ -0,0 +1,888 @@
|
||||
# Go Vehicle Gateway 旁路验证记录
|
||||
|
||||
验证时间:2026-07-01 22:00-22:10 CST
|
||||
|
||||
## 部署版本
|
||||
|
||||
- Git commit:`60ca42e`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-60ca42e-20260701220622`
|
||||
- 部署方式:ECS `115.29.187.205` 上 Docker Compose 旁路部署
|
||||
- Compose 目录:`/opt/lingniu-go`
|
||||
|
||||
## 旁路端口和 topic
|
||||
|
||||
为了不影响现有 Java 生产服务,本次没有占用生产端口和生产 topic。
|
||||
|
||||
| 项 | 值 |
|
||||
|---|---|
|
||||
| GB32960 旁路 TCP | `115.29.187.205:23296 -> container :32960` |
|
||||
| JT808 旁路 TCP | `115.29.187.205:18080 -> container :808` |
|
||||
| Realtime API | `115.29.187.205:20210` |
|
||||
| GB32960 RAW topic | `vehicle.raw.go.gb32960.v1` |
|
||||
| JT808 RAW topic | `vehicle.raw.go.jt808.v1` |
|
||||
| Yutong MQTT RAW topic | `vehicle.raw.go.yutong-mqtt.v1` |
|
||||
| Unified topic | `vehicle.event.go.unified.v1` |
|
||||
|
||||
## 容器状态
|
||||
|
||||
ECS 上四个 Go 容器均已启动:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway Up
|
||||
go-history-writer Up
|
||||
go-stat-writer Up
|
||||
go-realtime-api Up
|
||||
```
|
||||
|
||||
## Kafka 验证
|
||||
|
||||
向 `18080` 注入 JT808 样例帧后,`vehicle.raw.go.jt808.v1` 和
|
||||
`vehicle.event.go.unified.v1` 均消费到统一 JSON envelope。
|
||||
|
||||
样例解析结果包含:
|
||||
|
||||
- `protocol=JT808`
|
||||
- `message_id=0x0200`
|
||||
- `phone=013307795425`
|
||||
- `longitude=121.069881`
|
||||
- `latitude=30.590151`
|
||||
- `speed_kmh=23`
|
||||
- `total_mileage_km=10241.2`
|
||||
- 表 27 附加项 `0x01` 原始值 `0001900C`
|
||||
|
||||
## TDengine 验证
|
||||
|
||||
目标库:`lingniu_vehicle_ts`
|
||||
|
||||
```text
|
||||
raw_frames count = 3
|
||||
vehicle_locations count = 2
|
||||
vehicle_mileage_points count = 2
|
||||
```
|
||||
|
||||
最新 RAW 标识:
|
||||
|
||||
```text
|
||||
GB32960 | LNBSCB3D4R1234567 | LNBSCB3D4R1234567 |
|
||||
JT808 | JT808:013307795425 | | 013307795425
|
||||
JT808 | JT808:013307795425 | | 013307795425
|
||||
```
|
||||
|
||||
说明:JT808 样例 phone 当前没有在 `vehicle_identity_binding` 中解析到 VIN,因此
|
||||
TDengine tag 使用 `vehicle_key=JT808:013307795425`。
|
||||
|
||||
## Redis 验证
|
||||
|
||||
向 `23296` 注入带 VIN 的 GB32960 合成帧后,Realtime API 可查询:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LNBSCB3D4R1234567
|
||||
```
|
||||
|
||||
返回字段包含:
|
||||
|
||||
- `vin=LNBSCB3D4R1234567`
|
||||
- `protocols=["GB32960"]`
|
||||
- `online=true`
|
||||
- `speed_kmh=30`
|
||||
- `total_mileage_km=10000`
|
||||
- `soc_percent=85`
|
||||
- `longitude=121`
|
||||
- `latitude=30.56`
|
||||
|
||||
## MySQL 统计验证
|
||||
|
||||
向 `23296` 注入带 VIN 的 GB32960 合成帧后,`vehicle_daily_metric` 写入:
|
||||
|
||||
```text
|
||||
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_mileage_km | 0.000 | 10000.000 | 10000.000 | 1
|
||||
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_total_mileage_km | 10000.000 | 10000.000 | 10000.000 | 1
|
||||
```
|
||||
|
||||
说明:本次 GB32960 合成帧的设备时间为测试值,因此统计日期为 `2020-07-01`。
|
||||
|
||||
## 已修复问题
|
||||
|
||||
- TDengine WebSocket 普通 `INSERT` 使用参数占位符时,字符串未按预期加引号,导致语法错误。已在 `60ca42e` 中改为显式 SQL literal 转义。
|
||||
- MySQL Go DSN 中 `charset=utf8mb4%2Cutf8` 会被 driver 传给 MySQL,导致语法错误;ECS 旁路 env 已改为 `charset=utf8mb4`。
|
||||
- TDengine ECS 当前可用 root 密码与早先记录不一致,本次旁路 env 已按实际可认证值修正。
|
||||
|
||||
## 未完成
|
||||
|
||||
- 未切换真实生产端口 `32960/808`。
|
||||
- 未将真实外部 32960/808 流量转发到 Go 旁路端口。
|
||||
- 未开启宇通 MQTT 正式订阅。
|
||||
- JT808 样例未解析到 VIN,需维护 `vehicle_identity_binding` 后再验证 808 每日统计。
|
||||
|
||||
## 2026-07-01 22:22 真实归档帧复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`5f4f4fd`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5f4f4fd-20260701222128`
|
||||
|
||||
本轮补充验证内容:
|
||||
|
||||
- JT808 支持 2019 版 versioned header,真实帧 `0x0200` 可解析出 `phone=14894135060`、`longitude=117.178187`、`latitude=30.570155`、`speed_kmh=33.4`、`total_mileage_km=7868.9`。
|
||||
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 可解析 `0x01` 整车、`0x02` 驱动电机、`0x03` 燃料电池、`0x04` 发动机、`0x05` 位置、`0x06` 极值、`0x07` 报警、`0x08` 电压、`0x09` 温度;后续 `0x30` 广东燃料电池扩展暂以 unknown raw 保留。
|
||||
- GB32960 真实帧字段包含 `fuel_cell_hydrogen_consumption_kg_per_100km=1.8`、`longitude=120.800326`、`latitude=31.634907`、`total_mileage_km=53490.9`。
|
||||
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回上述经纬度和氢耗字段。
|
||||
- TDengine `vehicle_locations` 和 `vehicle_mileage_points` 已写入真实 GB32960 位置和里程点:
|
||||
|
||||
```text
|
||||
2020-07-01 08:09:22 | GB32960 | LB9A32A21R0LS1707 | lon=120.800326 | lat=31.634907 | mileage=53490.9
|
||||
```
|
||||
|
||||
后续仍需补齐:
|
||||
|
||||
- JT808 真实 phone `14894135060` 仍未解析到 VIN,需维护身份绑定后再验证 VIN 维度实时和统计。
|
||||
|
||||
## 2026-07-01 22:28 广东燃料电池扩展复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`a15de91`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-a15de91-20260701222743`
|
||||
|
||||
本轮补充验证内容:
|
||||
|
||||
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 的 vendor 尾部已从 `unknown_unit` 补为结构化块:
|
||||
`0x30 gd_fc_stack`、`0x31 gd_fc_auxiliary`、`0x32 gd_fc_dcdc`、`0x33 gd_fc_air_conditioner`、
|
||||
`0x34 gd_fc_vehicle_info`、`0x80 gd_fc_demo_extension`、`0x83 gd_fc_vendor_tlv`。
|
||||
- Kafka `vehicle.raw.go.gb32960.v1` 最新消息包含上述 block,并输出关键扁平字段:
|
||||
`gd_fc_stack_hydrogen_inlet_pressure_kpa=97`、`gd_fc_stack_cell_count=108`、
|
||||
`gd_fc_vehicle_hydrogen_mass_kg=4.3`、`gd_fc_demo_stack_temp_c=65`。
|
||||
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回 vendor 扁平字段:
|
||||
|
||||
```text
|
||||
gd_fc_dcdc_controller_temp_c=47
|
||||
gd_fc_demo_stack_temp_c=65
|
||||
gd_fc_stack_cell_count=108
|
||||
gd_fc_stack_frame_cell_count=108
|
||||
gd_fc_stack_hydrogen_inlet_pressure_kpa=97
|
||||
gd_fc_stack_water_outlet_temp_c=65
|
||||
gd_fc_vehicle_hydrogen_mass_kg=4.3
|
||||
```
|
||||
|
||||
- TDengine `raw_frames` 中 GB32960 行数增加到 `4`,最新 RAW 标识:
|
||||
|
||||
```text
|
||||
2026-07-01 14:28:24.869 | 172.20.0.1:47082 | GB32960 | LB9A32A21R0LS1707 | OK
|
||||
```
|
||||
|
||||
后续仍需补齐:
|
||||
|
||||
- JT808 真实 phone `14894135060` 到 VIN 的绑定数据维护与统计复验。
|
||||
- Go 旁路尚未切换到生产 `32960/808` 端口;仍运行在 `23296/18080`。
|
||||
|
||||
## 2026-07-01 22:43 JT808 前导零 phone 身份解析复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`6b55694`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-6b55694-20260701224300`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- JT808 2011 老版包头的终端手机号按 BCD[6] 解析会保留前导 `0`,例如包头为 `013079963379`。
|
||||
- `vehicle_identity_binding.phone` 中实际维护的是去前导零后的手机号 `13079963379`。
|
||||
- 身份解析候选键已调整为先查原始 phone,再查去前导零 phone,之后再按 `device_id`、`plate` 降级。
|
||||
|
||||
真实帧复验样本:
|
||||
|
||||
- 归档帧:`/opt/lingniu/vehicle-ingest/gb32960/archive/2026/07/01/JT808/unknown/1782903942845000.bin`
|
||||
- 消息:`0x0200` 位置信息汇报
|
||||
- 包头 phone:`013079963379`
|
||||
- 绑定表 phone:`13079963379`
|
||||
- 解析 VIN:`LKLG7C4E3NA774736`
|
||||
|
||||
Kafka `vehicle.raw.go.jt808.v1` 最新消息确认:
|
||||
|
||||
```text
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
parsed.header.phone_trim_zero=13079963379
|
||||
parsed.identity.resolved=true
|
||||
parsed.identity.source=phone
|
||||
parsed.identity.value=13079963379
|
||||
```
|
||||
|
||||
Realtime API 确认:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
speed_kmh=0
|
||||
total_mileage_km=0
|
||||
device_time=2026-07-01T19:05:40+08:00
|
||||
```
|
||||
|
||||
TDengine 确认:
|
||||
|
||||
```text
|
||||
raw_frames:
|
||||
2026-07-01 14:49:10.913 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | message_id=512
|
||||
|
||||
vehicle_locations:
|
||||
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | lon=119.557997 | lat=29.049092 | speed=0 | mileage=0
|
||||
|
||||
vehicle_mileage_points:
|
||||
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | mileage=0
|
||||
```
|
||||
|
||||
MySQL `vehicle_daily_metric` 确认:
|
||||
|
||||
```text
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
说明:该真实位置帧携带的表 27 附加项 `0x01` 总里程值为 `0`,因此本次只验证身份解析、链路落地和差值统计写入;要验证非零日里程,需要回放或等待同一 VIN 的非零总里程连续位置点。
|
||||
|
||||
## 2026-07-01 22:55 Kafka 发布重试部署复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`7e7066d`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7e7066d-20260701225503`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- gateway 的 Kafka 发布增加同步重试封装。
|
||||
- 默认配置:
|
||||
- `KAFKA_PUBLISH_ATTEMPTS=3`
|
||||
- `KAFKA_PUBLISH_BACKOFF_MS=100`
|
||||
- `KAFKA_PUBLISH_TIMEOUT_MS=3000`
|
||||
- RAW 仍先于 unified event 写入;RAW 写失败不会继续写 unified。
|
||||
- 单元测试覆盖首次失败后成功、重试耗尽返回错误、上下文取消时停止重试。
|
||||
|
||||
部署后容器状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway | go-7e7066d-20260701225503 | Up
|
||||
go-history-writer | go-7e7066d-20260701225503 | Up
|
||||
go-stat-writer | go-7e7066d-20260701225503 | Up
|
||||
go-realtime-api | go-7e7066d-20260701225503 | Up
|
||||
```
|
||||
|
||||
真实 808 帧回放复验:
|
||||
|
||||
```text
|
||||
source_endpoint=172.20.0.1:51966
|
||||
protocol=JT808
|
||||
message_id=0x0200
|
||||
phone=013079963379
|
||||
vin=LKLG7C4E3NA774736
|
||||
parsed.identity.value=13079963379
|
||||
```
|
||||
|
||||
Kafka `vehicle.raw.go.jt808.v1` 确认新消息写入,partition 6 offset 从 `1` 增加到 `2`。最新消息包含:
|
||||
|
||||
```text
|
||||
received_at_ms=1782917751903
|
||||
source_endpoint=172.20.0.1:51966
|
||||
vin=LKLG7C4E3NA774736
|
||||
fields.longitude=119.557997
|
||||
fields.latitude=29.049092
|
||||
fields.total_mileage_km=0
|
||||
```
|
||||
|
||||
TDengine `raw_frames` 确认新 RAW 落地:
|
||||
|
||||
```text
|
||||
2026-07-01 14:55:51.903 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | source=172.20.0.1:51966
|
||||
```
|
||||
|
||||
Realtime API 确认 Redis 已刷新:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
|
||||
received_at_ms=1782917751903
|
||||
source_endpoint=172.20.0.1:51966
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=0
|
||||
```
|
||||
|
||||
说明:当前重试层只能覆盖 Kafka 短暂抖动;如果 Kafka 长时间不可用,仍需后续增加本地磁盘 spool/WAL 和恢复补发。
|
||||
|
||||
## 2026-07-01 23:02 切换 Go 到生产 32960/808 端口
|
||||
|
||||
本轮操作:
|
||||
|
||||
- 停止 ECS 上原有 Java Docker 容器:
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
- `vehicle-history-app`
|
||||
- `vehicle-analytics-app`
|
||||
- `vehicle-state-app`
|
||||
- `telemetry-field-parser-app`
|
||||
- 保留 Go sidecar 和 Portainer agent。
|
||||
- 修改 `/opt/lingniu-go/.env`:
|
||||
- `GO_GB32960_TCP_PORT=32960`
|
||||
- `GO_JT808_TCP_PORT=808`
|
||||
- Go gateway 使用镜像:
|
||||
- `crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-243de5c-20260701230028`
|
||||
- 开启 Kafka 本地 spool:
|
||||
- host:`/opt/lingniu-go/spool/gateway`
|
||||
- container:`/data/spool/gateway`
|
||||
|
||||
端口验证:
|
||||
|
||||
```text
|
||||
0.0.0.0:32960 -> go-vehicle-gateway:32960
|
||||
0.0.0.0:808 -> go-vehicle-gateway:808
|
||||
18080/23296 -> 已释放
|
||||
```
|
||||
|
||||
启动日志确认:
|
||||
|
||||
```text
|
||||
kafka durable spool enabled dir=/data/spool/gateway replay_interval_ms=1000
|
||||
tcp listener started protocol=GB32960 addr=[::]:32960
|
||||
tcp listener started protocol=JT808 addr=[::]:808
|
||||
```
|
||||
|
||||
真实连接确认:
|
||||
|
||||
```text
|
||||
GB32960 external remote: 8.134.95.166:50084
|
||||
JT808 external remote: 115.231.168.135:13801
|
||||
```
|
||||
|
||||
生产 `808` 回放复验:
|
||||
|
||||
```text
|
||||
source_endpoint=172.20.0.1:54680
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=0
|
||||
```
|
||||
|
||||
TDengine 最新 RAW 复验显示生产 808 已有真实外部数据持续落地:
|
||||
|
||||
```text
|
||||
2026-07-01 15:03:02.960 | JT808 | LKLG7C4E1NA774802 | 013079963291 | 222.66.200.68:28394
|
||||
2026-07-01 15:03:02.114 | JT808 | | 14894135583 | 122.152.221.156:33871
|
||||
2026-07-01 15:03:01.352 | JT808 | | 013307795519 | 115.231.168.135:13801
|
||||
2026-07-01 15:03:01.330 | JT808 | LKLG7C4E3NA774736 | 013079963379 | 222.66.200.68:28393
|
||||
```
|
||||
|
||||
Kafka spool 目录确认:
|
||||
|
||||
```text
|
||||
/opt/lingniu-go/spool/gateway file count = 0
|
||||
```
|
||||
|
||||
说明:生产 808 中仍有部分 phone 未解析出 VIN,后续需要继续维护 `vehicle_identity_binding` 的 phone/device_id/plate 到 VIN 映射,或者增强 808 注册/鉴权帧中的 identity 回写。
|
||||
|
||||
## 2026-07-01 23:09 808 里程统计防 0 污染复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`d347525`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-d347525-20260701230845`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- `stat-writer` 不再把 `total_mileage_km <= 0` 的采样写入 MySQL 每日指标。
|
||||
- MySQL upsert 对历史 `first_total_mileage_km=0` / `latest_total_mileage_km=0` 做纠偏;后续首次有效总里程会替换掉历史 0。
|
||||
- `realtime-api` 合并实时快照时,非正 `total_mileage_km` 不再覆盖已有正值;速度、位置、状态等其他字段仍按事件时间更新。
|
||||
|
||||
测试验证:
|
||||
|
||||
```text
|
||||
go test ./...
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
生产回放验证:
|
||||
|
||||
- 第一条 808 帧:`device_time=2026-07-01T23:09:35+08:00`,`total_mileage_km=12345.6`
|
||||
- 第二条 808 帧:`device_time=2026-07-01T23:09:36+08:00`,`total_mileage_km=0`
|
||||
- 两条帧使用同一 VIN:`LKLG7C4E3NA774736`
|
||||
|
||||
Redis merged 快照确认:第二条 0 里程帧没有覆盖已有有效总里程。
|
||||
|
||||
```text
|
||||
event_time_ms=1782918576000
|
||||
source_endpoint=172.20.0.1:40946
|
||||
speed_kmh=0
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=12345.6
|
||||
field_times_ms.total_mileage_km=1782918575000
|
||||
```
|
||||
|
||||
MySQL `vehicle_daily_metric` 确认:0 里程采样未进入统计,sample_count 只因有效总里程采样增加一次。
|
||||
|
||||
```text
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=13 | first=12345.600 | latest=12345.600
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 12345.600 | sample_count=13 | first=12345.600 | latest=12345.600
|
||||
```
|
||||
|
||||
说明:生产 TDengine 中已经存在大量 808 非零 `total_mileage_km` 采样,但许多 phone 尚未映射到 VIN,因此 MySQL 按 VIN 的统计只会覆盖已解析 VIN 的车辆。下一步应继续补齐 `vehicle_identity_binding`,让更多 808 车辆进入统计。
|
||||
|
||||
## 2026-07-01 23:23 808 注册身份自动维护复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`7b44f97`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7b44f97-20260701232200`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- 808 `0x0100` 注册帧解析 `province`、`city`、`manufacturer`、`device_type`、`device_id`、`plate_color`、`plate`。
|
||||
- 808 `0x0102` 鉴权帧解析 `auth_token`。
|
||||
- Go gateway 在 identity resolve 后写入 `jt808_registration`,普通上行也会刷新 `latest_seen_at`。
|
||||
- 注册帧通过车牌解析到 VIN 后,会尝试把 phone/device/plate 反写到 `vehicle_identity_binding`。
|
||||
- 注册帧车牌支持 GBK 解码,避免 `Incorrect string value` 写入错误。
|
||||
|
||||
验证命令:
|
||||
|
||||
```text
|
||||
go test ./...
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
ECS 运行状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway | go-7b44f97-20260701232200 | 0.0.0.0:808->808, 0.0.0.0:32960->32960
|
||||
go-history-writer | go-7b44f97-20260701232200 | Up
|
||||
go-stat-writer | go-7b44f97-20260701232200 | Up
|
||||
go-realtime-api | go-7b44f97-20260701232200 | 0.0.0.0:20210->20210
|
||||
```
|
||||
|
||||
生产日志确认:
|
||||
|
||||
- 新版本启动后真实 808、32960 连接持续进入。
|
||||
- GBK 车牌注册帧不再出现 `Incorrect string value`。
|
||||
|
||||
MySQL `jt808_registration` 确认:
|
||||
|
||||
```text
|
||||
registration_rows=214
|
||||
distinct_phone=214
|
||||
vin_rows=74
|
||||
plate_rows=74
|
||||
device_rows=72
|
||||
|
||||
013079963379 | device_id=9963379 | plate=沪A06788F | vin=LKLG7C4E3NA774736 | latest_registered_at=2026-07-01 23:23:01
|
||||
013079963321 | device_id=9963321 | plate=沪A59613F | vin=LKLG7C4EXNA774796 | latest_registered_at=2026-07-01 23:23:04
|
||||
```
|
||||
|
||||
说明:`jt808_registration.phone` 保存 808 包头原始 phone;`vehicle_identity_binding.phone` 中已有部分去前导零 phone。Go resolver 会同时查询原始 phone 和去前导零 phone。
|
||||
|
||||
## 2026-07-01 23:28 生产链路与指标清理复验
|
||||
|
||||
本轮复验目标:
|
||||
|
||||
- 确认 Go gateway 的 Kafka durable spool 是重启窗口残留,而不是持续发布失败。
|
||||
- 确认 stat-writer / history-writer / realtime-api 的 Kafka consumer group 无 lag。
|
||||
- 确认真实 808 车辆 registration、TDengine RAW/里程点、Redis 实时查询均在持续更新。
|
||||
- 清理之前生产合成回放产生的 JT808 当日指标测试数据。
|
||||
|
||||
Kafka topic / consumer group:
|
||||
|
||||
```text
|
||||
vehicle.raw.go.jt808.v1 有持续 offset
|
||||
vehicle.raw.go.gb32960.v1 有持续 offset
|
||||
vehicle.event.go.unified.v1 有持续 offset
|
||||
|
||||
go-stat-writer LAG=0
|
||||
go-history-writer LAG=0
|
||||
go-realtime-api LAG=0
|
||||
```
|
||||
|
||||
TDengine:
|
||||
|
||||
```text
|
||||
raw_frames:
|
||||
JT808 2922
|
||||
GB32960 27
|
||||
|
||||
vehicle_mileage_points:
|
||||
JT808 2050
|
||||
GB32960 2
|
||||
```
|
||||
|
||||
Redis 实时查询样例:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
vin=LKLG7C4E3NA774736
|
||||
source_endpoint=222.66.200.68:29646
|
||||
plate=沪A06788F
|
||||
longitude=119.556866
|
||||
latitude=29.047911
|
||||
total_mileage_km=12345.6
|
||||
```
|
||||
|
||||
说明:该 Redis merged 快照保留了此前有效 `total_mileage_km=12345.6`,后续真实 808 位置帧上报的 `total_mileage_km=0` 不会覆盖该字段。TDengine 中该 VIN 后续真实里程点的 `total_mileage_km` 均为 0,因此 stat-writer 按“只统计正总里程”的规则不再写 JT808 每日指标。
|
||||
|
||||
MySQL 指标清理:
|
||||
|
||||
```text
|
||||
删除前:
|
||||
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_mileage_km | first=0 | latest=0 | sample_count=7
|
||||
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_total_mileage_km | first=0 | latest=0 | sample_count=7
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
|
||||
|
||||
删除后等待真实流量窗口:
|
||||
JT808 2026-07-01 metric rows = 0
|
||||
```
|
||||
|
||||
清理原因:
|
||||
|
||||
- `LKLG7C4E3NA774736` 的 12345.6 来自此前生产连通性合成回放,不是真实平台上报。
|
||||
- `LKLG7C4E2NA774775` 的 0 指标来自旧逻辑污染,当前 stat-writer 已跳过非正总里程。
|
||||
|
||||
Kafka spool:
|
||||
|
||||
```text
|
||||
23:24 spool_count=904
|
||||
23:25 spool_count=862
|
||||
23:28 spool_count=722
|
||||
new_spool_1min=0
|
||||
```
|
||||
|
||||
说明:spool 正在回放下降,且最近 1 分钟没有新增文件;生产 gateway 日志未见 `Incorrect string value`、Kafka publish error 或 replay error。
|
||||
|
||||
## 2026-07-01 23:34 每日指标查询 API 复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`56f4811`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-56f4811-20260701233311`
|
||||
|
||||
新增接口:
|
||||
|
||||
```text
|
||||
GET /api/stats/daily-metrics
|
||||
```
|
||||
|
||||
查询参数:
|
||||
|
||||
```text
|
||||
vin 可选
|
||||
protocol 可选,GB32960 / JT808 / YUTONG_MQTT
|
||||
metricKey 可选,daily_mileage_km / daily_total_mileage_km
|
||||
dateFrom 可选,YYYY-MM-DD
|
||||
dateTo 可选,YYYY-MM-DD
|
||||
limit 可选,默认 50,最大 1000
|
||||
offset 可选,默认 0
|
||||
```
|
||||
|
||||
生产可访问样例:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01&limit=10
|
||||
```
|
||||
|
||||
返回确认:
|
||||
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"vin": "LB9A32A21R0LS1707",
|
||||
"stat_date": "2020-07-01",
|
||||
"protocol": "GB32960",
|
||||
"metric_key": "daily_mileage_km",
|
||||
"metric_value": 0,
|
||||
"metric_unit": "km",
|
||||
"first_total_mileage_km": 53490.9,
|
||||
"latest_total_mileage_km": 53490.9,
|
||||
"sample_count": 3,
|
||||
"calculation_method": "TOTAL_MILEAGE_DIFF",
|
||||
"created_at": "2026-07-01 22:14:13",
|
||||
"updated_at": "2026-07-01 22:28:25"
|
||||
},
|
||||
{
|
||||
"vin": "LB9A32A21R0LS1707",
|
||||
"stat_date": "2020-07-01",
|
||||
"protocol": "GB32960",
|
||||
"metric_key": "daily_total_mileage_km",
|
||||
"metric_value": 53490.9,
|
||||
"metric_unit": "km",
|
||||
"first_total_mileage_km": 53490.9,
|
||||
"latest_total_mileage_km": 53490.9,
|
||||
"sample_count": 3,
|
||||
"calculation_method": "TOTAL_MILEAGE_DIFF",
|
||||
"created_at": "2026-07-01 22:14:13",
|
||||
"updated_at": "2026-07-01 22:28:25"
|
||||
}
|
||||
],
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
"total": 2
|
||||
}
|
||||
```
|
||||
|
||||
JT808 真实生产查询:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/stats/daily-metrics?protocol=JT808&dateFrom=2026-07-01&dateTo=2026-07-01&limit=10
|
||||
```
|
||||
|
||||
返回 `total=0`。原因是此前合成回放产生的测试指标已清理,当前真实 808 已绑定 VIN 的车辆仍上报 `total_mileage_km=0`,stat-writer 按规则不写非正总里程指标。
|
||||
|
||||
实时接口回归:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
```
|
||||
|
||||
返回 `online=true`,说明新增 stats API 没有影响 Redis 实时查询。
|
||||
|
||||
发布后 spool 观察:
|
||||
|
||||
```text
|
||||
发布后立即:spool_count=1101,new_spool_1min=102
|
||||
等待 60 秒:spool_count=1033,new_spool_1min=0
|
||||
```
|
||||
|
||||
说明:本次发布重启窗口产生的 spool 正在回放下降,且最近 1 分钟没有新增;gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。
|
||||
|
||||
## 2026-07-01 23:50 生产端口接管与 RAW 查询 API 复验
|
||||
|
||||
部署版本:
|
||||
|
||||
- Git commit:`5937132`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5937132-20260701234220`
|
||||
|
||||
生产端口状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway 0.0.0.0:808->808/tcp
|
||||
go-vehicle-gateway 0.0.0.0:32960->32960/tcp
|
||||
```
|
||||
|
||||
旧 Java 容器状态:
|
||||
|
||||
```text
|
||||
gb32960-ingest-app restart=no status=exited
|
||||
jt808-ingest-app restart=no status=exited
|
||||
yutong-mqtt-app restart=no status=exited
|
||||
vehicle-history-app restart=no status=exited
|
||||
vehicle-analytics-app restart=no status=exited
|
||||
vehicle-state-app restart=no status=exited
|
||||
telemetry-field-parser-app restart=no status=exited
|
||||
```
|
||||
|
||||
生产连接确认:
|
||||
|
||||
- JT808 端口 `808` 已收到外部连接:`222.66.200.68`、`115.231.168.135`。
|
||||
- GB32960 端口 `32960` 已收到外部连接:`8.134.95.166`、`117.160.0.65`。
|
||||
- ECS 当前没有 `8089` 监听。
|
||||
|
||||
RAW 查询 API:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames
|
||||
```
|
||||
|
||||
生产可访问样例:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
|
||||
http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
|
||||
```
|
||||
|
||||
接口复验结果:
|
||||
|
||||
```text
|
||||
JT808_RAW:
|
||||
total=1
|
||||
protocol=JT808
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
message_id=512
|
||||
message_id_hex=0x0200
|
||||
parse_status=OK
|
||||
raw_hex_len=94
|
||||
parsed_json_len=651
|
||||
fields_json_len=186
|
||||
|
||||
GB32960_RAW:
|
||||
total=1
|
||||
protocol=GB32960
|
||||
vin=LB9A32A21R0LS1707
|
||||
message_id=2
|
||||
message_id_hex=0x0002
|
||||
parse_status=OK
|
||||
raw_hex_len=1548
|
||||
parsed_json_len=3698
|
||||
fields_json_len=563
|
||||
```
|
||||
|
||||
同轮回归验证:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
online=true
|
||||
protocols=["JT808"]
|
||||
|
||||
GET /api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2026-07-01&limit=1
|
||||
total=1
|
||||
protocol=GB32960
|
||||
metric_key=daily_mileage_km
|
||||
calculation_method=TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- RAW 查询已返回 `raw_hex`、`parsed_json`、`fields_json`,可以用于查看完整解析结果。
|
||||
- 日统计和实时查询在生产端口接管后仍可用。
|
||||
|
||||
## 2026-07-02 00:00 原生 systemd 部署与宇通 MQTT 复验
|
||||
|
||||
根据本次 goal 的新要求,Go 接入链路后续不再使用 Docker 部署。本轮已中断 Docker 镜像构建路径,改为 ECS 原生 Linux 二进制 + systemd。
|
||||
|
||||
部署版本:
|
||||
|
||||
- Git commit:`594ab2d`
|
||||
- 部署目录:`/opt/lingniu-go-native`
|
||||
- 当前 release:`/opt/lingniu-go-native/releases/594ab2d`
|
||||
- 当前指针:`/opt/lingniu-go-native/current -> /opt/lingniu-go-native/releases/594ab2d`
|
||||
|
||||
systemd 服务:
|
||||
|
||||
```text
|
||||
lingniu-go-gateway.service active
|
||||
lingniu-go-history-writer.service active
|
||||
lingniu-go-stat-writer.service active
|
||||
lingniu-go-realtime-api.service active
|
||||
```
|
||||
|
||||
端口确认:
|
||||
|
||||
```text
|
||||
[::]:32960 gateway
|
||||
[::]:808 gateway
|
||||
[::]:20210 realtime-api
|
||||
```
|
||||
|
||||
Go Docker 容器已停止:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway Exited
|
||||
go-history-writer Exited
|
||||
go-stat-writer Exited
|
||||
go-realtime-api Exited
|
||||
```
|
||||
|
||||
宇通 MQTT:
|
||||
|
||||
```text
|
||||
yutong mqtt client started
|
||||
mqtt subscribed broker=ssl://cpxlm.axxc.cn:38883 topic=/ytforward/shln/+ qos=1
|
||||
```
|
||||
|
||||
说明:MQTT broker 存在短连接 `EOF` 后自动重连现象,但订阅后已收到真实数据并落 RAW。
|
||||
|
||||
RAW 查询复验:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=3
|
||||
|
||||
total=3
|
||||
protocol=YUTONG_MQTT
|
||||
vin=LMRKH9AC1R1004131
|
||||
vehicle_key=LMRKH9AC1R1004131
|
||||
parse_status=OK
|
||||
parsed_json_len=1104
|
||||
fields_json_len=116
|
||||
```
|
||||
|
||||
同轮回归:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
|
||||
total=1
|
||||
parse_status=OK
|
||||
|
||||
GET /api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
|
||||
total=1
|
||||
parse_status=OK
|
||||
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
online=true
|
||||
protocols=["JT808"]
|
||||
```
|
||||
|
||||
本轮落地文件:
|
||||
|
||||
- `deploy/systemd/README.md`
|
||||
- `deploy/systemd/lingniu-go-gateway.service`
|
||||
- `deploy/systemd/lingniu-go-history-writer.service`
|
||||
- `deploy/systemd/lingniu-go-stat-writer.service`
|
||||
- `deploy/systemd/lingniu-go-realtime-api.service`
|
||||
|
||||
## 2026-07-02 00:04 宇通 MQTT EOF 诊断
|
||||
|
||||
现象:
|
||||
|
||||
Go 原生部署后,`lingniu-go-gateway` 日志中出现多次:
|
||||
|
||||
```text
|
||||
mqtt connection lost error=EOF
|
||||
mqtt connection lost error=write: broken pipe
|
||||
mqtt subscribed topic=/ytforward/shln/+ qos=1
|
||||
```
|
||||
|
||||
10 分钟窗口统计:
|
||||
|
||||
```text
|
||||
mqtt subscribed = 33
|
||||
mqtt connection lost = 32
|
||||
```
|
||||
|
||||
对照旧 Java `yutong-mqtt-app` 历史日志,停止前也存在同类行为:
|
||||
|
||||
```text
|
||||
mqtt endpoint [yutong] received topic=/ytforward/shln/1 bytes=574
|
||||
mqtt endpoint [yutong] connection lost
|
||||
Caused by: java.io.EOFException
|
||||
mqtt endpoint [yutong] re-subscribed topic=/ytforward/shln/+ qos=1
|
||||
mqtt endpoint [yutong] received topic=/ytforward/shln/3 bytes=586
|
||||
```
|
||||
|
||||
判断:
|
||||
|
||||
- EOF/短连接不是 Go TLS 支持或 systemd 原生部署新引入的问题。
|
||||
- 旧 Java 和新 Go 都表现为:订阅、收到数据、broker 断开、自动重连。
|
||||
- 当前链路可持续收到宇通 MQTT 数据并写入 TDengine RAW。
|
||||
|
||||
复验:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=1
|
||||
|
||||
total=1
|
||||
protocol=YUTONG_MQTT
|
||||
vin=LMRKH9ACXR1004094
|
||||
parse_status=OK
|
||||
parsed_json_len=1080
|
||||
fields_json_len=115
|
||||
```
|
||||
|
||||
运维建议:
|
||||
|
||||
- 单条 EOF 不作为不可用判断。
|
||||
- 以 `YUTONG_MQTT` RAW 入库增长、最近成功 `mqtt subscribed` 时间、服务 `systemctl is-active` 作为可用性判断。
|
||||
- 如果 EOF 频率继续升高且 RAW 不再增长,再联系宇通侧确认 broker 长连接策略、同一 clientId 并发限制或网络出口策略。
|
||||
158
docs/operations/go-vehicle-gateway-verification.md
Normal file
158
docs/operations/go-vehicle-gateway-verification.md
Normal file
@@ -0,0 +1,158 @@
|
||||
# Go Vehicle Gateway 验证手册
|
||||
|
||||
本文档用于验证 Go 重构运行面:
|
||||
|
||||
- `go-vehicle-gateway`:GB32960 TCP、JT808 TCP、宇通 MQTT 接入。
|
||||
- `go-history-writer`:Kafka RAW 写 TDengine。
|
||||
- `go-stat-writer`:Kafka RAW 写 MySQL 每日指标。
|
||||
- `go-realtime-api`:Kafka unified event 写 Redis,并提供实时查询 API。
|
||||
|
||||
## 本地构建验证
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./...
|
||||
go build ./cmd/gateway
|
||||
go build ./cmd/history-writer
|
||||
go build ./cmd/stat-writer
|
||||
go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
## 生产原生部署验证
|
||||
|
||||
```bash
|
||||
python3 tools/go_prod_acceptance.py --date 2026-07-02
|
||||
```
|
||||
|
||||
该命令会聚合执行:
|
||||
|
||||
- `go_systemd_prod_smoke.py`:应用 ECS systemd 服务 active/enabled、端口归属和 gateway spool
|
||||
- `go_kafka_prod_smoke.py`:Kafka topic 和消费组 lag
|
||||
- `go_native_prod_smoke.py`:HTTP API、RAW `parsed_json`、历史核心表 `frame_id` 回链、每日指标公式、Redis 实时新鲜度、TDengine/MySQL/Redis 数据链路
|
||||
|
||||
如需单独检查 systemd:
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
```
|
||||
|
||||
脚本通过 SSH 检查:
|
||||
|
||||
- `lingniu-go-gateway.service`
|
||||
- `lingniu-go-history-writer.service`
|
||||
- `lingniu-go-stat-writer.service`
|
||||
- `lingniu-go-realtime-api.service`
|
||||
- 上述服务均为 `active` 且 `enabled`
|
||||
- `808`、`32960` 由 `gateway` 监听
|
||||
- `20210` 由 `realtime-api` 监听
|
||||
- `/opt/lingniu-go-native/spool/gateway` 无待回放文件
|
||||
|
||||
## 生产环境变量
|
||||
|
||||
当前 goal 的生产面使用 ECS 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。以下环境变量维护在 `/opt/lingniu-go-native/env/*.env`:
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=172.17.111.56:9092
|
||||
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
|
||||
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
|
||||
IDENTITY_MYSQL_DSN=${MYSQL_DSN}
|
||||
VEHICLE_IDENTITY_TABLE=vehicle_identity_binding
|
||||
REDIS_ADDR=r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379
|
||||
REDIS_PASSWORD=<password>
|
||||
REDIS_DB=50
|
||||
```
|
||||
|
||||
宇通 MQTT 开启时再配置:
|
||||
|
||||
```bash
|
||||
YUTONG_MQTT_ENABLED=true
|
||||
YUTONG_MQTT_URI=<mqtt-uri>
|
||||
YUTONG_MQTT_TOPICS=/ytforward/shln/+
|
||||
YUTONG_MQTT_CLIENT_ID=lingniu-go-yutong-mqtt
|
||||
YUTONG_MQTT_USERNAME=<username>
|
||||
YUTONG_MQTT_PASSWORD=<password>
|
||||
```
|
||||
|
||||
## ECS 进程检查
|
||||
|
||||
```bash
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
```
|
||||
|
||||
## Kafka 验证
|
||||
|
||||
优先使用 smoke 脚本做可重复检查:
|
||||
|
||||
```bash
|
||||
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
|
||||
```
|
||||
|
||||
脚本会检查 `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1`、`vehicle.event.go.unified.v1` 是否存在,并汇总 `go-history-writer`、`go-stat-writer`、`go-realtime-api` 的消费 lag。运行环境需要 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
手工核对命令:
|
||||
|
||||
```bash
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.jt808.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.gb32960.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.yutong-mqtt.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.event.go.unified.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-history-writer
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-stat-writer
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-realtime-api
|
||||
```
|
||||
|
||||
## TDengine 验证
|
||||
|
||||
```sql
|
||||
USE lingniu_vehicle_ts;
|
||||
SHOW STABLES;
|
||||
SELECT COUNT(*) FROM raw_frames;
|
||||
SELECT COUNT(*) FROM vehicle_locations;
|
||||
SELECT COUNT(*) FROM vehicle_mileage_points;
|
||||
```
|
||||
|
||||
## MySQL 验证
|
||||
|
||||
```sql
|
||||
SELECT protocol, metric_key, COUNT(*)
|
||||
FROM vehicle_daily_metric
|
||||
GROUP BY protocol, metric_key;
|
||||
|
||||
SELECT *
|
||||
FROM vehicle_daily_metric
|
||||
WHERE metric_key IN ('daily_mileage_km', 'daily_total_mileage_km')
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
## Redis 实时查询
|
||||
|
||||
```bash
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>'
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/online'
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/protocols/JT808'
|
||||
```
|
||||
|
||||
## 验收证据
|
||||
|
||||
正式切换前至少记录:
|
||||
|
||||
- 一个真实 GB32960 VIN 的 RAW、location、mileage point、Redis snapshot。
|
||||
- 一个真实 JT808 VIN/phone 的 RAW、location、mileage point、daily metric、Redis snapshot。
|
||||
- 一个真实宇通 MQTT VIN 或 device_id 的 RAW、Redis snapshot。
|
||||
- `go-history-writer` 和 `go-stat-writer` 重启后 Kafka offset 能继续推进。
|
||||
34
docs/operations/jt808-daily-mileage-runbook.md
Normal file
34
docs/operations/jt808-daily-mileage-runbook.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# JT808 Daily Mileage Streaming
|
||||
|
||||
## Scope
|
||||
|
||||
The analytics app can calculate daily mileage from JT808 telemetry only. It consumes `vehicle.event.jt808.v1` and saves the current daily result into the common `vehicle-stat` metric repository.
|
||||
|
||||
Only supported message backbone: Kafka.
|
||||
|
||||
## Storage
|
||||
|
||||
Runtime state: none outside `vehicle_stat_metric`. There is no separate JT808 daily-mileage table. The derived value is stored as one `daily_mileage_km` metric row in the common JDBC/MySQL table `vehicle_stat_metric`; the local-day minimum and maximum GPS total mileage values are kept on that same row as calculation source columns.
|
||||
|
||||
The JT808 daily-mileage value is calculated from the GPS total mileage reported in location additional information:
|
||||
|
||||
```text
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
calculation_method = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
The first valid JT808 location point for a vehicle and local day stores both calculation endpoints on the metric row and writes `daily_mileage_km=0.0`. Later ordered or replayed points update the lower endpoint when the reported GPS total mileage is smaller, update the upper endpoint when it is larger, and rewrite the derived `daily_mileage_km` on the same row.
|
||||
|
||||
## Runtime Settings
|
||||
|
||||
Set these in Portainer or Nacos, without committing secrets:
|
||||
|
||||
```text
|
||||
KAFKA_TOPIC_JT808_EVENT=vehicle.event.jt808.v1
|
||||
VEHICLE_STAT_JT808_MILEAGE_ENABLED=true
|
||||
MYSQL_JDBC_URL=<jdbc-url>
|
||||
MYSQL_USERNAME=<user>
|
||||
MYSQL_PASSWORD=<password>
|
||||
```
|
||||
|
||||
Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `daily_mileage_km` metric row.
|
||||
351
docs/operations/vehicle-ingest-tdengine-verification.md
Normal file
351
docs/operations/vehicle-ingest-tdengine-verification.md
Normal file
@@ -0,0 +1,351 @@
|
||||
# Vehicle Ingest TDengine 链路验收手册
|
||||
|
||||
这份手册用于验收当前 GB32960/JT808/Yutong MQTT 接入重构链路:
|
||||
|
||||
1. 协议 App 接收 TCP 报文。
|
||||
2. 协议 App 归档原始帧元数据,并把事件信封发布到 Kafka。
|
||||
3. `vehicle-history-app` 消费 Kafka。
|
||||
4. history 写入 raw、location 到 TDengine;不写 telemetry_fields。
|
||||
5. Swagger/API 通过 TDengine 做历史分页查询。
|
||||
|
||||
每一步都必须执行命令并检查输出后,才能标记为已验证。
|
||||
|
||||
## 必需运行参数
|
||||
|
||||
启动服务前先设置:
|
||||
|
||||
```bash
|
||||
export KAFKA_BROKERS=114.55.58.251:9092
|
||||
export TDENGINE_HISTORY_ENABLED=true
|
||||
export TDENGINE_HISTORY_DATABASE=vehicle_ts
|
||||
export TDENGINE_JDBC_URL='jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts'
|
||||
export TDENGINE_DRIVER_CLASS_NAME=com.taosdata.jdbc.ws.WebSocketDriver
|
||||
export TDENGINE_USERNAME=root
|
||||
export TDENGINE_PASSWORD='<tdengine-password>'
|
||||
export TDENGINE_MIN_IDLE=0
|
||||
export TDENGINE_MAX_POOL_SIZE=32
|
||||
```
|
||||
|
||||
`TDENGINE_MIN_IDLE=0` 用于减少冷启动时 TDengine 连接重试日志。TDengine 地址稳定后,再按生产并发调大连接池。
|
||||
|
||||
高吞吐生产验收只验证 TDengine `raw_frames`、位置表和分页查询闭环。RAW 帧的 `parsedJson`、`metadataJson`、`rawUri` 直接进入 TDengine,GB32960 snapshot/fields 接口基于 `raw_frames` 与当前解析器即时返回结构化结果;接入服务写出的原始 `.bin` 只作为冷备复核材料。逐字段趋势宽表由独立字段解析服务消费 Kafka RAW/事件后维护,不由 history-app 写入。
|
||||
|
||||
## 构建
|
||||
|
||||
```bash
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
预期产物:
|
||||
|
||||
- `modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar`
|
||||
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
|
||||
- `modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar`
|
||||
- `modules/apps/vehicle-history-app/target/vehicle-history-app.jar`
|
||||
- `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar`
|
||||
|
||||
## ECS 服务状态验证
|
||||
|
||||
生产服务只在 ECS 上运行,通过 Portainer/Docker 管理。不要在本机用 `java -jar`、plist 或其它守护进程方式启动 GB32960、JT808、Yutong MQTT、history、analytics。
|
||||
|
||||
```bash
|
||||
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' \
|
||||
| egrep 'gb32960|jt808|yutong|vehicle-history|vehicle-analytics|NAMES'
|
||||
|
||||
ss -lntp | egrep ':(808|32960|20100|20200|20310|20400|20500)\b'
|
||||
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/v3/api-docs \
|
||||
| grep -E '/api/event-history/locations|/api/event-history/raw-frames'
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
预期:五个容器均为运行状态;健康检查为 `UP`;JT808/GB32960 TCP 端口在 ECS 上监听;OpenAPI 中包含通用位置分页查询和 RAW 帧查询接口。history-app 不暴露 `/api/event-history/telemetry/fields`,也不持续写入逐字段宽表。JT808 位置帧中有 GPS 总里程时,`vehicle-analytics-app` 按差值法写入 MySQL `vehicle_stat_metric` 的 `daily_mileage_km` 指标。
|
||||
|
||||
## 运行位置约束
|
||||
|
||||
生产身份绑定固定使用 MySQL,不再提供 file/memory/sqlite 运行时切换入口。生产服务固定运行在 ECS/Portainer,不提供本机守护进程模板。
|
||||
|
||||
- `jt808-ingest-app` 监听 TCP `808`,HTTP `20400`。
|
||||
- `gb32960-ingest-app` 监听 TCP `32960`,HTTP `20100`。
|
||||
- `yutong-mqtt-app` 监听 HTTP `20500`。
|
||||
- `vehicle-history-app` 监听 HTTP `20200`。
|
||||
- `vehicle-analytics-app` 监听 HTTP `20310`。
|
||||
- history 热查询只依赖 Kafka 和 TDengine `raw_frames`;history 不需要共享 archive volume。
|
||||
- GB32960/JT808/Yutong MQTT 接入服务可以继续使用 `SINK_ARCHIVE_PATH` 保存原始冷备,但这不是 history API 的实时查询前置条件。
|
||||
|
||||
JT808 注册身份绑定:
|
||||
|
||||
- 生产需要把 0x0100 注册信息维护到 MySQL 时,只需要配置 `VEHICLE_IDENTITY_MYSQL_*` 连接参数。
|
||||
- 需要提供 `VEHICLE_IDENTITY_MYSQL_JDBC_URL`、`VEHICLE_IDENTITY_MYSQL_USERNAME`、`VEHICLE_IDENTITY_MYSQL_PASSWORD`。
|
||||
- 服务启动时自动创建 `vehicle_identity_binding` 和 `vehicle_identity_binding_registration` 两张表。`vehicle_identity_binding` 只维护 `plate`、`vin` 两列;`vehicle_identity_binding_registration` 按 `protocol + phone` 保存 JT808 0x0100 注册字段。
|
||||
- 注册帧无真实 VIN 时也会写入 registration 表,`vin` 默认为 `unknown`。外部系统识别车辆后只需要维护车牌和 VIN,例如:`INSERT INTO vehicle_identity_binding (plate, vin) VALUES ('沪A61559F', 'LNVIN000000000001') ON DUPLICATE KEY UPDATE vin = VALUES(vin);`
|
||||
- MySQL 绑定在启动时加载到内存索引,`plate/vin` 反写后会按 `VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL` 周期刷新,默认 `60s`;帧解析热路径只查内存,不按帧访问 MySQL。
|
||||
- 后续同一终端上报 raw/event 时,服务会用 registration 表里的 `phone/device_id/plate` 加绑定表里的 `plate/vin` 解析成真实 VIN。
|
||||
|
||||
健康检查:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
API / Swagger:
|
||||
|
||||
- JT808 ingest health: `http://127.0.0.1:20400/actuator/health`
|
||||
- GB32960 ingest health: `http://127.0.0.1:20100/actuator/health`
|
||||
- Yutong MQTT ingest health: `http://127.0.0.1:20500/actuator/health`
|
||||
- History query: `http://127.0.0.1:20200/swagger-ui/index.html`
|
||||
- Analytics metrics: `http://127.0.0.1:20310/swagger-ui/index.html`
|
||||
|
||||
## Kafka Topic 验证
|
||||
|
||||
历史服务必须消费事件 topic 和 raw topic:
|
||||
|
||||
- `vehicle.event.gb32960.v1`
|
||||
- `vehicle.raw.gb32960.v1`
|
||||
- `vehicle.event.jt808.v1`
|
||||
- `vehicle.raw.jt808.v1`
|
||||
- `vehicle.event.mqtt-yutong.v1`
|
||||
- `vehicle.raw.mqtt-yutong.v1`
|
||||
|
||||
使用 Kafka CLI 或管理工具检查 topic 是否存在,并确认 history consumer group 在测试流量后没有持续堆积。默认 `KAFKA_GROUP_HISTORY=vehicle-history` 时,实际 group 会拆成:
|
||||
|
||||
- `vehicle-history-gb32960-event`
|
||||
- `vehicle-history-gb32960-raw`
|
||||
- `vehicle-history-jt808-event`
|
||||
- `vehicle-history-jt808-raw`
|
||||
- `vehicle-history-yutong-mqtt-event`
|
||||
- `vehicle-history-yutong-mqtt-raw`
|
||||
|
||||
## JT808 实时转发验收
|
||||
|
||||
如果外部平台已经把 JT808 报文转发到 ECS TCP `808`,至少观察 60 秒日志和 Kafka 数据。
|
||||
|
||||
必须拿到以下证据:
|
||||
|
||||
- `jt808-ingest-app` 日志显示接入连接或解析到上游消息 ID。
|
||||
- Kafka 的 `vehicle.event.jt808.v1` 和 `vehicle.raw.jt808.v1` 有新增记录。
|
||||
- `vehicle-history-app` 日志没有持续出现 consumer 或 TDengine 写入失败。
|
||||
|
||||
消费完成后,使用已知终端手机号查询:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/jt808/locations?phone=<phone>&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
```
|
||||
|
||||
预期:响应包含 `items`,并且每条记录包含 `eventTime`、`phone`、`longitude`、`latitude`、`speedKmh`、`rawUri`、`metadataJson`。
|
||||
|
||||
## GB32960 Snapshot / 字段投影验收
|
||||
|
||||
默认高吞吐模式下,GB32960 全字段查询不依赖 telemetry_fields 宽表。history 先查 TDengine `raw_frames`,直接使用 RAW 行中的 `parsedJson`/`parsedFields` 生成 snapshot 和字段投影:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots?vin=<vin>&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin=<vin>&fields=VEHICLE.speedKmh,VEHICLE.totalMileageKm,POSITION_V2016.longitude,POSITION_V2016.latitude&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
```
|
||||
|
||||
预期:snapshot 返回 `sourceFrames.rawArchiveUri` 和解析后的 `blocks`;字段投影返回所选字段。新增协议字段后,先通过 replay 或专用解析任务补写 `raw_frames.parsedJson`,再基于历史 RAW 结构化结果查询。
|
||||
|
||||
## TDengine 直接检查
|
||||
|
||||
使用 TDengine CLI 或 JDBC 客户端检查超级表和子表是否创建:
|
||||
|
||||
```sql
|
||||
USE vehicle_ts;
|
||||
SHOW STABLES;
|
||||
SELECT COUNT(*) FROM raw_frames;
|
||||
SELECT COUNT(*) FROM vehicle_locations;
|
||||
SELECT COUNT(*) FROM jt808_locations;
|
||||
```
|
||||
|
||||
预期:
|
||||
|
||||
- `raw_frames`、`vehicle_locations`、`jt808_locations` 存在;有对应协议实时流量后,计数持续增长。
|
||||
- 字段趋势宽表不属于 history-app 验收范围,由独立字段解析服务负责。
|
||||
- 有实时流量后,计数持续增加。
|
||||
|
||||
## 失败语义
|
||||
|
||||
- TDengine 不可达时,TDengine 查询接口应该返回 HTTP `503`,消息类似 `tdengine history query failed`。
|
||||
- TDengine 未启用时,TDengine 专属接口不应该出现在 OpenAPI 中。
|
||||
- 路径不存在时,API 层应该返回 HTTP `404`,而不是泛化的存储失败。
|
||||
|
||||
## 2026-06-29 Superseded Local Live Verification Record
|
||||
|
||||
该记录仅保留为解析和存储行为的历史证据;生产运行位置已经收敛到 ECS/Portainer。
|
||||
|
||||
当时参与验证的服务:
|
||||
|
||||
- `jt808-ingest-app`:TCP `808`,HTTP `20400`
|
||||
- `gb32960-ingest-app`:TCP `32960`,HTTP `20100`
|
||||
- `vehicle-history-app`:HTTP `20200`
|
||||
|
||||
当次健康检查均返回 `{"status":"UP"}`。
|
||||
|
||||
可重复运行 live 验收工具:
|
||||
|
||||
```bash
|
||||
python3 tools/vehicle_ingest_live_verify.py \
|
||||
--tdengine-rest-url 'http://115.29.185.82:6041/rest/sql/vehicle_ts' \
|
||||
--tdengine-username root \
|
||||
--tdengine-password '<tdengine-password>' \
|
||||
--history-base-url 'http://127.0.0.1:20200' \
|
||||
--date-from '2026-06-29 00:00:00' \
|
||||
--date-to '2026-06-29 23:59:59' \
|
||||
--jt808-peer-like '222.66.200.68:%' \
|
||||
--gb32960-peer-like '115.29.187.205:%'
|
||||
```
|
||||
|
||||
该工具输出 `pass`、`warn` 或 `fail`:当前正式 GB32960 只有平台登录、没有车辆 `0x02` 时会输出 `warn`,避免把平台在线误判为车辆数据在线。正式车辆数据验收时增加 `--require-gb32960-vehicle-realtime`,缺少非测试 VIN 的 `0x02` 会直接返回非 0 退出码。
|
||||
|
||||
JT808 真实转发链路已验证:
|
||||
|
||||
- ECS TCP `808` 有外部平台 `222.66.200.68` 持续发送 JT808 报文。
|
||||
- `vehicle.raw.jt808.v1` 和 `vehicle.event.jt808.v1` 被 history 消费。
|
||||
- `raw_frames` 中 `protocol='JT808'` 的真实行数已超过 `36000`。
|
||||
- 外部平台真实 0x0100 注册帧已超过 `347` 条,真实 0x0200 位置帧已超过 `12400` 条。
|
||||
- `jt808_locations` 中位置行数已超过 `8783`。
|
||||
- 当前高吞吐默认链路不维护 `telemetry_fields`;JT808 明细验证以 `raw_frames`、`vehicle_locations` 和 API 分页结果为准。
|
||||
- 分页 API 已用终端号 `13079963301` 验证第一页和 nextCursor 第二页,样例位置包含 `longitude=118.913846`、`latitude=31.927309`、`statusFlag=3` 和 `archive://2026/06/29/JT808/...` 原始帧引用。
|
||||
- 注册帧样例:终端号 `13079963320`、deviceId `9963320`、deviceType `SEG-9888G`、plate `沪A61559F`、maker `70112`、province `31`、city `113`、plateColor `2`。生产 VIN 反写依赖 MySQL identity store,后续 raw/event 的 VIN 解析也从 MySQL 绑定读取。
|
||||
|
||||
## JT808 ECS smoke / 轻压测工具
|
||||
|
||||
仓库提供 `tools/jt808_e2e_smoke.py`,用于重复验证 TCP `808` 到 TDengine history 的闭环:
|
||||
|
||||
```bash
|
||||
export TDENGINE_REST_URL='http://<tdengine-host>:6041/rest/sql/vehicle_ts'
|
||||
export TDENGINE_USERNAME='root'
|
||||
export TDENGINE_PASSWORD='<tdengine-password>'
|
||||
|
||||
python3 tools/jt808_e2e_smoke.py \
|
||||
--connection-mode session \
|
||||
--start-phone 13079962000 \
|
||||
--frames 3 \
|
||||
--expect-history-count 3 \
|
||||
--verify-pagination \
|
||||
--archive-root "$PROJECT_ROOT/data/archive-jt808" \
|
||||
--tdengine-rest-url "$TDENGINE_REST_URL" \
|
||||
--tdengine-username "$TDENGINE_USERNAME" \
|
||||
--tdengine-password "$TDENGINE_PASSWORD"
|
||||
```
|
||||
|
||||
默认 `--connection-mode per-frame`,适合单帧连接 smoke。生产验收建议使用 `--connection-mode session` 覆盖同一连接内连续位置帧。脚本会输出发送数量、history 可见数量、分页验证结果、raw archive 检查结果和 `tdengineRawFrames`。`tdengineRawFrames` 表示本次可见位置记录中的唯一 `rawUri` 有多少个已确认写入 TDengine `raw_frames`。
|
||||
- 合成 0x0100 注册帧终端号 `13079969999` 已验证:TCP `808` 返回 `0x8100` 注册 ACK,`raw_frames.metadata_json` 包含 `jt808.register.province`、`jt808.register.city`、`jt808.register.maker`、`jt808.register.deviceType`、`jt808.register.deviceId`、`jt808.register.plateColor`、`jt808.register.plate`,对应 raw archive 文件存在。
|
||||
|
||||
复查 SQL:
|
||||
|
||||
```sql
|
||||
USE vehicle_ts;
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM jt808_locations WHERE protocol = 'JT808';
|
||||
SELECT event_time, vehicle_key, phone, message_id, raw_uri
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'JT808'
|
||||
ORDER BY event_time DESC
|
||||
LIMIT 5;
|
||||
SELECT ts, frame_id, phone, message_id, metadata_json, raw_uri
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'JT808' AND phone = '13079969999' AND message_id = 256
|
||||
ORDER BY ts DESC
|
||||
LIMIT 3;
|
||||
```
|
||||
|
||||
复查 API:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/jt808/locations?phone=13079963301&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&order=DESC&limit=3'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/jt808/raw-frames?phone=13079963320&messageId=256&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&order=DESC&limit=3'
|
||||
```
|
||||
|
||||
`/api/event-history/jt808/raw-frames` 用于排查注册、鉴权、心跳、位置等原始帧索引。返回项会保留 `messageId`、`messageIdHex`、`rawUri`、`parseStatus`、`peer`、`vin`、`phone` 和完整 `metadataJson`;注册帧的 `metadataJson` 包含 `jt808.register.province`、`jt808.register.city`、`jt808.register.maker`、`jt808.register.deviceType`、`jt808.register.deviceId`、`jt808.register.plateColor`、`jt808.register.plate`、`identitySource`、`identityResolved` 等字段。
|
||||
|
||||
GB32960 链路已用正式平台登录和合成实时帧验证:
|
||||
|
||||
- 正式平台 `115.29.187.205` 已连接 TCP `32960`,服务收到 `0x05 PLATFORM_LOGIN` 并返回 ACK;日志显示账号 `Hyundai`、策略 `ALLOW`。
|
||||
- 2026-06-29 正式平台登录 raw 已有 `3` 条,均落入 TDengine `raw_frames`,可通过 RAW 查询看到 `parsedJson` 中的 `PLATFORM_LOGIN` 解析结果。
|
||||
- 当前正式平台尚未发送车辆实时上报 `0x02`;非测试 VIN 的 `message_id=2` 计数为 `0`,因此不能把平台登录误判为车辆数据在线。
|
||||
- TCP `32960` 返回 GB32960 `2323` ACK。
|
||||
- `raw_frames` 中测试 VIN `LTEST202606290001` 已有 `2` 条 raw 记录。
|
||||
- 当前默认通过 GB32960 snapshot/fields 即时解析 `raw_frames` 中的 RAW JSON,不维护字段宽表增长。
|
||||
- latest raw 包含 `archive://2026/06/29/GB32960/LTEST202606290001/...`。
|
||||
- snapshots API 可查到 `speedKmh=62.4`、`totalMileageKm=223456.7`、`longitude=116.397128`、`latitude=39.916527`。
|
||||
|
||||
复查 SQL:
|
||||
|
||||
```sql
|
||||
USE vehicle_ts;
|
||||
SELECT COUNT(*) FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001';
|
||||
SELECT event_time, vin, raw_uri, frame_id
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001'
|
||||
ORDER BY event_time DESC
|
||||
LIMIT 5;
|
||||
SELECT ts, vin, message_id, event_time, raw_uri, peer, metadata_json
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND peer LIKE '115.29.187.205:%'
|
||||
ORDER BY ts DESC
|
||||
LIMIT 5;
|
||||
SELECT COUNT(*) FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND message_id = 2 AND vin <> '' AND vin NOT LIKE 'LTEST%';
|
||||
```
|
||||
|
||||
也可以直接运行仓库 smoke 工具验证 TCP `32960`、GB32960 snapshot/fields、TDengine `raw_frames`。如果需要复核冷备 `.bin`,再传入接入服务 archive 根目录:
|
||||
|
||||
```bash
|
||||
export TDENGINE_REST_URL='http://<tdengine-host>:6041/rest/sql/vehicle_ts'
|
||||
export TDENGINE_USERNAME='root'
|
||||
export TDENGINE_PASSWORD='<tdengine-password>'
|
||||
|
||||
python3 tools/gb32960_e2e_smoke.py \
|
||||
--archive-root "$PROJECT_ROOT/data/archive-jt808" \
|
||||
--tdengine-rest-url "$TDENGINE_REST_URL" \
|
||||
--tdengine-username "$TDENGINE_USERNAME" \
|
||||
--tdengine-password "$TDENGINE_PASSWORD"
|
||||
```
|
||||
|
||||
预期输出包含 `records >= 1`、`fieldCounts` 中关键字段均大于 0、`tdengineRawFrames` 等于本次可见唯一 `rawUri` 数量。传入 `--archive-root` 时,`archiveChecked >= 1` 说明冷备文件也能复核;这里的 `records` 字段表示 snapshot 数量,用于兼容早期脚本输出名。
|
||||
|
||||
复查 API:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots?vin=LTEST202606290001&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin=LTEST202606290001&fields=VEHICLE.speedKmh,VEHICLE.totalMileageKm,POSITION_V2016.longitude,POSITION_V2016.latitude&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
```
|
||||
|
||||
## 当前注意事项
|
||||
|
||||
- 当前生产链路使用 TDengine WebSocket JDBC,例如 `jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts`。
|
||||
- `KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest` 适合生产接入新流量;如果要回放历史 Kafka 数据,需要切换 consumer group 或重置 offset。
|
||||
- `vehicle-history-app` raw 和 event 使用不同 consumer binding;raw consumer 必须开启,否则 `raw_frames` 不会持续增长。
|
||||
- JT808 设备没有 VIN 映射时,`vehicle_key` 使用 `jt808:<phone>`;启用 MySQL identity store 后,registration 表中已反写 VIN 的 phone/deviceId/plate 会用于后续 raw/event 的 VIN 解析。
|
||||
- GB32960 正式对端目前只验证到平台登录;等待真实车辆 `0x02` 到达后,再用非测试 VIN 复查 `vehicle_locations`、snapshot/fields API 和导出能力。
|
||||
@@ -543,7 +543,7 @@ EOF
|
||||
Run: `mvn test -q`
|
||||
Expected: BUILD SUCCESS。重点关注:
|
||||
- `protocol-gb32960` 模块全绿(Golden、FullBlocks、Isolation、GuangdongFcEndToEnd、Decoder、Mapper、所有 parser 单测、profile)
|
||||
- 其它模块(`ingest-core`、`sink-mq` 等)不受影响(无改动应自动通过)
|
||||
- 其它模块(`ingest-core`、`sink-kafka` 等)不受影响(无改动应自动通过)
|
||||
|
||||
若有红,停下来诊断。
|
||||
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
> **Superseded:** This 2026-06-23 DuckDB hot-store plan is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine stores hot history, `sink-archive`
|
||||
> owns raw bytes, and `event-file-store` / DuckDB are no longer part of the
|
||||
> current build surface.
|
||||
|
||||
# GB32960 History DuckDB Hot Store Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
> **Superseded:** This 2026-06-23 service-split plan is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: GB32960, JT808, and Yutong MQTT are the
|
||||
> active ingest apps, history is backed by TDengine, and Xinda/event-file-store
|
||||
> paths have been removed from the current build surface.
|
||||
|
||||
# GB32960 Service Split Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
@@ -13,12 +20,14 @@
|
||||
## Current-State Notes
|
||||
|
||||
- Existing all-in-one entrypoint: `modules/apps/bootstrap-all/src/main/java/com/lingniu/ingest/bootstrap/IngestApplication.java`.
|
||||
- Existing all-in-one module: `modules/apps/bootstrap-all/pom.xml` imports protocols, sinks, history, state, stat, command gateway, MQTT, Xinda, and JT modules.
|
||||
- Existing all-in-one module at the time of this plan imported protocols,
|
||||
sinks, history, state, stat, command gateway, MQTT, and JT modules. Xinda was
|
||||
later removed from source and build configuration.
|
||||
- Existing GB32960 handler: `modules/protocols/protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/inbound/Gb32960ChannelHandler.java`.
|
||||
- Existing handler currently writes report ACK before `dispatcher.dispatch(rf)`.
|
||||
- Existing `DisruptorEventBus` publishes to sinks asynchronously and does not wait for sink completion.
|
||||
- Existing `KafkaEventSink.accepts()` rejects `VehicleEvent.RawArchive`.
|
||||
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.mq.topics`.
|
||||
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.kafka.topics`.
|
||||
- Existing history consumer logic: `modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java`.
|
||||
- Existing analytics consumers: `VehicleStateEnvelopeIngestor` and `VehicleStatEnvelopeIngestor`.
|
||||
- There is an unrelated working-tree change in `modules/apps/bootstrap-all/src/main/resources/application.yml` for GB32960 diagnostics capacity. Do not revert it. Do not include it in service-split commits unless the user explicitly asks.
|
||||
@@ -48,11 +57,11 @@ Create app modules:
|
||||
Modify shared build and Kafka contract files:
|
||||
|
||||
- `pom.xml`
|
||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
|
||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
|
||||
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
|
||||
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
|
||||
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java`
|
||||
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java`
|
||||
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java`
|
||||
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java`
|
||||
- `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java`
|
||||
- `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java`
|
||||
|
||||
Modify ACK/Kafka durability boundary files:
|
||||
|
||||
@@ -157,7 +166,7 @@ Create `modules/apps/gb32960-ingest-app/pom.xml`:
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -222,7 +231,7 @@ Create `modules/apps/vehicle-history-app/pom.xml`:
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -303,7 +312,7 @@ Create `modules/apps/vehicle-analytics-app/pom.xml`:
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -466,16 +475,19 @@ lingniu:
|
||||
rate-limit:
|
||||
per-vin-qps: 50
|
||||
session:
|
||||
store: ${SESSION_STORE:memory}
|
||||
store: ${SESSION_STORE:redis}
|
||||
ttl: ${SESSION_TTL:30m}
|
||||
identity:
|
||||
store: ${VEHICLE_IDENTITY_STORE:file}
|
||||
file:
|
||||
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
|
||||
store: ${VEHICLE_IDENTITY_STORE:mysql}
|
||||
mysql:
|
||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
|
||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||
sink:
|
||||
mq:
|
||||
kafka:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
compression-type: zstd
|
||||
linger-ms: 20
|
||||
@@ -568,9 +580,8 @@ lingniu:
|
||||
gb32960:
|
||||
enabled: false
|
||||
sink:
|
||||
mq:
|
||||
kafka:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
topics:
|
||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
@@ -666,9 +677,8 @@ lingniu:
|
||||
gb32960:
|
||||
enabled: false
|
||||
sink:
|
||||
mq:
|
||||
kafka:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
topics:
|
||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
@@ -754,7 +764,7 @@ import com.lingniu.ingest.eventfilestore.EventFileStore;
|
||||
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
@@ -770,7 +780,7 @@ class Gb32960IngestAppCompositionTest {
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.gb32960.enabled=true",
|
||||
"lingniu.ingest.gb32960.port=0",
|
||||
"lingniu.ingest.sink.mq.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=false",
|
||||
"lingniu.ingest.sink.archive.enabled=false",
|
||||
"lingniu.ingest.event-file-store.enabled=false",
|
||||
"lingniu.ingest.event-history.enabled=false",
|
||||
@@ -835,7 +845,7 @@ class VehicleHistoryAppCompositionTest {
|
||||
"lingniu.ingest.event-history.enabled=true",
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=false",
|
||||
"lingniu.ingest.sink.mq.enabled=false")
|
||||
"lingniu.ingest.sink.kafka.enabled=false")
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration.class,
|
||||
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration.class,
|
||||
@@ -896,7 +906,7 @@ class VehicleAnalyticsAppCompositionTest {
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=true",
|
||||
"lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"),
|
||||
"lingniu.ingest.sink.mq.enabled=false")
|
||||
"lingniu.ingest.sink.kafka.enabled=false")
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration.class,
|
||||
com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration.class))
|
||||
@@ -932,22 +942,22 @@ git commit -m "test: cover split service composition"
|
||||
### Task 4: Version Kafka Topics for GB32960 Raw/Event/DLQ
|
||||
|
||||
**Files:**
|
||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
|
||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
|
||||
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
|
||||
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
|
||||
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java`
|
||||
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java`
|
||||
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java`
|
||||
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java`
|
||||
- Create or modify: `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java`
|
||||
- Create or modify: `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java`
|
||||
|
||||
- [ ] **Step 1: Write topic routing tests**
|
||||
|
||||
Create `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java` if it does not exist. Add tests proving GB32960 raw and normalized events can route to versioned topics. Use the existing `VehicleEvent` constructors from `EventFileStoreSinkTest` and `EnvelopeMapperTelemetrySnapshotTest` as examples.
|
||||
Create `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java` if it does not exist. Add tests proving GB32960 raw and normalized events can route to versioned topics. Use the existing `VehicleEvent` constructors from `EventFileStoreSinkTest` and `EnvelopeMapperTelemetrySnapshotTest` as examples.
|
||||
|
||||
Expected test names:
|
||||
|
||||
```java
|
||||
@Test
|
||||
void rawArchiveRoutesToVersionedGb32960RawTopic() {
|
||||
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
|
||||
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
|
||||
topics.setRawArchive("vehicle.raw.gb32960.v1");
|
||||
TopicRouter router = new TopicRouter(topics);
|
||||
|
||||
@@ -970,7 +980,7 @@ Also add a normalized realtime event test:
|
||||
```java
|
||||
@Test
|
||||
void realtimeRoutesToVersionedGb32960EventTopic() {
|
||||
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
|
||||
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
|
||||
topics.setRealtime("vehicle.event.gb32960.v1");
|
||||
TopicRouter router = new TopicRouter(topics);
|
||||
|
||||
@@ -995,14 +1005,14 @@ Adjust constructor arguments to match the current sealed `VehicleEvent` definiti
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :sink-mq -Dtest=TopicRouterTest test
|
||||
mvn -pl :sink-kafka -Dtest=TopicRouterTest test
|
||||
```
|
||||
|
||||
Expected before implementation: compile failure if constructors are wrong or assertion failure if defaults are old. Fix constructors first; keep routing assertions.
|
||||
|
||||
- [ ] **Step 3: Update `SinkMqProperties.Topics` defaults**
|
||||
- [ ] **Step 3: Update `KafkaSinkProperties.Topics` defaults**
|
||||
|
||||
In `SinkMqProperties.Topics`, change defaults:
|
||||
In `KafkaSinkProperties.Topics`, change defaults:
|
||||
|
||||
```java
|
||||
private String realtime = "vehicle.event.gb32960.v1";
|
||||
@@ -1027,12 +1037,12 @@ public boolean accepts(VehicleEvent event) {
|
||||
|
||||
Update the JavaDoc above it to say Kafka carries both normalized events and raw archive records in split-service mode.
|
||||
|
||||
- [ ] **Step 5: Run sink-mq tests**
|
||||
- [ ] **Step 5: Run sink-kafka tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :sink-mq test
|
||||
mvn -pl :sink-kafka test
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
@@ -1040,11 +1050,11 @@ Expected: PASS.
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java \
|
||||
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java \
|
||||
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java \
|
||||
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java \
|
||||
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java
|
||||
git add modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java \
|
||||
modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java \
|
||||
modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java \
|
||||
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java \
|
||||
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java
|
||||
git commit -m "feat: route gb32960 kafka topics"
|
||||
```
|
||||
|
||||
@@ -1353,8 +1363,8 @@ void gb32960IngestDefaultsOnlyEnableProtocolAndKafkaProducer() {
|
||||
assertThat(props.getProperty("spring.application.name")).isEqualTo("gb32960-ingest-app");
|
||||
assertThat(props.getProperty("lingniu.ingest.gb32960.enabled")).isEqualTo("true");
|
||||
assertThat(props.getProperty("lingniu.ingest.gb32960.port")).isEqualTo("${GB32960_PORT:32960}");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.kafka.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
|
||||
@@ -1378,7 +1388,7 @@ void historyDefaultsEnableStorageAndHistoryConsumerOnly() {
|
||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("true");
|
||||
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
|
||||
.isEqualTo("${KAFKA_GROUP_HISTORY:vehicle-history}");
|
||||
}
|
||||
```
|
||||
@@ -1398,7 +1408,7 @@ void analyticsDefaultsEnableStatConsumerAndDisableProtocolAndHistoryStorage() {
|
||||
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
|
||||
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("${VEHICLE_STAT_ENABLED:true}");
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
|
||||
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
|
||||
.isEqualTo("${KAFKA_GROUP_STAT:vehicle-stat}");
|
||||
}
|
||||
```
|
||||
@@ -1566,7 +1576,7 @@ git commit -m "docs: add gb32960 split service runbook"
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :sink-mq,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test
|
||||
mvn -pl :sink-kafka,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test
|
||||
```
|
||||
|
||||
Expected: BUILD SUCCESS.
|
||||
|
||||
1519
docs/superpowers/plans/2026-06-29-vehicle-ingest-redesign-phase1.md
Normal file
1519
docs/superpowers/plans/2026-06-29-vehicle-ingest-redesign-phase1.md
Normal file
File diff suppressed because it is too large
Load Diff
46
docs/superpowers/plans/2026-06-30-kafka-streaming-mileage.md
Normal file
46
docs/superpowers/plans/2026-06-30-kafka-streaming-mileage.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# JT808 Kafka Streaming Mileage Plan
|
||||
|
||||
## Current Goal
|
||||
|
||||
Consume JT808 Kafka location events and write the derived daily mileage into the common vehicle-stat metric repository.
|
||||
|
||||
## Current Design
|
||||
|
||||
- Only supported message backbone: Kafka.
|
||||
- Source topic: `vehicle.event.jt808.v1`
|
||||
- Runtime app: `vehicle-analytics-app`
|
||||
- Runtime state: none outside `vehicle_stat_metric`
|
||||
- Metric output: `VehicleStatRepository.recordDailyMileageSample(...)`
|
||||
- Production metric storage: JDBC/MySQL `vehicle_stat_metric`
|
||||
- Date boundary: `Asia/Shanghai`
|
||||
- Calculation method: `JT808_TOTAL_MILEAGE_DIFF`
|
||||
|
||||
JT808 daily mileage is calculated only from the GPS total mileage reported by JT808 location additional information:
|
||||
|
||||
```text
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
```
|
||||
|
||||
The first valid local-day sample writes `daily_mileage_km=0.0`. Later ordered or replayed samples update the same metric row with:
|
||||
|
||||
```text
|
||||
metric_value = max_total_mileage_km - min_total_mileage_km
|
||||
metric_key = daily_mileage_km
|
||||
```
|
||||
|
||||
The local-day minimum and maximum GPS total mileage values stay on that same metric row as calculation source columns so restarts recover from MySQL without Redis or another state table.
|
||||
|
||||
## Runtime Properties
|
||||
|
||||
```text
|
||||
KAFKA_TOPIC_JT808_EVENT=vehicle.event.jt808.v1
|
||||
VEHICLE_STAT_ENABLED=true
|
||||
VEHICLE_STAT_JT808_MILEAGE_ENABLED=true
|
||||
MYSQL_JDBC_URL=<jdbc-url>
|
||||
MYSQL_USERNAME=<user>
|
||||
MYSQL_PASSWORD=<password>
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
Do not create or write a protocol-specific JT808 daily-mileage table. Do not add another message backbone, distance accumulation, integral calculation, Redis state, or memory state back into this path unless the mileage definition changes again.
|
||||
720
docs/superpowers/plans/2026-07-01-go-ingest-redesign-phase1.md
Normal file
720
docs/superpowers/plans/2026-07-01-go-ingest-redesign-phase1.md
Normal file
@@ -0,0 +1,720 @@
|
||||
# Go Vehicle Ingest Redesign Phase 1 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build the first production-capable Go runtime for GB32960, JT808, and Yutong MQTT ingestion with unified Kafka, TDengine, MySQL statistics, and Redis realtime state boundaries.
|
||||
|
||||
**Architecture:** Create a new single Go module at `go/vehicle-gateway` and migrate only the useful pieces from the existing `go/ingest-edge` and `go/vehicle-state` prototypes. The gateway produces unified envelopes to Kafka; independent consumers write TDengine history, MySQL daily metrics, and Redis realtime state.
|
||||
|
||||
**Tech Stack:** Go 1.26+, Kafka (`segmentio/kafka-go`), Redis (`redis/go-redis`), MySQL (`go-sql-driver/mysql`), TDengine official Go connector, MQTT (`eclipse/paho.mqtt.golang`), standard library TCP.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- Create: `go/vehicle-gateway/go.mod`
|
||||
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/*`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/*`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/yutongmqtt/*`
|
||||
- Create: `go/vehicle-gateway/internal/gateway/*`
|
||||
- Create: `go/vehicle-gateway/internal/identity/*`
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/*`
|
||||
- Create: `go/vehicle-gateway/internal/history/*`
|
||||
- Create: `go/vehicle-gateway/internal/stats/*`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/*`
|
||||
- Create: `go/vehicle-gateway/internal/observability/*`
|
||||
- Modify: `README.md`
|
||||
- Modify: `docs/target-architecture.md`
|
||||
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||
|
||||
The existing `go/ingest-edge` and `go/vehicle-state` directories are migration sources only. After phase 1 is verified, remove or mark them superseded.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Create Single Go Module
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/go.mod`
|
||||
- Create: `go/vehicle-gateway/internal/observability/logger.go`
|
||||
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
|
||||
- [ ] **Step 1: Create module manifest**
|
||||
|
||||
Add `go/vehicle-gateway/go.mod`:
|
||||
|
||||
```go
|
||||
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/taosdata/driver-go/v3 v3.8.1
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add logger helper**
|
||||
|
||||
Add `go/vehicle-gateway/internal/observability/logger.go`:
|
||||
|
||||
```go
|
||||
package observability
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
func NewLogger(service string) *slog.Logger {
|
||||
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||
return slog.New(handler).With("service", service)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add temporary gateway entrypoint**
|
||||
|
||||
Add `go/vehicle-gateway/cmd/gateway/main.go`:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-gateway")
|
||||
logger.Info("vehicle gateway scaffold started")
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify scaffold builds**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go mod tidy
|
||||
go test ./...
|
||||
go build ./cmd/gateway
|
||||
```
|
||||
|
||||
Expected: all commands exit `0`.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Define Unified Envelope
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||
|
||||
- [ ] **Step 1: Write envelope tests**
|
||||
|
||||
Add `go/vehicle-gateway/internal/envelope/envelope_test.go`:
|
||||
|
||||
```go
|
||||
package envelope
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||
e := FrameEnvelope{
|
||||
Protocol: ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013307795425",
|
||||
Sequence: 1,
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7e02000000ff7e",
|
||||
}
|
||||
a := e.StableEventID()
|
||||
b := e.StableEventID()
|
||||
if a == "" || a != b {
|
||||
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests and confirm failure**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/envelope
|
||||
```
|
||||
|
||||
Expected: fail because `FrameEnvelope` is not defined.
|
||||
|
||||
- [ ] **Step 3: Implement envelope**
|
||||
|
||||
Add `go/vehicle-gateway/internal/envelope/envelope.go`:
|
||||
|
||||
```go
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Protocol string
|
||||
|
||||
const (
|
||||
ProtocolGB32960 Protocol = "GB32960"
|
||||
ProtocolJT808 Protocol = "JT808"
|
||||
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||
)
|
||||
|
||||
type ParseStatus string
|
||||
|
||||
const (
|
||||
ParseOK ParseStatus = "OK"
|
||||
ParsePartial ParseStatus = "PARTIAL"
|
||||
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||
)
|
||||
|
||||
type FrameEnvelope struct {
|
||||
EventID string `json:"event_id"`
|
||||
TraceID string `json:"trace_id"`
|
||||
Protocol Protocol `json:"protocol"`
|
||||
MessageID string `json:"message_id"`
|
||||
Sequence uint16 `json:"sequence"`
|
||||
VIN string `json:"vin,omitempty"`
|
||||
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
Parsed map[string]any `json:"parsed,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
ParseStatus ParseStatus `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) VehicleKey() string {
|
||||
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
return string(e.Protocol) + ":unknown"
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) StableEventID() string {
|
||||
if strings.TrimSpace(e.EventID) != "" {
|
||||
return e.EventID
|
||||
}
|
||||
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||
sum := sha256.Sum256([]byte(input))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||
if e.EventID == "" {
|
||||
e.EventID = e.StableEventID()
|
||||
}
|
||||
if e.ParseStatus == "" {
|
||||
e.ParseStatus = ParseOK
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify tests pass**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/envelope
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Implement JT808 Frame and 0200 Parser
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser_test.go`
|
||||
|
||||
- [ ] **Step 1: Add sample frame tests**
|
||||
|
||||
Use the production sample provided in the thread:
|
||||
|
||||
```text
|
||||
7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E
|
||||
```
|
||||
|
||||
Expected assertions:
|
||||
|
||||
- message id is `0x0200`
|
||||
- phone keeps normalized BCD string `013307795425`
|
||||
- sequence is `1`
|
||||
- `fields.total_mileage_km` is parsed from additional item `0x01`
|
||||
- latitude, longitude, speed, direction, alarm, status, and device time are present
|
||||
|
||||
- [ ] **Step 2: Implement parser**
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- strip `0x7e` start/end delimiters
|
||||
- unescape `0x7d 0x02 -> 0x7e`
|
||||
- unescape `0x7d 0x01 -> 0x7d`
|
||||
- verify XOR checksum
|
||||
- parse 2011/2013 common header
|
||||
- parse BCD phone from 6 bytes and keep both raw and normalized values in `parsed.header`
|
||||
- parse 0200 fixed body
|
||||
- parse additional item list as `id,length,value_hex`
|
||||
- parse additional `0x01` as `total_mileage_km = uint32 / 10`
|
||||
|
||||
- [ ] **Step 3: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/protocol/jt808
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Implement GB32960 Frame and Data Unit Parser
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser_test.go`
|
||||
|
||||
- [ ] **Step 1: Add parser tests**
|
||||
|
||||
Tests must cover:
|
||||
|
||||
- `##` frame boundary
|
||||
- command id
|
||||
- response flag
|
||||
- 17-byte VIN
|
||||
- encryption flag
|
||||
- payload length
|
||||
- BCC verification
|
||||
- realtime data command `0x02`
|
||||
- reissue data command `0x03`
|
||||
- data unit `0x01` vehicle status fields
|
||||
- data unit `0x05` position fields
|
||||
|
||||
- [ ] **Step 2: Implement parser**
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- parse header without allocating large temporary buffers
|
||||
- keep RAW as hex in envelope
|
||||
- write all recognized data units to `Parsed`
|
||||
- write core fields to `Fields`
|
||||
- unsupported data unit stays in `Parsed["unknown_units"]`
|
||||
|
||||
- [ ] **Step 3: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/protocol/gb32960
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Implement Kafka Sink
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink.go`
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink_test.go`
|
||||
|
||||
- [ ] **Step 1: Add sink interface**
|
||||
|
||||
Create a sink interface:
|
||||
|
||||
```go
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Sink interface {
|
||||
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||
Close() error
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Implement Kafka topic routing**
|
||||
|
||||
Rules:
|
||||
|
||||
- `GB32960 -> vehicle.raw.gb32960.v1`
|
||||
- `JT808 -> vehicle.raw.jt808.v1`
|
||||
- `YUTONG_MQTT -> vehicle.raw.yutong-mqtt.v1`
|
||||
- unified topic is `vehicle.event.unified.v1`
|
||||
- message key is `env.VehicleKey()`
|
||||
|
||||
- [ ] **Step 3: Verify routing with unit tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/eventbus
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Implement TDengine History Writer
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/history/schema.go`
|
||||
- Create: `go/vehicle-gateway/internal/history/writer.go`
|
||||
- Create: `go/vehicle-gateway/internal/history/writer_test.go`
|
||||
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||
|
||||
- [ ] **Step 1: Add schema bootstrap SQL**
|
||||
|
||||
Implement schema strings for:
|
||||
|
||||
- database `lingniu_vehicle_ts`
|
||||
- stable `raw_frames`
|
||||
- stable `vehicle_locations`
|
||||
- stable `vehicle_mileage_points`
|
||||
|
||||
- [ ] **Step 2: Implement writer**
|
||||
|
||||
Rules:
|
||||
|
||||
- `AppendRawFrame` always writes one raw row.
|
||||
- `AppendLocation` writes only when longitude and latitude exist.
|
||||
- `AppendMileagePoint` writes only when `total_mileage_km` exists.
|
||||
- child table name is deterministic hash of `protocol + vehicle_key`.
|
||||
- escape tag values.
|
||||
|
||||
- [ ] **Step 3: Use TDengine official driver**
|
||||
|
||||
Import WebSocket driver in command:
|
||||
|
||||
```go
|
||||
import _ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
```
|
||||
|
||||
Default driver name:
|
||||
|
||||
```text
|
||||
TDENGINE_DRIVER=taosWS
|
||||
```
|
||||
|
||||
Short-term compatibility:
|
||||
|
||||
```text
|
||||
TDENGINE_DRIVER=taosSql
|
||||
```
|
||||
|
||||
Only use `taosSql` when ECS TDengine WebSocket is not available.
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/history
|
||||
go build ./cmd/history-writer
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Implement MySQL Daily Metric Writer
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/stats/schema.go`
|
||||
- Create: `go/vehicle-gateway/internal/stats/daily_metric.go`
|
||||
- Create: `go/vehicle-gateway/internal/stats/daily_metric_test.go`
|
||||
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||
|
||||
- [ ] **Step 1: Add schema bootstrap**
|
||||
|
||||
Implement `vehicle_daily_metric` schema from the design spec.
|
||||
|
||||
- [ ] **Step 2: Add metric derivation tests**
|
||||
|
||||
Test cases:
|
||||
|
||||
- no `total_mileage_km` produces no metric
|
||||
- one sample produces:
|
||||
- `daily_mileage_km = 0`
|
||||
- `daily_total_mileage_km = sample`
|
||||
- later larger sample updates:
|
||||
- `latest_total_mileage_km`
|
||||
- `daily_mileage_km`
|
||||
- `daily_total_mileage_km`
|
||||
- out-of-order smaller sample updates:
|
||||
- `first_total_mileage_km`
|
||||
- `daily_mileage_km`
|
||||
|
||||
- [ ] **Step 3: Implement MySQL upsert**
|
||||
|
||||
Use one table and one idempotent upsert:
|
||||
|
||||
```sql
|
||||
INSERT INTO vehicle_daily_metric
|
||||
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
first_total_mileage_km = LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)),
|
||||
latest_total_mileage_km = GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)),
|
||||
metric_value = CASE
|
||||
WHEN metric_key = 'daily_mileage_km'
|
||||
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
- LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
WHEN metric_key = 'daily_total_mileage_km'
|
||||
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
ELSE VALUES(metric_value)
|
||||
END,
|
||||
sample_count = sample_count + 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/stats
|
||||
go build ./cmd/stat-writer
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Implement Redis Realtime API
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/realtime/repository.go`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/repository_test.go`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/http.go`
|
||||
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||
|
||||
- [ ] **Step 1: Add repository contract**
|
||||
|
||||
Methods:
|
||||
|
||||
- `Update(ctx, envelope.FrameEnvelope) error`
|
||||
- `GetMerged(ctx, vin string) (Snapshot, error)`
|
||||
- `GetProtocol(ctx, vin string, protocol envelope.Protocol) (Snapshot, error)`
|
||||
- `IsOnline(ctx, vin string) (OnlineStatus, error)`
|
||||
|
||||
- [ ] **Step 2: Implement merge logic**
|
||||
|
||||
Rules:
|
||||
|
||||
- update `vehicle:latest:{vin}:{protocol}`
|
||||
- update `vehicle:latest:{vin}` with newest fields
|
||||
- update `vehicle:online:{vin}`
|
||||
- update sorted set `vehicle:last_seen`
|
||||
|
||||
- [ ] **Step 3: Implement HTTP API**
|
||||
|
||||
Routes:
|
||||
|
||||
- `GET /api/realtime/vehicles/{vin}`
|
||||
- `GET /api/realtime/vehicles/{vin}/online`
|
||||
- `GET /api/realtime/vehicles/{vin}/protocols/{protocol}`
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/realtime
|
||||
go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Wire Gateway Runtime
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/gateway/tcp_server.go`
|
||||
- Create: `go/vehicle-gateway/internal/gateway/mqtt_client.go`
|
||||
- Modify: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
|
||||
- [ ] **Step 1: Implement TCP server**
|
||||
|
||||
Rules:
|
||||
|
||||
- one goroutine per accepted connection
|
||||
- bounded max connections
|
||||
- read timeout and idle timeout
|
||||
- protocol-specific frame extractor
|
||||
- structured peer endpoint
|
||||
- graceful shutdown on SIGTERM
|
||||
|
||||
- [ ] **Step 2: Implement MQTT client**
|
||||
|
||||
Rules:
|
||||
|
||||
- connect with official production config from environment
|
||||
- subscribe configured topic list
|
||||
- convert each message into envelope
|
||||
- publish raw and unified events
|
||||
- reconnect with backoff
|
||||
|
||||
- [ ] **Step 3: Verify local JSON mode**
|
||||
|
||||
Run without Kafka:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
GB32960_TCP_ADDR=:132960 JT808_TCP_ADDR=:18080 go run ./cmd/gateway
|
||||
```
|
||||
|
||||
Expected: service starts and logs configured listeners.
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Docker and ECS Deployment
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/Dockerfile`
|
||||
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||
- Modify: `docs/operations/current-ecs-deployment.md`
|
||||
|
||||
- [ ] **Step 1: Add multi-stage Dockerfile**
|
||||
|
||||
Build all commands:
|
||||
|
||||
- `gateway`
|
||||
- `history-writer`
|
||||
- `stat-writer`
|
||||
- `realtime-api`
|
||||
|
||||
- [ ] **Step 2: Add Portainer compose**
|
||||
|
||||
Services:
|
||||
|
||||
- `go-vehicle-gateway`
|
||||
- `go-history-writer`
|
||||
- `go-stat-writer`
|
||||
- `go-realtime-api`
|
||||
|
||||
Each service must include:
|
||||
|
||||
- restart policy
|
||||
- memory limit
|
||||
- Kafka env
|
||||
- MySQL/TDengine/Redis env as needed
|
||||
- logging options
|
||||
|
||||
- [ ] **Step 3: Verify Linux build**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
Expected: all commands exit `0`.
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Production Verification
|
||||
|
||||
**Files:**
|
||||
- Create: `docs/operations/go-vehicle-gateway-verification.md`
|
||||
|
||||
- [ ] **Step 1: Record test commands**
|
||||
|
||||
Document commands to verify:
|
||||
|
||||
- gateway process health
|
||||
- Kafka topic consumption
|
||||
- TDengine row counts
|
||||
- MySQL daily metric rows
|
||||
- Redis realtime lookup
|
||||
|
||||
- [ ] **Step 2: Validate real traffic**
|
||||
|
||||
Evidence required:
|
||||
|
||||
- one real 32960 VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||
- one real JT808 phone/VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||
- one real Yutong MQTT VIN with RAW and Redis snapshot
|
||||
|
||||
- [ ] **Step 3: Keep old Java services until evidence is captured**
|
||||
|
||||
Only disable Java equivalents after the evidence file contains successful command outputs and timestamps.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review Checklist
|
||||
|
||||
- The plan creates one new Go module instead of extending scattered prototypes.
|
||||
- The plan covers GB32960, JT808, and Yutong MQTT ingress.
|
||||
- The plan covers Kafka, TDengine, MySQL, and Redis.
|
||||
- The plan includes 32960 and 808 daily mileage and daily total mileage.
|
||||
- The plan includes local and ECS verification.
|
||||
- The plan does not restore Xinda Push.
|
||||
- The plan keeps Java services untouched until Go evidence exists.
|
||||
@@ -1,3 +1,9 @@
|
||||
> **Superseded:** This 2026-06-23 DuckDB-based design is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine is the default hot history store,
|
||||
> `event-file-store` and Xinda Push have been removed from the production codebase.
|
||||
|
||||
# GB32960 Production Readiness Design
|
||||
|
||||
## Goal
|
||||
@@ -269,28 +275,20 @@ MYSQL_USERNAME
|
||||
MYSQL_PASSWORD
|
||||
```
|
||||
|
||||
`vehicle_daily_stat`
|
||||
Daily derived metrics are stored in the common metric table, not in a
|
||||
protocol-specific daily-mileage table.
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS vehicle_daily_stat (
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
CREATE TABLE IF NOT EXISTS vehicle_stat_metric (
|
||||
vin VARCHAR(64) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
platform_account VARCHAR(64),
|
||||
total_mileage_km DECIMAL(14,3),
|
||||
daily_mileage_km DECIMAL(14,3),
|
||||
daily_power_kwh DECIMAL(14,3),
|
||||
daily_hydrogen_kg DECIMAL(14,4),
|
||||
daily_hydrogen_kg_per_100km DECIMAL(14,4),
|
||||
first_event_time DATETIME(3),
|
||||
last_event_time DATETIME(3),
|
||||
first_total_mileage_km DECIMAL(14,3),
|
||||
last_total_mileage_km DECIMAL(14,3),
|
||||
last_soc_percent DECIMAL(8,3),
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
late_sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
calculation_quality VARCHAR(32) NOT NULL,
|
||||
updated_at DATETIME(3) NOT NULL,
|
||||
PRIMARY KEY (vin, stat_date)
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,6) NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (vin, stat_date, metric_key)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
```
|
||||
|
||||
@@ -345,23 +343,19 @@ Validation:
|
||||
|
||||
### Daily Mileage
|
||||
|
||||
Preferred:
|
||||
JT808 daily mileage uses the GPS total mileage reported in location additional
|
||||
information. The local-day minimum and maximum GPS total mileage values rewrite
|
||||
the same metric row, so ordered streaming and historical replay use one
|
||||
calculation path.
|
||||
|
||||
```text
|
||||
dailyMileage = today.lastTotalMileage - previousDay.lastTotalMileage
|
||||
dailyMileage = today.maxTotalMileage - today.minTotalMileage
|
||||
calculationMethod = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
Fallback:
|
||||
|
||||
```text
|
||||
dailyMileage = today.lastTotalMileage - today.firstTotalMileage
|
||||
```
|
||||
|
||||
Quality:
|
||||
|
||||
- `PREVIOUS_DAY_BASELINE` when previous day baseline exists;
|
||||
- `INTRADAY_DELTA` when only same-day baseline exists;
|
||||
- `INSUFFICIENT_DATA` when neither is reliable.
|
||||
The result is stored in `vehicle_stat_metric` with
|
||||
`metric_key=daily_mileage_km`. There is no JT808-specific daily mileage table,
|
||||
no Redis mileage state, and no previous-day baseline calculation.
|
||||
|
||||
### Daily Power Consumption
|
||||
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
> **Superseded:** This 2026-06-23 split design is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine is the default hot history store,
|
||||
> `sink-archive` owns raw bytes, and `event-file-store` is optional
|
||||
> compatibility only.
|
||||
|
||||
# GB32960 Service Split Design
|
||||
|
||||
Date: 2026-06-23
|
||||
@@ -53,7 +60,7 @@ Initial module dependencies:
|
||||
- `ingest-codec-common`
|
||||
- `session-core`
|
||||
- `protocol-gb32960`
|
||||
- `sink-mq`
|
||||
- `sink-kafka`
|
||||
- `observability`
|
||||
|
||||
### `vehicle-history-app`
|
||||
@@ -79,7 +86,7 @@ Non-responsibilities:
|
||||
Initial module dependencies:
|
||||
|
||||
- `ingest-api`
|
||||
- `sink-mq`
|
||||
- `sink-kafka`
|
||||
- `sink-archive`
|
||||
- `event-file-store`
|
||||
- `event-history-service`
|
||||
@@ -109,7 +116,7 @@ Non-responsibilities:
|
||||
Initial module dependencies:
|
||||
|
||||
- `ingest-api`
|
||||
- `sink-mq`
|
||||
- `sink-kafka`
|
||||
- `vehicle-state-service`
|
||||
- `vehicle-stat-service`
|
||||
- `observability`
|
||||
@@ -126,7 +133,7 @@ Use versioned topics:
|
||||
- `vehicle.event.gb32960.v1`
|
||||
- `vehicle.dlq.gb32960.v1`
|
||||
|
||||
The current topic names under `lingniu.ingest.sink.mq.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
|
||||
The current topic names under `lingniu.ingest.sink.kafka.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
|
||||
|
||||
### Keys
|
||||
|
||||
|
||||
621
docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md
Normal file
621
docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md
Normal file
@@ -0,0 +1,621 @@
|
||||
# Vehicle Ingest Redesign Design
|
||||
|
||||
## Background
|
||||
|
||||
Current GB32960 and JT808 ingestion can receive, decode, publish, and query data, but the responsibilities are split across several partially overlapping paths:
|
||||
|
||||
- Netty handlers produce `RawFrame` and the dispatcher emits both raw archive events and normalized vehicle events.
|
||||
- Kafka envelopes carry raw archive references, while raw bytes are stored separately by the archive sink.
|
||||
- GB32960 historical queries use raw archive records plus file replay decoding.
|
||||
- JT808 location history is materialized separately into TDengine.
|
||||
- DuckDB/Parquet, local raw archive, Kafka, and TDengine each own part of the story, so query and durability semantics are hard to reason about.
|
||||
|
||||
The new design intentionally keeps useful protocol parsers and app deployment knowledge, but replaces the ingestion, durability, storage, and query boundaries with a first-principles model.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Store every original inbound protocol frame, including malformed frames when bytes are available.
|
||||
2. Query historical uploaded data for GB32960 and JT808 by vehicle, protocol, message type, and time range.
|
||||
3. Extend to new fields, protocol messages, and vendor profiles without rewriting the core pipeline.
|
||||
4. Sustain production load for 10,000 vehicles and 1,000 concurrent historical query requests per second.
|
||||
5. Make ACK and durability rules explicit enough that an acknowledged frame is not silently lost.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Do not make Kafka the long-term historical query database.
|
||||
- Do not store large raw byte arrays directly in Kafka topics.
|
||||
- Do not keep DuckDB/Parquet as the production hot query store for GB32960 and JT808.
|
||||
- Do not build one generic API that hides protocol-specific semantics. Shared internals are preferred; external APIs may remain protocol-aware.
|
||||
|
||||
## Recommended Architecture
|
||||
|
||||
Use `Raw Archive + Kafka + TDengine` as the core architecture.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
GB["GB32960 TCP"] --> Edge["Protocol Edge"]
|
||||
JT["JT808 TCP"] --> Edge
|
||||
Edge --> RawStore["Raw Archive"]
|
||||
RawStore --> RawTopic["Kafka vehicle.raw-frame.v1"]
|
||||
Edge --> Decode["Protocol Decoder"]
|
||||
Decode --> FactTopic["Kafka vehicle.decoded-fact.v1"]
|
||||
RawTopic --> Writer["History Writer"]
|
||||
FactTopic --> Writer
|
||||
Writer --> TD["TDengine"]
|
||||
Writer --> Redis["Realtime State"]
|
||||
API["History API"] --> TD
|
||||
API --> RawStore
|
||||
RealtimeAPI["Realtime API"] --> Redis
|
||||
```
|
||||
|
||||
The architecture has three facts:
|
||||
|
||||
- `RawFrameFact`: one row per inbound frame, whether parsing succeeds or fails.
|
||||
- `DecodedFact`: zero or more normalized facts derived from a raw frame.
|
||||
- `RawArchiveObject`: immutable raw bytes addressed by `raw_uri`.
|
||||
|
||||
The raw archive is the forensic source of truth. TDengine is the query source of truth. Kafka is the decoupling and replay pipe.
|
||||
|
||||
## Module Boundaries
|
||||
|
||||
### Protocol Edge Apps
|
||||
|
||||
Apps:
|
||||
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Accept TCP connections.
|
||||
- Split frames.
|
||||
- Perform protocol-level validation, authentication, session binding, and ACK.
|
||||
- Create `RawFrameFact`.
|
||||
- Persist raw bytes through `RawArchiveWriter`.
|
||||
- Publish raw frame fact to Kafka.
|
||||
- Decode frames and publish decoded facts.
|
||||
|
||||
They do not query history, export data, or write business tables directly.
|
||||
|
||||
### Ingest Core
|
||||
|
||||
New module:
|
||||
|
||||
- `modules/core/ingest-facts`
|
||||
|
||||
Core types:
|
||||
|
||||
```java
|
||||
public record RawFrameFact(
|
||||
String frameId,
|
||||
ProtocolId protocol,
|
||||
String vehicleKey,
|
||||
String vin,
|
||||
String phone,
|
||||
int messageId,
|
||||
int subType,
|
||||
Instant eventTime,
|
||||
Instant receivedAt,
|
||||
String peer,
|
||||
String rawUri,
|
||||
String checksum,
|
||||
long rawSizeBytes,
|
||||
ParseStatus parseStatus,
|
||||
String parseError,
|
||||
Map<String, String> metadata
|
||||
) {}
|
||||
```
|
||||
|
||||
```java
|
||||
public sealed interface DecodedFact permits
|
||||
DecodedFact.Location,
|
||||
DecodedFact.Realtime,
|
||||
DecodedFact.Alarm,
|
||||
DecodedFact.Session,
|
||||
DecodedFact.Register,
|
||||
DecodedFact.Extension {
|
||||
|
||||
String factId();
|
||||
String frameId();
|
||||
ProtocolId protocol();
|
||||
String vehicleKey();
|
||||
String vin();
|
||||
Instant eventTime();
|
||||
Instant receivedAt();
|
||||
String rawUri();
|
||||
Map<String, String> metadata();
|
||||
}
|
||||
```
|
||||
|
||||
`vehicleKey` is the stable partition key:
|
||||
|
||||
- GB32960: VIN.
|
||||
- JT808: resolved VIN when known, otherwise `jt808:<phone>`.
|
||||
- Unknown/malformed: `unknown:<protocol>:<hash>`.
|
||||
|
||||
### Raw Archive
|
||||
|
||||
New or refactored module:
|
||||
|
||||
- `modules/sinks/raw-archive-store`
|
||||
|
||||
Interface:
|
||||
|
||||
```java
|
||||
public interface RawArchiveWriter {
|
||||
RawArchiveReceipt write(RawArchiveWriteRequest request) throws IOException;
|
||||
}
|
||||
```
|
||||
|
||||
Receipt:
|
||||
|
||||
```java
|
||||
public record RawArchiveReceipt(
|
||||
String rawUri,
|
||||
String checksum,
|
||||
long sizeBytes
|
||||
) {}
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- Raw bytes are immutable once written.
|
||||
- Write path is content-addressed or deterministic by `protocol/date/vehicleKey/frameId.bin`.
|
||||
- Write must be fsync-capable for local storage and pluggable for object storage.
|
||||
- `rawUri` must be resolvable by history API for single-frame replay.
|
||||
- Malformed frames are written with `parseStatus=FAILED` when bytes are available.
|
||||
|
||||
Deployment constraints:
|
||||
|
||||
- Raw archive, TDengine, and the history service may run on different ECS instances, but should use private-network access in the same VPC and availability zone.
|
||||
- High-frequency historical queries only read TDengine and must not read raw archive.
|
||||
- Raw archive is only used by single-frame detail, troubleshooting, replay, and asynchronous export paths.
|
||||
- `rawUri` must be globally resolvable by services and must not be a private local absolute path on one ECS instance.
|
||||
- The first local raw archive implementation must stay behind a replaceable interface so it can later move to OSS, MinIO, or another object store.
|
||||
|
||||
### Kafka Topics
|
||||
|
||||
Use stable versioned topics:
|
||||
|
||||
- `vehicle.raw-frame.v1`
|
||||
- `vehicle.decoded-fact.v1`
|
||||
- `vehicle.dlq.v1`
|
||||
|
||||
Kafka key:
|
||||
|
||||
```text
|
||||
<protocol>:<vehicleKey>
|
||||
```
|
||||
|
||||
`vehicle.raw-frame.v1` payload contains `RawFrameFact` without raw bytes. It carries `rawUri`, checksum, byte size, message id, parse status, and metadata.
|
||||
|
||||
`vehicle.decoded-fact.v1` payload contains one `DecodedFact`. It always references `frameId` and `rawUri`.
|
||||
|
||||
`vehicle.dlq.v1` carries failed store, publish, decode, and write attempts with enough metadata to replay from raw archive when possible.
|
||||
|
||||
### History Writer
|
||||
|
||||
New app or refactored app:
|
||||
|
||||
- `vehicle-history-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume `vehicle.raw-frame.v1`.
|
||||
- Consume `vehicle.decoded-fact.v1`.
|
||||
- Batch write TDengine.
|
||||
- Update realtime state store for latest vehicle state.
|
||||
- Write DLQ on invalid payloads or storage failures.
|
||||
|
||||
The writer is idempotent by `(frame_id)` for raw facts and `(fact_id)` for decoded facts.
|
||||
|
||||
## TDengine Model
|
||||
|
||||
TDengine is the production historical query store.
|
||||
|
||||
### Raw Frames Super Table
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
message_id INT,
|
||||
sub_type INT,
|
||||
event_time TIMESTAMP,
|
||||
raw_uri NCHAR(512),
|
||||
checksum NCHAR(128),
|
||||
raw_size_bytes BIGINT,
|
||||
parse_status NCHAR(16),
|
||||
parse_error NCHAR(512),
|
||||
peer NCHAR(128),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
Child table:
|
||||
|
||||
```text
|
||||
raw_<protocol>_<hash(vehicleKey)>
|
||||
```
|
||||
|
||||
Primary query patterns:
|
||||
|
||||
- Raw frames by vehicle and time.
|
||||
- Raw frames by protocol, message id, parse status, and time.
|
||||
- Single raw frame by `frame_id` or `raw_uri`.
|
||||
|
||||
### Location Super Table
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg DOUBLE,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
### Realtime Super Table
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_realtime (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
speed_kmh DOUBLE,
|
||||
total_mileage_km DOUBLE,
|
||||
battery_soc DOUBLE,
|
||||
total_voltage_v DOUBLE,
|
||||
total_current_a DOUBLE,
|
||||
running_mode NCHAR(64),
|
||||
vehicle_state NCHAR(64),
|
||||
charging_state NCHAR(64),
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
`fields_json` stores sparse full-field values for low-frequency inspection. High-frequency queried fields should be promoted to explicit columns or a dedicated extension table.
|
||||
|
||||
### Alarm Super Table
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_alarms (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
alarm_level NCHAR(32),
|
||||
alarm_code BIGINT,
|
||||
alarm_name NCHAR(128),
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
### Session and Registration Tables
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_sessions (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
session_type NCHAR(32),
|
||||
result_code INT,
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
JT808 registration details go into `vehicle_sessions` first and may later be projected into MySQL identity bindings for durable identity lookup.
|
||||
|
||||
## ACK and Durability Rules
|
||||
|
||||
For frames that require a protocol ACK:
|
||||
|
||||
1. Frame bytes are accepted from Netty.
|
||||
2. Raw bytes are written to raw archive.
|
||||
3. `RawFrameFact` is published to Kafka.
|
||||
4. Protocol-specific ACK is sent.
|
||||
5. Decoding and `DecodedFact` publication may run before or after ACK, but failures must emit a DLQ event and update raw frame parse status.
|
||||
|
||||
For GB32960 realtime/report frames, this prevents acknowledging a frame whose original bytes cannot be recovered.
|
||||
|
||||
For JT808 register/auth/heartbeat, ACK follows the same raw archive and raw fact boundary. Register ACK can still include token generation before raw fact publish, but must not be flushed to the channel until the durability boundary succeeds.
|
||||
|
||||
If raw archive write fails, close or keep the channel according to protocol tolerance, but do not ACK success.
|
||||
|
||||
If Kafka raw fact publish fails, do not ACK success for durable-ACK frames. Non-durable frames go to local DLQ when configured.
|
||||
|
||||
## Protocol Decoding and Extension
|
||||
|
||||
Use a plugin contract:
|
||||
|
||||
```java
|
||||
public interface ProtocolFrameDecoder {
|
||||
ProtocolId protocol();
|
||||
DecodeResult decode(RawFrameFact rawFact, byte[] rawBytes);
|
||||
}
|
||||
```
|
||||
|
||||
```java
|
||||
public record DecodeResult(
|
||||
ParseStatus status,
|
||||
List<DecodedFact> facts,
|
||||
String errorMessage,
|
||||
Map<String, String> metadata
|
||||
) {}
|
||||
```
|
||||
|
||||
GB32960:
|
||||
|
||||
- Frame command maps to session, realtime, location, alarm, and extension facts.
|
||||
- Vendor extensions are selected by platform account, VIN, or configured profile.
|
||||
- Full raw replay remains possible by reading `rawUri` and running the current decoder.
|
||||
|
||||
JT808:
|
||||
|
||||
- Message id maps to register, auth, heartbeat, location, batch location, media metadata, and extension facts.
|
||||
- Identity resolution updates `vehicleKey`.
|
||||
- Registration facts carry province, city, maker, device id, plate, plate color, and auth token metadata.
|
||||
|
||||
New protocol messages do not require TDengine schema changes unless they become high-frequency query dimensions. Unknown fields are stored as extension facts with `fields_json`.
|
||||
|
||||
## Query APIs
|
||||
|
||||
Keep API surfaces explicit.
|
||||
|
||||
Raw frame APIs:
|
||||
|
||||
- `GET /api/history/raw-frames`
|
||||
- Filters: `protocol`, `vin`, `phone`, `vehicleKey`, `messageId`, `parseStatus`, `dateFrom`, `dateTo`, `pageSize`, `cursor`.
|
||||
- Returns frame metadata, raw URI, checksum, parse status, and compact decoded summary when available.
|
||||
|
||||
Single raw frame API:
|
||||
|
||||
- `GET /api/history/raw-frames/{frameId}`
|
||||
- Returns raw metadata and decoded detail.
|
||||
- Supports `includeRawHex=true` with a strict size limit.
|
||||
|
||||
Location APIs:
|
||||
|
||||
- `GET /api/history/locations`
|
||||
- Filters: `protocol`, `vin`, `phone`, `vehicleKey`, time range, cursor.
|
||||
|
||||
Realtime APIs:
|
||||
|
||||
- `GET /api/history/realtime`
|
||||
- Historical time-series query.
|
||||
|
||||
Realtime state APIs:
|
||||
|
||||
- `GET /api/realtime/vehicles/{vehicleKey}`
|
||||
- Reads Redis latest state, not TDengine.
|
||||
|
||||
Export:
|
||||
|
||||
- Export is asynchronous.
|
||||
- Request creates an export job.
|
||||
- Worker streams TDengine query results to object storage or local export files.
|
||||
- API returns job status and download URI.
|
||||
|
||||
## Pagination
|
||||
|
||||
Use cursor pagination, not deep offset, for high-QPS history queries.
|
||||
|
||||
Cursor fields:
|
||||
|
||||
```text
|
||||
ts, received_at, fact_id
|
||||
```
|
||||
|
||||
Descending query predicate:
|
||||
|
||||
```sql
|
||||
WHERE (
|
||||
ts < :cursorTs
|
||||
OR (ts = :cursorTs AND received_at < :cursorReceivedAt)
|
||||
OR (ts = :cursorTs AND received_at = :cursorReceivedAt AND fact_id < :cursorFactId)
|
||||
)
|
||||
```
|
||||
|
||||
Page size defaults to 100 and is capped at 1000.
|
||||
|
||||
## Capacity Design
|
||||
|
||||
Assumptions:
|
||||
|
||||
- 10,000 vehicles.
|
||||
- Normal report interval: 5 to 10 seconds.
|
||||
- Expected writes: 1,000 to 2,000 frames per second.
|
||||
- Peak burst factor: 3x.
|
||||
- Historical query target: 1,000 requests per second.
|
||||
|
||||
Write path:
|
||||
|
||||
- Netty worker threads stay CPU-light.
|
||||
- Raw archive writes use bounded async workers.
|
||||
- Kafka producer uses compression, batching, idempotence, and vehicle-key partitioning.
|
||||
- History writer batches TDengine inserts by super table and child table.
|
||||
- TDengine child tables are partitioned by vehicle key to make single-vehicle queries cheap.
|
||||
|
||||
Read path:
|
||||
|
||||
- Most high-QPS queries require vehicle key and time range.
|
||||
- Unbounded cross-vehicle queries are admin-only and capped.
|
||||
- Realtime latest state is served from Redis latest-state cache.
|
||||
- Raw detail replay reads one raw object and decodes on demand.
|
||||
- Common dictionary and metadata responses are cached in memory.
|
||||
|
||||
Backpressure:
|
||||
|
||||
- Raw archive worker queue has a fixed bound.
|
||||
- Kafka publish futures are tracked.
|
||||
- If durability boundary cannot complete within timeout, durable ACK fails and the channel is closed or throttled.
|
||||
- History writer lag is observable by Kafka consumer lag and TDengine batch latency.
|
||||
|
||||
## Migration Plan
|
||||
|
||||
Phase 1: Build new fact model beside existing code.
|
||||
|
||||
- Add `ingest-facts`.
|
||||
- Add raw archive receipt contract.
|
||||
- Add serializers for raw frame and decoded fact Kafka messages.
|
||||
- Add tests for stable ids, vehicle key, and raw URI generation.
|
||||
|
||||
Phase 2: Harden raw durability boundary.
|
||||
|
||||
- Refactor GB32960 and JT808 handlers so ACK waits for raw archive and raw fact publish.
|
||||
- Keep existing parsers.
|
||||
- Emit decoded facts to the new Kafka topic.
|
||||
|
||||
Phase 3: Build TDengine writer.
|
||||
|
||||
- Create TDengine schema manager.
|
||||
- Write raw frame facts.
|
||||
- Write location, realtime, alarm, session facts.
|
||||
- Add idempotent insert behavior by stable ids.
|
||||
|
||||
Phase 4: Build query APIs.
|
||||
|
||||
- Raw frame query.
|
||||
- Single frame replay.
|
||||
- Location history query.
|
||||
- Realtime history query.
|
||||
- Latest realtime state query.
|
||||
|
||||
Phase 5: Cut over and retire old hot paths.
|
||||
|
||||
- Stop using DuckDB/Parquet as production hot history for GB32960/JT808.
|
||||
- Keep raw replay compatibility while historical data migrates.
|
||||
- Remove protocol-specific ad hoc history storage once TDengine coverage is verified.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
Unit tests:
|
||||
|
||||
- `RawFrameFact` validation and vehicle key derivation.
|
||||
- Raw archive path generation and checksum.
|
||||
- GB32960 decoder facts from golden frames.
|
||||
- JT808 decoder facts from sample frames.
|
||||
- ACK boundary success and failure cases.
|
||||
- TDengine SQL generation.
|
||||
|
||||
Integration tests:
|
||||
|
||||
- Start app with embedded or mocked Kafka producer and fake raw archive.
|
||||
- Send GB32960 sample frame and verify raw fact, decoded fact, raw archive receipt.
|
||||
- Send JT808 register/auth/location frames and verify registration/session/location facts.
|
||||
- Simulate raw archive failure and verify no success ACK.
|
||||
- Simulate Kafka publish failure and verify durable ACK failure.
|
||||
|
||||
Storage tests:
|
||||
|
||||
- TDengine schema initialization.
|
||||
- Batch insert raw frames and decoded facts.
|
||||
- Query by vehicle key and time range.
|
||||
- Cursor pagination correctness.
|
||||
- Single raw frame replay by `rawUri`.
|
||||
|
||||
Load tests:
|
||||
|
||||
- 2,000 frames/s sustained write for at least 30 minutes.
|
||||
- 6,000 frames/s burst for 5 minutes.
|
||||
- 1,000 QPS location/realtime history queries with bounded p95 latency.
|
||||
- Verify no raw frame count gap between raw archive receipts, Kafka raw facts, and TDengine raw frame rows.
|
||||
|
||||
## Operational Requirements
|
||||
|
||||
Metrics:
|
||||
|
||||
- TCP connections.
|
||||
- Frame receive rate.
|
||||
- Raw archive write latency and failure count.
|
||||
- Kafka publish latency and failure count.
|
||||
- Decoder success/failure count by protocol and message id.
|
||||
- TDengine batch size, latency, and failure count.
|
||||
- Query QPS and latency by endpoint.
|
||||
- Kafka consumer lag.
|
||||
|
||||
Logs:
|
||||
|
||||
- One structured log per failed frame with `frameId`, protocol, vehicle key, message id, raw URI, and error.
|
||||
- No full raw bytes in normal logs.
|
||||
- Raw hex only in explicit debug tools with size limits.
|
||||
|
||||
Alerts:
|
||||
|
||||
- Raw archive write failure.
|
||||
- Durable ACK timeout.
|
||||
- Kafka producer failure.
|
||||
- History writer lag.
|
||||
- TDengine write failure.
|
||||
- Raw archive count and TDengine raw frame row count divergence.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
The redesign is complete only when all of the following are true:
|
||||
|
||||
1. Every accepted GB32960 and JT808 frame creates a raw archive object and a TDengine `raw_frames` row.
|
||||
2. Malformed frames with bytes are queryable from raw history with parse status and error reason.
|
||||
3. GB32960 and JT808 location history can be queried by vehicle and time range with cursor pagination.
|
||||
4. GB32960 realtime history can be queried by vehicle and time range.
|
||||
5. JT808 registration/session data is queryable and can update identity binding.
|
||||
6. Single-frame detail can reload raw bytes by `rawUri` and decode with the current decoder.
|
||||
7. ACK-required frames do not receive success ACK before raw archive and raw fact publication succeed.
|
||||
8. New protocol fields can be added by implementing a decoder/projector and, only when needed, a TDengine table migration.
|
||||
9. Load tests demonstrate sustained 2,000 frames/s writes and 1,000 QPS bounded historical queries.
|
||||
10. Old DuckDB/Parquet hot-history paths are no longer required for GB32960/JT808 production queries.
|
||||
|
||||
## Open Decisions
|
||||
|
||||
The following choices are intentionally fixed for the first implementation:
|
||||
|
||||
- Use local raw archive first, with the interface shaped for object storage later.
|
||||
- Use TDengine as the only production hot historical query store for GB32960/JT808.
|
||||
- Use explicit protocol-aware query APIs instead of a single generic query API.
|
||||
- Use cursor pagination for all high-QPS historical queries.
|
||||
- Keep Kafka payloads byte-light by sending raw references instead of raw bytes.
|
||||
624
docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.zh.md
Normal file
624
docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.zh.md
Normal file
@@ -0,0 +1,624 @@
|
||||
# 车辆接入重设计方案
|
||||
|
||||
## 背景
|
||||
|
||||
当前 GB32960 和 JT808 链路已经能够完成接收、解析、发布和查询,但职责边界被拆散在多条路径里:
|
||||
|
||||
- Netty Handler 生成 `RawFrame`,Dispatcher 同时发出原始归档事件和标准化车辆事件。
|
||||
- Kafka Envelope 只携带 raw archive 引用,原始 bytes 由 archive sink 另行存储。
|
||||
- GB32960 历史查询依赖 raw archive 记录,再回读文件重新解析。
|
||||
- JT808 位置历史又单独物化到 TDengine。
|
||||
- DuckDB/Parquet、本地 raw archive、Kafka、TDengine 分别承担一部分职责,导致查询语义和持久化边界不好推理。
|
||||
|
||||
新设计保留已有协议解析器和部署经验,但重新定义接入、持久化、存储和查询边界。
|
||||
|
||||
## 目标
|
||||
|
||||
1. 保存每一条入站协议原始帧,包括可拿到 bytes 的异常帧。
|
||||
2. 支持按车辆、协议、消息类型、时间范围查询 GB32960 和 JT808 历史上报数据。
|
||||
3. 新增字段、协议消息、厂商扩展时,不需要重写核心管线。
|
||||
4. 支撑 10,000 辆车生产写入,以及 1,000 QPS 历史查询。
|
||||
5. 明确 ACK 和持久化规则,避免“已经 ACK 但数据悄悄丢失”。
|
||||
|
||||
## 非目标
|
||||
|
||||
- 不把 Kafka 作为长期历史查询数据库。
|
||||
- 不把大块原始 bytes 直接塞进 Kafka topic。
|
||||
- 不继续把 DuckDB/Parquet 作为 GB32960/JT808 的生产热查询库。
|
||||
- 不做一个掩盖协议差异的万能接口。内部可以统一,外部 API 保持协议语义清晰。
|
||||
|
||||
## 推荐架构
|
||||
|
||||
采用 `Raw Archive + Kafka + TDengine` 作为核心架构。
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
GB["GB32960 TCP"] --> Edge["协议接入边界"]
|
||||
JT["JT808 TCP"] --> Edge
|
||||
Edge --> RawStore["Raw Archive"]
|
||||
RawStore --> RawTopic["Kafka vehicle.raw-frame.v1"]
|
||||
Edge --> Decode["协议解析器"]
|
||||
Decode --> FactTopic["Kafka vehicle.decoded-fact.v1"]
|
||||
RawTopic --> Writer["History Writer"]
|
||||
FactTopic --> Writer
|
||||
Writer --> TD["TDengine"]
|
||||
Writer --> Redis["实时状态"]
|
||||
API["历史 API"] --> TD
|
||||
API --> RawStore
|
||||
RealtimeAPI["实时 API"] --> Redis
|
||||
```
|
||||
|
||||
架构里只有三类事实:
|
||||
|
||||
- `RawFrameFact`:每一条入站帧一条记录,不管解析成功还是失败。
|
||||
- `DecodedFact`:从 raw frame 派生出来的零条或多条标准事实。
|
||||
- `RawArchiveObject`:不可变原始 bytes,通过 `raw_uri` 寻址。
|
||||
|
||||
raw archive 是排障和回放的事实源。TDengine 是查询事实源。Kafka 是解耦和重放管道。
|
||||
|
||||
## 模块边界
|
||||
|
||||
### 协议接入 APP
|
||||
|
||||
APP:
|
||||
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
|
||||
职责:
|
||||
|
||||
- 接收 TCP 连接。
|
||||
- 完成帧切分。
|
||||
- 执行协议层校验、鉴权、会话绑定和 ACK。
|
||||
- 创建 `RawFrameFact`。
|
||||
- 通过 `RawArchiveWriter` 持久化原始 bytes。
|
||||
- 发布 raw frame fact 到 Kafka。
|
||||
- 解析协议帧并发布 decoded facts。
|
||||
|
||||
接入 APP 不负责历史查询、导出,也不直接写业务查询表。
|
||||
|
||||
### Ingest Core
|
||||
|
||||
新增模块:
|
||||
|
||||
- `modules/core/ingest-facts`
|
||||
|
||||
核心类型:
|
||||
|
||||
```java
|
||||
public record RawFrameFact(
|
||||
String frameId,
|
||||
ProtocolId protocol,
|
||||
String vehicleKey,
|
||||
String vin,
|
||||
String phone,
|
||||
int messageId,
|
||||
int subType,
|
||||
Instant eventTime,
|
||||
Instant receivedAt,
|
||||
String peer,
|
||||
String rawUri,
|
||||
String checksum,
|
||||
long rawSizeBytes,
|
||||
ParseStatus parseStatus,
|
||||
String parseError,
|
||||
Map<String, String> metadata
|
||||
) {}
|
||||
```
|
||||
|
||||
```java
|
||||
public sealed interface DecodedFact permits
|
||||
DecodedFact.Location,
|
||||
DecodedFact.Realtime,
|
||||
DecodedFact.Alarm,
|
||||
DecodedFact.Session,
|
||||
DecodedFact.Register,
|
||||
DecodedFact.Extension {
|
||||
|
||||
String factId();
|
||||
String frameId();
|
||||
ProtocolId protocol();
|
||||
String vehicleKey();
|
||||
String vin();
|
||||
Instant eventTime();
|
||||
Instant receivedAt();
|
||||
String rawUri();
|
||||
Map<String, String> metadata();
|
||||
}
|
||||
```
|
||||
|
||||
`vehicleKey` 是稳定分区键:
|
||||
|
||||
- GB32960:VIN。
|
||||
- JT808:已解析到 VIN 时使用 VIN,否则使用 `jt808:<phone>`。
|
||||
- 未知或异常帧:使用 `unknown:<protocol>:<hash>`。
|
||||
|
||||
### Raw Archive
|
||||
|
||||
新增或重构模块:
|
||||
|
||||
- `modules/sinks/raw-archive-store`
|
||||
|
||||
接口:
|
||||
|
||||
```java
|
||||
public interface RawArchiveWriter {
|
||||
RawArchiveReceipt write(RawArchiveWriteRequest request) throws IOException;
|
||||
}
|
||||
```
|
||||
|
||||
写入回执:
|
||||
|
||||
```java
|
||||
public record RawArchiveReceipt(
|
||||
String rawUri,
|
||||
String checksum,
|
||||
long sizeBytes
|
||||
) {}
|
||||
```
|
||||
|
||||
规则:
|
||||
|
||||
- 原始 bytes 一旦写入,不允许原地修改。
|
||||
- 写入路径可以按内容寻址,也可以按 `protocol/date/vehicleKey/frameId.bin` 生成。
|
||||
- 本地存储需要支持 fsync 能力,接口要预留对象存储实现。
|
||||
- `rawUri` 必须能被历史 API 解析,用于单帧回放。
|
||||
- 异常帧只要有 bytes,也必须写入,并记录 `parseStatus=FAILED`。
|
||||
|
||||
部署约束:
|
||||
|
||||
- Raw archive、TDengine、history 服务可以不在同一台 ECS,但应在同一 VPC 和同一可用区内通过内网访问。
|
||||
- 高频历史查询只访问 TDengine,不访问 raw archive。
|
||||
- Raw archive 只进入单帧详情、问题排查、回放和异步导出路径。
|
||||
- `rawUri` 必须是全局可解析地址,不能是某台 ECS 私有的本地绝对路径。
|
||||
- 第一版本地 raw archive 需要通过可替换接口封装,后续可以切换到 OSS、MinIO 或其它对象存储。
|
||||
|
||||
### Kafka Topic
|
||||
|
||||
使用稳定的版本化 topic:
|
||||
|
||||
- `vehicle.raw-frame.v1`
|
||||
- `vehicle.decoded-fact.v1`
|
||||
- `vehicle.dlq.v1`
|
||||
|
||||
Kafka key:
|
||||
|
||||
```text
|
||||
<protocol>:<vehicleKey>
|
||||
```
|
||||
|
||||
`vehicle.raw-frame.v1` 的 payload 是不含 raw bytes 的 `RawFrameFact`,携带 `rawUri`、checksum、byte size、message id、parse status 和 metadata。
|
||||
|
||||
`vehicle.decoded-fact.v1` 的 payload 是单条 `DecodedFact`,必须引用 `frameId` 和 `rawUri`。
|
||||
|
||||
`vehicle.dlq.v1` 保存存储失败、发布失败、解析失败和写入失败事件,并携带足够元数据,尽量支持从 raw archive 重放。
|
||||
|
||||
### History Writer
|
||||
|
||||
新增或重构 APP:
|
||||
|
||||
- `vehicle-history-app`
|
||||
|
||||
职责:
|
||||
|
||||
- 消费 `vehicle.raw-frame.v1`。
|
||||
- 消费 `vehicle.decoded-fact.v1`。
|
||||
- 批量写 TDengine。
|
||||
- 更新实时状态存储。
|
||||
- 对非法 payload 或存储失败写 DLQ。
|
||||
|
||||
Writer 写入需要幂等:
|
||||
|
||||
- raw fact 以 `(frame_id)` 幂等。
|
||||
- decoded fact 以 `(fact_id)` 幂等。
|
||||
|
||||
## TDengine 模型
|
||||
|
||||
TDengine 是生产历史查询主库。
|
||||
|
||||
### 原始帧超级表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
message_id INT,
|
||||
sub_type INT,
|
||||
event_time TIMESTAMP,
|
||||
raw_uri NCHAR(512),
|
||||
checksum NCHAR(128),
|
||||
raw_size_bytes BIGINT,
|
||||
parse_status NCHAR(16),
|
||||
parse_error NCHAR(512),
|
||||
peer NCHAR(128),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
子表命名:
|
||||
|
||||
```text
|
||||
raw_<protocol>_<hash(vehicleKey)>
|
||||
```
|
||||
|
||||
主要查询模式:
|
||||
|
||||
- 按车辆和时间查询原始帧。
|
||||
- 按协议、消息 ID、解析状态和时间查询原始帧。
|
||||
- 按 `frame_id` 或 `raw_uri` 查询单帧。
|
||||
|
||||
### 位置超级表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg DOUBLE,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
### 实时数据超级表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_realtime (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
speed_kmh DOUBLE,
|
||||
total_mileage_km DOUBLE,
|
||||
battery_soc DOUBLE,
|
||||
total_voltage_v DOUBLE,
|
||||
total_current_a DOUBLE,
|
||||
running_mode NCHAR(64),
|
||||
vehicle_state NCHAR(64),
|
||||
charging_state NCHAR(64),
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
`fields_json` 存放稀疏全字段值,用于低频排查。高频查询字段应提升为明确列,或放入专用扩展表。
|
||||
|
||||
### 报警超级表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_alarms (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
alarm_level NCHAR(32),
|
||||
alarm_code BIGINT,
|
||||
alarm_name NCHAR(128),
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
### 会话和注册表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_sessions (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
session_type NCHAR(32),
|
||||
result_code INT,
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
JT808 注册详情先进入 `vehicle_sessions`,后续可以投影到 MySQL 车辆身份绑定表,用于持久身份解析。
|
||||
|
||||
## ACK 和持久化规则
|
||||
|
||||
对需要协议 ACK 的帧:
|
||||
|
||||
1. Netty 接收到 frame bytes。
|
||||
2. 原始 bytes 写入 raw archive。
|
||||
3. `RawFrameFact` 发布到 Kafka。
|
||||
4. 发送协议 ACK。
|
||||
5. 解码和 `DecodedFact` 发布可以在 ACK 前或 ACK 后执行,但失败必须发 DLQ,并更新 raw frame 解析状态。
|
||||
|
||||
对 GB32960 实时上报和补发帧,这条规则保证不会 ACK 一条无法恢复原始 bytes 的帧。
|
||||
|
||||
对 JT808 注册、鉴权、心跳,也遵循相同 raw archive 和 raw fact 边界。注册 ACK 可以先生成 token,但在 durability boundary 成功前不能 flush 到 channel。
|
||||
|
||||
如果 raw archive 写入失败,不返回成功 ACK。具体是关闭连接还是保留连接,由协议容忍度决定。
|
||||
|
||||
如果 Kafka raw fact 发布失败,durable ACK 帧不返回成功 ACK。非 durable 帧可以写本地 DLQ。
|
||||
|
||||
## 协议解析和扩展
|
||||
|
||||
使用插件契约:
|
||||
|
||||
```java
|
||||
public interface ProtocolFrameDecoder {
|
||||
ProtocolId protocol();
|
||||
DecodeResult decode(RawFrameFact rawFact, byte[] rawBytes);
|
||||
}
|
||||
```
|
||||
|
||||
```java
|
||||
public record DecodeResult(
|
||||
ParseStatus status,
|
||||
List<DecodedFact> facts,
|
||||
String errorMessage,
|
||||
Map<String, String> metadata
|
||||
) {}
|
||||
```
|
||||
|
||||
GB32960:
|
||||
|
||||
- 命令帧映射为会话、实时、位置、报警和扩展事实。
|
||||
- 厂商扩展按平台账号、VIN 或配置 profile 选择。
|
||||
- 全量 raw replay 仍通过读取 `rawUri` 并运行当前解析器完成。
|
||||
|
||||
JT808:
|
||||
|
||||
- 消息 ID 映射为注册、鉴权、心跳、位置、批量位置、多媒体元数据和扩展事实。
|
||||
- 身份解析会更新 `vehicleKey`。
|
||||
- 注册事实携带省、市、厂商、设备 ID、车牌、车牌颜色和 auth token metadata。
|
||||
|
||||
新增协议消息通常不需要改 TDengine schema,除非它成为高频查询维度。未知字段先作为 extension fact 存入 `fields_json`。
|
||||
|
||||
## 查询 API
|
||||
|
||||
API 保持明确,不做一个含糊的统一接口。
|
||||
|
||||
原始帧查询:
|
||||
|
||||
- `GET /api/history/raw-frames`
|
||||
- 过滤条件:`protocol`、`vin`、`phone`、`vehicleKey`、`messageId`、`parseStatus`、`dateFrom`、`dateTo`、`pageSize`、`cursor`。
|
||||
- 返回 frame metadata、raw URI、checksum、parse status,以及可选的紧凑解析摘要。
|
||||
|
||||
单帧详情:
|
||||
|
||||
- `GET /api/history/raw-frames/{frameId}`
|
||||
- 返回 raw metadata 和解码详情。
|
||||
- 支持 `includeRawHex=true`,但必须有严格大小限制。
|
||||
|
||||
位置历史:
|
||||
|
||||
- `GET /api/history/locations`
|
||||
- 过滤条件:`protocol`、`vin`、`phone`、`vehicleKey`、时间范围、cursor。
|
||||
|
||||
实时历史:
|
||||
|
||||
- `GET /api/history/realtime`
|
||||
- 查询历史时序数据。
|
||||
|
||||
实时状态:
|
||||
|
||||
- `GET /api/realtime/vehicles/{vehicleKey}`
|
||||
- 从 Redis 或内存 latest state 读取,不扫 TDengine。
|
||||
|
||||
导出:
|
||||
|
||||
- 导出必须异步。
|
||||
- 请求创建 export job。
|
||||
- Worker 将 TDengine 查询结果流式写入对象存储或本地导出文件。
|
||||
- API 返回 job 状态和下载 URI。
|
||||
|
||||
## 分页
|
||||
|
||||
高 QPS 历史查询使用 cursor pagination,不使用深 offset。
|
||||
|
||||
Cursor 字段:
|
||||
|
||||
```text
|
||||
ts, received_at, fact_id
|
||||
```
|
||||
|
||||
倒序查询谓词:
|
||||
|
||||
```sql
|
||||
WHERE (
|
||||
ts < :cursorTs
|
||||
OR (ts = :cursorTs AND received_at < :cursorReceivedAt)
|
||||
OR (ts = :cursorTs AND received_at = :cursorReceivedAt AND fact_id < :cursorFactId)
|
||||
)
|
||||
```
|
||||
|
||||
默认 page size 为 100,上限为 1000。
|
||||
|
||||
## 容量设计
|
||||
|
||||
假设:
|
||||
|
||||
- 10,000 辆车。
|
||||
- 常规上报间隔 5 到 10 秒。
|
||||
- 预计写入 1,000 到 2,000 frames/s。
|
||||
- 峰值突发系数 3 倍。
|
||||
- 历史查询目标 1,000 requests/s。
|
||||
|
||||
写路径:
|
||||
|
||||
- Netty worker 保持 CPU-light。
|
||||
- Raw archive 写入使用有界异步 worker。
|
||||
- Kafka producer 开启压缩、批量、幂等,并按 vehicle key 分区。
|
||||
- History writer 按超级表和子表批量写 TDengine。
|
||||
- TDengine 子表按 vehicle key 分区,使单车查询足够便宜。
|
||||
|
||||
读路径:
|
||||
|
||||
- 高频查询必须带 vehicle key 和时间范围。
|
||||
- 无边界跨车查询仅限管理接口,并强制限流和限制返回量。
|
||||
- 实时 latest state 从 Redis 或内存状态读取。
|
||||
- Raw 详情只读取单个 raw object 并按需解析。
|
||||
- 字典和元数据接口使用内存缓存。
|
||||
|
||||
背压:
|
||||
|
||||
- Raw archive worker queue 固定上限。
|
||||
- Kafka publish future 必须被跟踪。
|
||||
- Durability boundary 超时时,durable ACK 失败,channel 关闭或被限流。
|
||||
- History writer lag 通过 Kafka consumer lag 和 TDengine batch latency 观察。
|
||||
|
||||
## 迁移计划
|
||||
|
||||
阶段 1:在现有代码旁边建立新的 fact model。
|
||||
|
||||
- 新增 `ingest-facts`。
|
||||
- 新增 raw archive receipt contract。
|
||||
- 新增 raw frame 和 decoded fact 的 Kafka 序列化。
|
||||
- 为 stable id、vehicle key、raw URI 生成规则加测试。
|
||||
|
||||
阶段 2:强化 raw durability boundary。
|
||||
|
||||
- 重构 GB32960 和 JT808 handler,让 ACK 等待 raw archive 和 raw fact publish。
|
||||
- 保留现有解析器。
|
||||
- 向新 Kafka topic 发 decoded facts。
|
||||
|
||||
阶段 3:构建 TDengine writer。
|
||||
|
||||
- 创建 TDengine schema manager。
|
||||
- 写入 raw frame facts。
|
||||
- 写入 location、realtime、alarm、session facts。
|
||||
- 按 stable id 实现幂等写入。
|
||||
|
||||
阶段 4:构建查询 API。
|
||||
|
||||
- Raw frame 查询。
|
||||
- 单帧回放。
|
||||
- 位置历史查询。
|
||||
- 实时历史查询。
|
||||
- 最新实时状态查询。
|
||||
|
||||
阶段 5:切换并下线旧热路径。
|
||||
|
||||
- GB32960/JT808 生产热历史不再依赖 DuckDB/Parquet。
|
||||
- 历史迁移期间保留 raw replay 兼容。
|
||||
- TDengine 覆盖验证完成后,移除协议特定的临时历史存储。
|
||||
|
||||
## 测试策略
|
||||
|
||||
单元测试:
|
||||
|
||||
- `RawFrameFact` 校验和 vehicle key 派生。
|
||||
- Raw archive 路径生成和 checksum。
|
||||
- GB32960 golden frame 到 decoded facts。
|
||||
- JT808 sample frame 到 decoded facts。
|
||||
- ACK boundary 成功和失败场景。
|
||||
- TDengine SQL 生成。
|
||||
|
||||
集成测试:
|
||||
|
||||
- 使用 mocked Kafka producer 和 fake raw archive 启动 APP。
|
||||
- 发送 GB32960 sample frame,验证 raw fact、decoded fact 和 raw archive receipt。
|
||||
- 发送 JT808 注册、鉴权、位置帧,验证 registration、session、location facts。
|
||||
- 模拟 raw archive 失败,验证不发送成功 ACK。
|
||||
- 模拟 Kafka publish 失败,验证 durable ACK 失败。
|
||||
|
||||
存储测试:
|
||||
|
||||
- TDengine schema 初始化。
|
||||
- 批量写入 raw frames 和 decoded facts。
|
||||
- 按 vehicle key 和时间范围查询。
|
||||
- Cursor 分页正确性。
|
||||
- 按 `rawUri` 单帧回放。
|
||||
|
||||
压测:
|
||||
|
||||
- 持续 30 分钟 2,000 frames/s 写入。
|
||||
- 持续 5 分钟 6,000 frames/s 突发写入。
|
||||
- 1,000 QPS location/realtime 历史查询,p95 延迟受控。
|
||||
- 校验 raw archive receipt、Kafka raw fact、TDengine raw frame row 三者数量无缺口。
|
||||
|
||||
## 运维要求
|
||||
|
||||
指标:
|
||||
|
||||
- TCP 连接数。
|
||||
- Frame 接收速率。
|
||||
- Raw archive 写入延迟和失败数。
|
||||
- Kafka 发布延迟和失败数。
|
||||
- 按协议和 message id 统计 decoder 成功/失败数。
|
||||
- TDengine batch size、延迟和失败数。
|
||||
- 各查询 endpoint 的 QPS 和延迟。
|
||||
- Kafka consumer lag。
|
||||
|
||||
日志:
|
||||
|
||||
- 每个失败帧输出一条结构化日志,包含 `frameId`、protocol、vehicle key、message id、raw URI 和 error。
|
||||
- 正常日志不输出完整 raw bytes。
|
||||
- Raw hex 只允许在显式 debug 工具里输出,并限制大小。
|
||||
|
||||
告警:
|
||||
|
||||
- Raw archive 写入失败。
|
||||
- Durable ACK 超时。
|
||||
- Kafka producer 失败。
|
||||
- History writer lag。
|
||||
- TDengine 写入失败。
|
||||
- Raw archive 数量和 TDengine raw frame row 数量不一致。
|
||||
|
||||
## 验收标准
|
||||
|
||||
重设计只有满足以下条件才算完成:
|
||||
|
||||
1. 每一条被接收的 GB32960 和 JT808 帧都会创建 raw archive object 和 TDengine `raw_frames` 行。
|
||||
2. 有 bytes 的异常帧可以从 raw history 查询到,包含 parse status 和错误原因。
|
||||
3. GB32960 和 JT808 位置历史可以按车辆和时间范围 cursor 分页查询。
|
||||
4. GB32960 实时历史可以按车辆和时间范围查询。
|
||||
5. JT808 注册和会话数据可查询,并且可以更新 identity binding。
|
||||
6. 单帧详情可以通过 `rawUri` 重新读取 raw bytes,并使用当前解析器解码。
|
||||
7. 需要 ACK 的帧,在 raw archive 和 raw fact publish 成功前,不会发送成功 ACK。
|
||||
8. 新协议字段可以通过新增 decoder/projector 扩展;只有成为高频查询字段时才需要 TDengine 表迁移。
|
||||
9. 压测证明可以持续 2,000 frames/s 写入,并支持 1,000 QPS 有边界历史查询。
|
||||
10. GB32960/JT808 生产查询不再依赖旧 DuckDB/Parquet 热历史路径。
|
||||
|
||||
## 已固定决策
|
||||
|
||||
第一版实现固定以下选择:
|
||||
|
||||
- 先使用本地 raw archive,实现接口时预留对象存储。
|
||||
- TDengine 是 GB32960/JT808 唯一生产热历史查询库。
|
||||
- 使用协议明确的查询 API,不做一个泛化万能查询 API。
|
||||
- 高频历史查询统一使用 cursor pagination。
|
||||
- Kafka payload 保持轻量,只传 raw 引用,不传 raw bytes。
|
||||
498
docs/superpowers/specs/2026-07-01-go-ingest-redesign-design.md
Normal file
498
docs/superpowers/specs/2026-07-01-go-ingest-redesign-design.md
Normal file
@@ -0,0 +1,498 @@
|
||||
# Go 车辆接入重构设计规格
|
||||
|
||||
## 目标
|
||||
|
||||
使用 Go 重新设计车辆数据接入运行面,让系统回到第一性原则:
|
||||
|
||||
- 接入层只负责可靠收包、协议解析、应答、身份解析和投递。
|
||||
- Kafka 是唯一在线消息通道。
|
||||
- TDengine 保存高吞吐时序明细和完整 RAW 解析 JSON。
|
||||
- MySQL 保存低频配置、身份映射和每日统计窄表。
|
||||
- Redis 保存可重建的准实时状态。
|
||||
- 32960、JT808、宇通 MQTT 使用同一套数据层接口,避免每个协议各自落库。
|
||||
|
||||
第一阶段只做生产链路的骨架和核心数据闭环:接收、解析、Kafka、TDengine、MySQL 统计、Redis 实时查询、ECS 验证。历史查询 API 和业务侧复杂分析放在后续阶段。
|
||||
|
||||
## 参考原则
|
||||
|
||||
本设计只吸收开源项目的边界思想,不复制外部实现代码。
|
||||
|
||||
- GB32960 参考 `DarkInno/gb32960-go-sdk`:连接管理、Handler、Forwarder、鉴权接口分离。
|
||||
- JT808 参考 `cuteLittleDevil/go-jt808`:高并发 Go 原生 TCP、协议核心小而可扩展、事件/适配器机制。
|
||||
- JT808 参考 `fakeyanss/jt808-server-go`:`FramePayload -> PacketData -> JT808Msg -> Reply` 的分层处理、Gateway 模式、保活和版本兼容。
|
||||
- TDengine 参考官方数据模型:按数据采集点建立 supertable,静态维度做 tags,明细数据按子表写入;Go 连接优先使用 WebSocket 驱动。
|
||||
|
||||
## 非目标
|
||||
|
||||
- 第一阶段不继续扩展 Java 服务。
|
||||
- 第一阶段不做统一大查询 API。
|
||||
- 第一阶段不恢复信达 Push。
|
||||
- 第一阶段不把所有协议字段展开成一张超宽 TDengine 表。
|
||||
- Redis 不作为历史事实源。
|
||||
- MySQL 不保存高频原始明细。
|
||||
|
||||
## 目标目录
|
||||
|
||||
新的 Go 运行面放在 `go/vehicle-gateway`,不继续沿用实验性的多 module 分散结构。现有 `go/ingest-edge` 和 `go/vehicle-state` 可以作为迁移素材,最终归并或删除。
|
||||
|
||||
```text
|
||||
go/vehicle-gateway/
|
||||
cmd/
|
||||
gateway/ # 协议接入进程:32960 TCP、JT808 TCP、宇通 MQTT
|
||||
history-writer/ # Kafka -> TDengine RAW/location/mileage points
|
||||
stat-writer/ # Kafka -> MySQL 每日统计窄表
|
||||
realtime-api/ # Redis 准实时查询 API
|
||||
internal/
|
||||
config/ # 环境变量、配置文件、校验
|
||||
gateway/ # TCP/MQTT 生命周期、连接限制、优雅停机
|
||||
protocol/
|
||||
gb32960/ # 32960 编解码
|
||||
jt808/ # 808 编解码
|
||||
yutongmqtt/ # 宇通 MQTT payload 解析
|
||||
identity/ # VIN、phone、device_id、plate 绑定
|
||||
envelope/ # 统一事件模型
|
||||
eventbus/ # Kafka producer/consumer
|
||||
history/ # TDengine adapter
|
||||
stats/ # 每日里程聚合
|
||||
realtime/ # Redis adapter 与快照合并
|
||||
observability/ # 日志、metrics、健康检查
|
||||
```
|
||||
|
||||
## 协议接入设计
|
||||
|
||||
### GB32960
|
||||
|
||||
职责:
|
||||
|
||||
- 监听 TCP `32960`。
|
||||
- 支持车辆登入、实时信息上报、补发信息上报、车辆登出、平台登入/登出、心跳、校时。
|
||||
- 正确返回协议应答。
|
||||
- 完整解析 2016 数据单元:
|
||||
- 整车数据
|
||||
- 驱动电机数据
|
||||
- 燃料电池数据
|
||||
- 发动机数据
|
||||
- 车辆位置数据
|
||||
- 极值数据
|
||||
- 报警数据
|
||||
- 可充电储能装置电压数据
|
||||
- 可充电储能装置温度数据
|
||||
- RAW 中保存完整 parsed JSON。
|
||||
- 关键字段标准化为统一 field:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `soc_percent`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `vehicle_status`
|
||||
- `charge_status`
|
||||
|
||||
### JT808
|
||||
|
||||
职责:
|
||||
|
||||
- 监听 TCP `808`。
|
||||
- 支持 2011/2013 基础兼容,预留 2019 版本标记。
|
||||
- 分层处理:
|
||||
- Frame:`0x7e` 包边界、转义、校验。
|
||||
- Packet:消息头、手机号 BCD、流水号、分包。
|
||||
- Message:注册、鉴权、心跳、注销、位置、批量位置、未知上行。
|
||||
- Reply:注册应答、平台通用应答。
|
||||
- 终端手机号按协议头 BCD 解析,内部同时保留原始 BCD、规范化 phone、去前导零 phone。
|
||||
- 注册帧和鉴权帧写入 808 registration 表。
|
||||
- 如果设备直接上报位置帧,也要走 identity 解析,尝试用 phone、device_id、plate 找到 VIN。
|
||||
- 0200 附加信息必须解析表 27:
|
||||
- `0x01` GPS 总里程,DWORD,单位 0.1 km。
|
||||
- 其他已知附加项结构化放入 `parsed_json.additional`。
|
||||
- 标准化字段:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `altitude_m`
|
||||
- `direction_deg`
|
||||
- `alarm_flag`
|
||||
- `status_flag`
|
||||
|
||||
### 宇通 MQTT
|
||||
|
||||
职责:
|
||||
|
||||
- 使用正式 MQTT 配置订阅生产 topic。
|
||||
- 接收 payload 后解析为统一 envelope。
|
||||
- 按 payload 中的车辆标识解析 VIN、车牌、设备号。
|
||||
- 保存完整 payload 和 parsed JSON。
|
||||
- 标准化字段向 32960/808 对齐:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `soc_percent`
|
||||
- `vehicle_status`
|
||||
|
||||
## 统一 Envelope
|
||||
|
||||
所有协议接入后输出同一结构。
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "string",
|
||||
"trace_id": "string",
|
||||
"protocol": "GB32960|JT808|YUTONG_MQTT",
|
||||
"message_id": "string",
|
||||
"vin": "string",
|
||||
"vehicle_key": "string",
|
||||
"phone": "string",
|
||||
"device_id": "string",
|
||||
"plate": "string",
|
||||
"source_endpoint": "ip:port or mqtt://broker/topic",
|
||||
"event_time_ms": 0,
|
||||
"received_at_ms": 0,
|
||||
"raw_hex": "string",
|
||||
"raw_text": "string",
|
||||
"parsed": {},
|
||||
"fields": {
|
||||
"speed_kmh": 0,
|
||||
"total_mileage_km": 0,
|
||||
"longitude": 0,
|
||||
"latitude": 0
|
||||
},
|
||||
"parse_status": "OK|PARTIAL|BAD_FRAME",
|
||||
"parse_error": ""
|
||||
}
|
||||
```
|
||||
|
||||
规则:
|
||||
|
||||
- `event_id` 使用协议、设备标识、消息流水、事件时间、RAW hash 生成,保证幂等。
|
||||
- Kafka partition key 优先使用 VIN;没有 VIN 时使用 `protocol:phone/device_id`。
|
||||
- `parsed` 保存协议完整结构化字段。
|
||||
- `fields` 只保存跨协议核心字段,用于统计、位置和实时快照。
|
||||
|
||||
## Kafka Topic
|
||||
|
||||
生产只支持 Kafka。
|
||||
|
||||
| Topic | 内容 | Key |
|
||||
|---|---|---|
|
||||
| `vehicle.raw.gb32960.v1` | 32960 完整 RAW envelope | VIN 或 vehicle_key |
|
||||
| `vehicle.raw.jt808.v1` | 808 完整 RAW envelope | VIN 或 phone |
|
||||
| `vehicle.raw.yutong-mqtt.v1` | 宇通 MQTT 完整 RAW envelope | VIN 或 vehicle_key |
|
||||
| `vehicle.event.unified.v1` | 统一轻量事件,用于业务消费 | VIN 或 vehicle_key |
|
||||
|
||||
接入层必须先写 RAW topic,再写 unified event。后续消费者只依赖 Kafka,不直接依赖接入进程内存。
|
||||
|
||||
发布可靠性:
|
||||
|
||||
- Kafka 生产端必须开启同步写入,RAW 写成功后才允许写 unified event。
|
||||
- Kafka 写入必须支持可配置重试、单次写超时和短退避,默认值为 `KAFKA_PUBLISH_ATTEMPTS=3`、`KAFKA_PUBLISH_TIMEOUT_MS=3000`、`KAFKA_PUBLISH_BACKOFF_MS=100`。
|
||||
- gateway 支持本地磁盘 spool/WAL,配置 `KAFKA_SPOOL_DIR` 后启用;Kafka 长时间不可用时,发布失败的 envelope 先原子写入本地 JSON 文件。
|
||||
- spool 补发按文件名顺序执行,补发成功后删除文件;默认补发间隔由 `KAFKA_SPOOL_REPLAY_INTERVAL_MS=1000` 控制。
|
||||
- 如果 RAW 写 Kafka 失败并已落盘,同一个 event 的 unified event 不允许抢先写 Kafka,也必须进入 spool,确保恢复后按 RAW -> unified 顺序补发。
|
||||
- Kafka topic 不允许自动创建,topic 和分区数由部署脚本或运维初始化,避免生产拼写错误造成隐性分流。
|
||||
|
||||
## TDengine 数据库设计
|
||||
|
||||
如果现有 TDengine 库不符合目标,可以新建库并迁移。建议库名:
|
||||
|
||||
```sql
|
||||
CREATE DATABASE IF NOT EXISTS lingniu_vehicle_ts KEEP 7300 DURATION 10 BUFFER 256;
|
||||
```
|
||||
|
||||
### raw_frames
|
||||
|
||||
保存完整 RAW 和完整 parsed JSON。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
event_id NCHAR(64),
|
||||
message_id INT,
|
||||
event_time TIMESTAMP,
|
||||
received_at TIMESTAMP,
|
||||
raw_size_bytes INT,
|
||||
raw_hex BINARY(16374),
|
||||
raw_text BINARY(16374),
|
||||
parsed_json BINARY(16374),
|
||||
fields_json BINARY(4096),
|
||||
parse_status NCHAR(16),
|
||||
parse_error BINARY(1024),
|
||||
source_endpoint NCHAR(128)
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- 子表按 `protocol + vehicle_key hash` 创建。
|
||||
- RAW 查询优先按 tags 和时间范围过滤。
|
||||
- `parsed_json` 是完整协议字段;查询 API 可选择是否返回,避免默认大字段拖慢。
|
||||
|
||||
### vehicle_locations
|
||||
|
||||
只保存位置查询核心字段。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg INT,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
### vehicle_mileage_points
|
||||
|
||||
保存总里程采样点,用于回放重算和查询。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
total_mileage_km DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
## MySQL 数据库设计
|
||||
|
||||
MySQL 只保存配置、身份映射、808 注册鉴权和每日统计窄表。
|
||||
|
||||
### vehicle_identity_binding
|
||||
|
||||
用于把外部标识定位到 VIN。
|
||||
|
||||
```sql
|
||||
CREATE TABLE vehicle_identity_binding (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
plate VARCHAR(32) NULL,
|
||||
phone VARCHAR(32) NULL,
|
||||
device_id VARCHAR(64) NULL,
|
||||
remark VARCHAR(255) NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_vin (vin),
|
||||
KEY idx_plate (plate),
|
||||
KEY idx_phone (phone),
|
||||
KEY idx_device_id (device_id)
|
||||
);
|
||||
```
|
||||
|
||||
### jt808_registration
|
||||
|
||||
只记录 808 独有的注册、鉴权和在线身份信息。
|
||||
|
||||
```sql
|
||||
CREATE TABLE jt808_registration (
|
||||
phone VARCHAR(32) PRIMARY KEY,
|
||||
vin VARCHAR(32) NULL,
|
||||
plate VARCHAR(32) NULL,
|
||||
device_id VARCHAR(64) NULL,
|
||||
manufacturer_id VARCHAR(32) NULL,
|
||||
terminal_model VARCHAR(64) NULL,
|
||||
terminal_id VARCHAR(64) NULL,
|
||||
auth_code VARCHAR(128) NULL,
|
||||
source_endpoint VARCHAR(128) NULL,
|
||||
first_registered_at DATETIME NULL,
|
||||
latest_registered_at DATETIME NULL,
|
||||
latest_authed_at DATETIME NULL,
|
||||
latest_seen_at DATETIME NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
KEY idx_vin (vin),
|
||||
KEY idx_plate (plate),
|
||||
KEY idx_device_id (device_id)
|
||||
);
|
||||
```
|
||||
|
||||
### vehicle_daily_metric
|
||||
|
||||
32960 和 808 的每日里程、每日总里程都进入同一张窄表。
|
||||
|
||||
```sql
|
||||
CREATE TABLE vehicle_daily_metric (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,3) NOT NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
|
||||
KEY idx_stat_date (stat_date),
|
||||
KEY idx_protocol_metric (protocol, metric_key)
|
||||
);
|
||||
```
|
||||
|
||||
指标:
|
||||
|
||||
- `daily_mileage_km = max(total_mileage_km) - min(total_mileage_km)`
|
||||
- `daily_total_mileage_km = max(total_mileage_km)`
|
||||
|
||||
统计消费者可幂等重放。每个样本 upsert 时只维护当天 `min/max total_mileage_km`,不依赖进程内状态。
|
||||
|
||||
## Redis 实时数据设计
|
||||
|
||||
Redis 保存可从 Kafka 重建的数据。
|
||||
|
||||
| Key | 内容 |
|
||||
|---|---|
|
||||
| `vehicle:online:{vin}` | 在线状态、最后活跃时间、协议列表 |
|
||||
| `vehicle:latest:{vin}` | 合并后的 VIN 最新快照 |
|
||||
| `vehicle:latest:{vin}:{protocol}` | 单协议最新快照 |
|
||||
| `vehicle:last_seen` | VIN 到最后接收时间的 sorted set |
|
||||
|
||||
合并规则:
|
||||
|
||||
- 位置以最新 event_time 为准。
|
||||
- 总里程取最新上报值,不做递增假设。
|
||||
- 32960 多帧字段按 field timestamp 合并。
|
||||
- 808 和 MQTT 不覆盖其他协议独有字段。
|
||||
- Redis TTL 用于在线判断,不作为数据删除依据。
|
||||
|
||||
## 数据流
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
source["车辆 / 平台"] --> gateway["Go gateway"]
|
||||
gateway --> parser["protocol parser"]
|
||||
parser --> identity["identity resolver"]
|
||||
identity --> kafkaRaw["Kafka RAW topics"]
|
||||
identity --> kafkaEvent["Kafka unified event"]
|
||||
kafkaRaw --> history["history-writer"]
|
||||
kafkaRaw --> stats["stat-writer"]
|
||||
kafkaEvent --> realtime["realtime consumer"]
|
||||
history --> td["TDengine raw_frames / locations / mileage_points"]
|
||||
stats --> mysql["MySQL vehicle_daily_metric"]
|
||||
realtime --> redis["Redis latest state"]
|
||||
```
|
||||
|
||||
## 错误处理
|
||||
|
||||
- 协议坏帧仍写 RAW topic,`parse_status=BAD_FRAME`。
|
||||
- 可部分解析的帧写 `parse_status=PARTIAL`,保留 parse error。
|
||||
- Kafka 写失败时接入层先按配置重试;重试耗尽后不写 unified event,必须打错误日志和 metrics。
|
||||
- 启用 `KAFKA_SPOOL_DIR` 后,Kafka 重试耗尽会落本地 spool;如果 spool 写失败,才视为接入层最终失败。
|
||||
- TDengine 写失败不提交 Kafka offset。
|
||||
- MySQL 统计写失败不提交 Kafka offset。
|
||||
- Redis 写失败不影响历史和统计,但要通过 metrics 暴露。
|
||||
- 808 相同 phone 新连接时,旧连接应被替换并记录 latest_seen。
|
||||
|
||||
## 部署设计
|
||||
|
||||
生产 ECS 运行 Go 二进制或容器:
|
||||
|
||||
- `vehicle-gateway`
|
||||
- `vehicle-history-writer`
|
||||
- `vehicle-stat-writer`
|
||||
- `vehicle-realtime-api`
|
||||
|
||||
配置来源:
|
||||
|
||||
- 环境变量优先。
|
||||
- Nacos 可作为后续配置中心,但 Go 第一阶段必须能只靠环境变量启动,便于 Portainer 部署和故障恢复。
|
||||
|
||||
TDengine 连接:
|
||||
|
||||
- 优先 WebSocket `taosWS`。
|
||||
- 若当前 ECS 只开放原生端口,可短期兼容 `taosSql`,但代码配置必须显式标记为兼容模式。
|
||||
|
||||
## 验证计划
|
||||
|
||||
### 本地验证
|
||||
|
||||
- 用样例 808 报文验证:
|
||||
- phone BCD 解析。
|
||||
- 0200 经纬度、速度、方向、时间。
|
||||
- 表 27 `0x01` GPS 总里程。
|
||||
- 用真实 32960 报文验证:
|
||||
- 登录应答正确。
|
||||
- 实时帧和补发帧都能写 RAW。
|
||||
- 位置和总里程字段进入统一 fields。
|
||||
- 用宇通 MQTT payload 验证:
|
||||
- payload 完整保存。
|
||||
- 核心字段归一化。
|
||||
- 不连 Kafka 时可输出 JSON log。
|
||||
- 连接 Kafka 时 RAW topic 可消费到 envelope。
|
||||
|
||||
### ECS 验证
|
||||
|
||||
- 部署前先旁路接收或短窗口切流。
|
||||
- 验证端口:
|
||||
- `32960`
|
||||
- `808`
|
||||
- MQTT 订阅连接。
|
||||
- 验证 Kafka topic 有持续写入。
|
||||
- 验证 TDengine:
|
||||
- `raw_frames` 有三种协议数据。
|
||||
- `vehicle_locations` 能按 VIN、协议、时间分页查询。
|
||||
- `vehicle_mileage_points` 有 32960 和 808 总里程采样。
|
||||
- 验证 MySQL:
|
||||
- `vehicle_daily_metric` 有 32960 和 808 的 `daily_mileage_km`、`daily_total_mileage_km`。
|
||||
- 验证 Redis:
|
||||
- 查询 VIN 在线。
|
||||
- 查询 VIN 合并实时快照。
|
||||
- 查询 VIN 分协议实时快照。
|
||||
|
||||
## 验收标准
|
||||
|
||||
- Go gateway 能在 ECS 上接收真实 32960、808、宇通 MQTT 数据。
|
||||
- 三种协议 RAW 都进入 Kafka。
|
||||
- Kafka 短暂写失败时,gateway 按配置重试;单元测试覆盖首次失败后成功和重试耗尽返回错误。
|
||||
- Kafka 长时间不可用时,gateway 可把 RAW/unified 写入本地 spool;单元测试覆盖 RAW 已落盘时 unified 也落盘、恢复后按顺序补发并删除文件。
|
||||
- 三种协议 RAW 和 parsed JSON 都进入 TDengine。
|
||||
- 32960 和 808 的位置进入 `vehicle_locations`。
|
||||
- 32960 和 808 的总里程采样进入 `vehicle_mileage_points`。
|
||||
- 32960 和 808 的每日里程、每日总里程进入 MySQL 窄表。
|
||||
- Redis 可以查询 VIN 在线和实时数据。
|
||||
- 任一消费者宕机后可通过 Kafka offset 继续恢复。
|
||||
- 查询和统计验证使用 ECS 上的真实数据完成。
|
||||
|
||||
## 实施顺序
|
||||
|
||||
1. 创建 `go/vehicle-gateway` 单 module,迁移现有 Go 原型中可用的解析、Kafka、TDengine、MySQL、Redis 代码。
|
||||
2. 先写协议解析单元测试,覆盖 808 样例报文和 32960 核心字段。
|
||||
3. 完成统一 envelope 和 Kafka sink。
|
||||
4. 完成 TDengine schema bootstrap 与 history writer。
|
||||
5. 完成 MySQL schema bootstrap 与 stat writer。
|
||||
6. 完成 Redis realtime writer/API。
|
||||
7. 本地启动 gateway,用 ECS 转发流量验证。
|
||||
8. 构建 Linux 镜像,部署 ECS。
|
||||
9. 按验收标准逐项验证并记录证据。
|
||||
@@ -1,270 +1,202 @@
|
||||
# Target Architecture
|
||||
|
||||
This document defines the target architecture for Lingniu vehicle ingest. The
|
||||
main goal is to keep protocol ingest, raw archive, historical detail storage,
|
||||
realtime state, and statistics as separate responsibilities connected by stable
|
||||
event contracts.
|
||||
This document records the production target for `lingniu-vehicle-ingest`.
|
||||
The project should stay small at runtime: protocol apps ingest and publish,
|
||||
history consumes and indexes, analytics derives metrics, and business systems
|
||||
read through explicit APIs or Kafka.
|
||||
|
||||
Only supported message backbone: Kafka.
|
||||
|
||||
## Active Scope
|
||||
|
||||
Active production protocols:
|
||||
|
||||
| Protocol | Runtime app | Event topic | Raw topic |
|
||||
|---|---|---|---|
|
||||
| GB/T 32960 | `gb32960-ingest-app` | `vehicle.event.gb32960.v1` | `vehicle.raw.gb32960.v1` |
|
||||
| JT/T 808 | `jt808-ingest-app` | `vehicle.event.jt808.v1` | `vehicle.raw.jt808.v1` |
|
||||
| Yutong MQTT | `yutong-mqtt-app` | `vehicle.event.mqtt-yutong.v1` | `vehicle.raw.mqtt-yutong.v1` |
|
||||
|
||||
Xinda Push source, Maven profile, deployment bindings, and history consumers
|
||||
are removed. New optimization work should target the active protocols above.
|
||||
|
||||
JSATL12 attachment upload is also outside the default production reactor. Keep
|
||||
it available through the `optional-attachments` profile until an attachment app
|
||||
is explicitly deployed.
|
||||
|
||||
## Goals
|
||||
|
||||
- The ingest service receives protocol messages, archives raw frames, and
|
||||
publishes normalized full-field events to Kafka.
|
||||
- Historical detail storage consumes Kafka events and stores raw parsed event
|
||||
details in Parquet, queried through DuckDB.
|
||||
- Realtime vehicle state consumes Kafka events and writes hot state to Redis.
|
||||
- Statistics consumes Kafka events and calculates daily operational metrics by
|
||||
configurable rules.
|
||||
- Kafka consumers expose non-throwing ingest results for bad protobuf payloads,
|
||||
unsupported envelopes, and downstream failures so one bad record does not
|
||||
block a vehicle partition.
|
||||
- Consumer services expose `EnvelopeConsumerProcessor` beans. Kafka workers or
|
||||
framework listeners should pass topic, partition, offset, key, and value into
|
||||
the processor; `SKIPPED`, `INVALID_ENVELOPE`, and `FAILED` results are routed
|
||||
through `EnvelopeDeadLetterSink`.
|
||||
- `sink-mq` owns Kafka-specific consumer plumbing: `KafkaEnvelopeConsumerFactory`
|
||||
creates one independent Kafka consumer per service processor binding,
|
||||
`KafkaEnvelopeConsumerRunner` manages the long-running poll loop, and
|
||||
`KafkaEnvelopeDeadLetterSink` publishes failed records to the configured DLQ
|
||||
topic with diagnostic headers.
|
||||
- Business consumers depend on internal fields, not protocol-specific field
|
||||
names.
|
||||
- Keep protocol IO, raw archive, history, latest-state, and statistics as
|
||||
separate responsibilities.
|
||||
- Publish normalized Kafka envelopes with VIN as the partition key whenever a
|
||||
VIN is known.
|
||||
- Store complete raw payload metadata and parsed JSON in TDengine `raw_frames`.
|
||||
- Store query-friendly location rows separately from raw payload JSON.
|
||||
- Keep derived statistics in metric tables, not protocol-specific daily tables.
|
||||
- Use Redis only for optional latest-state APIs that can be rebuilt from Kafka.
|
||||
- Keep optional compatibility modules outside the production hot path.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- The ingest service must not write business tables.
|
||||
- The ingest service must not serve historical detail queries.
|
||||
- Parquet history storage must not store derived daily statistics.
|
||||
- Redis hot state must not become the long-term historical store.
|
||||
- Statistics services must not parse GB/T 32960 frames directly.
|
||||
- Ingest apps do not write business tables.
|
||||
- Ingest apps do not serve history query APIs.
|
||||
- Redis is not the long-term historical store.
|
||||
- File-based event indexes are not part of the current build surface.
|
||||
- `telemetry_fields` parsing and field-trend APIs are owned by a separate
|
||||
field parsing service. This project stores RAW parsed JSON plus compact
|
||||
location rows and does not expose a telemetry-fields history API.
|
||||
|
||||
## Service Boundaries
|
||||
## Runtime Responsibilities
|
||||
|
||||
### Ingest Service
|
||||
### Protocol Apps
|
||||
|
||||
The ingest service owns protocol IO and event publication.
|
||||
Apps:
|
||||
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Accept GB/T 32960, JT/T 808, MQTT, Xinda Push, and future inbound protocols.
|
||||
- Decode frames and acknowledge protocol messages where required.
|
||||
- Emit `RawArchive` before deduplication and rate limiting when raw bytes are
|
||||
available.
|
||||
- Write raw bytes through `sink-archive`.
|
||||
- Map protocol messages into internal telemetry events.
|
||||
- Publish internal events to Kafka with VIN as the partition key.
|
||||
- Accept protocol traffic.
|
||||
- Decode, authenticate, acknowledge, and maintain sessions where the protocol
|
||||
requires it.
|
||||
- Archive raw bytes through `sink-archive` when bytes are available.
|
||||
- Publish parsed event envelopes to Kafka.
|
||||
- Publish raw envelopes to Kafka.
|
||||
- Resolve vehicle identity through the shared MySQL identity binding table.
|
||||
|
||||
The ingest service must not depend on Redis, DuckDB query endpoints, or
|
||||
statistics rule repositories.
|
||||
Protocol apps must stay independent from TDengine history readers, Redis state
|
||||
repositories, and statistic calculators.
|
||||
|
||||
### Event Contract
|
||||
|
||||
The event contract is the only dependency shared by ingest, history, realtime
|
||||
state, and statistics.
|
||||
The Kafka sink owns the protobuf envelope and consumer plumbing.
|
||||
Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped
|
||||
envelopes, and downstream failures are converted to structured results and DLQ
|
||||
records instead of blocking a partition.
|
||||
|
||||
Core event identity:
|
||||
Core identity fields:
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `event_id` | Idempotency key for the parsed event |
|
||||
| `event_id` | Parsed event idempotency key |
|
||||
| `trace_id` | Cross-service trace id |
|
||||
| `vin` | Vehicle VIN and Kafka partition key |
|
||||
| `source_protocol` | Protocol source such as `GB32960`; consumers may map unknown future source names to `UNKNOWN` and preserve the original source in metadata |
|
||||
| `protocol_version` | Source protocol version |
|
||||
| `event_type` | Business event type, for example `REALTIME`, `LOCATION`, `ALARM` |
|
||||
| `event_time` | Device collection time |
|
||||
| `ingest_time` | Platform receive time |
|
||||
| `raw_archive_uri` | Optional reference to archived raw bytes |
|
||||
| `vin` | Vehicle id and Kafka partition key |
|
||||
| `source` | Source protocol, for example `GB32960`, `JT808`, `YUTONG_MQTT` |
|
||||
| `event_time_ms` | Device event time |
|
||||
| `ingest_time_ms` | Platform receive time |
|
||||
| `raw_uri` | `archive://...` reference to raw bytes when available |
|
||||
|
||||
Full-field telemetry is represented as internal field values:
|
||||
Full-field telemetry uses `TelemetrySnapshot` fields with stable keys such as
|
||||
`total_mileage_km`, `speed_kmh`, `longitude`, and `latitude`.
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `field_key` | Stable internal field key such as `total_mileage_km` |
|
||||
| `value_type` | `STRING`, `DOUBLE`, `LONG`, `BOOLEAN`, `INSTANT`, `JSON` |
|
||||
| `value` | String representation used for durable serialization |
|
||||
| `unit` | Stable unit when applicable |
|
||||
| `quality` | `GOOD`, `ESTIMATED`, `INVALID`, `MISSING` |
|
||||
| `source_path` | Protocol source path, for traceability |
|
||||
### History App
|
||||
|
||||
The internal field list is defined in
|
||||
`docs/vehicle-telemetry-internal-fields.md`. New protocol-specific fields start
|
||||
as extension fields and are promoted only after a business meaning and unit are
|
||||
stable.
|
||||
|
||||
### Historical Detail Service
|
||||
|
||||
The historical detail service owns raw parsed event detail storage and exports.
|
||||
App: `vehicle-history-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume normalized telemetry events from Kafka.
|
||||
- Store events in Parquet partitioned by protocol and event date.
|
||||
- Query Parquet through DuckDB.
|
||||
- Provide HTTP APIs for date-range query and export.
|
||||
- Support ascending and descending time order.
|
||||
- Support multi-day concatenated export.
|
||||
- Consume active protocol event topics.
|
||||
- Consume active protocol raw topics.
|
||||
- Write raw frame rows into TDengine `raw_frames`.
|
||||
- Write compact location rows into TDengine location tables.
|
||||
- Keep `payloadJson.parsed` on raw rows for full raw inspection.
|
||||
- Expose paged history APIs for raw frames and locations.
|
||||
- Keep specialized APIs off by default unless explicitly enabled.
|
||||
|
||||
Storage partition:
|
||||
Production default:
|
||||
|
||||
- `TDENGINE_HISTORY_ENABLED=true` in deployment.
|
||||
- `EVENT_FILE_STORE_ENABLED=false`.
|
||||
|
||||
File-based event indexes have been removed; this app should keep history
|
||||
queries on TDengine and raw replay on `archive://...` references.
|
||||
|
||||
### Analytics App
|
||||
|
||||
App: `vehicle-analytics-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume JT808 event envelopes.
|
||||
- Calculate 808 daily mileage from the reported GPS total mileage only:
|
||||
|
||||
```text
|
||||
<root>/
|
||||
protocol=GB32960/
|
||||
date=2026-06-22/
|
||||
part-*.parquet
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
calculation_method = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
The service stores detail events and field snapshots only. It does not calculate
|
||||
or store daily mileage, daily hydrogen usage, or other derived metrics.
|
||||
- Store the metric in `vehicle_stat_metric` with
|
||||
`metric_key = daily_mileage_km`; the local-day minimum and maximum GPS total
|
||||
mileage used for the subtraction stay on the same metric row as calculation
|
||||
source columns.
|
||||
|
||||
### Realtime State Service
|
||||
Runtime state: none outside `vehicle_stat_metric`. There is no JT808-specific
|
||||
daily mileage table. Restart recovery reads the same metric row.
|
||||
|
||||
The realtime state service owns hot vehicle state.
|
||||
### Latest State
|
||||
|
||||
Module: `vehicle-state-service`
|
||||
|
||||
This module is outside the default production reactor. Build it with
|
||||
`-Poptional-latest-state` when Redis latest-state APIs are explicitly needed.
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume normalized telemetry events from Kafka.
|
||||
- Keep latest vehicle state by VIN in Redis.
|
||||
- Keep latest location by VIN in Redis.
|
||||
- Keep latest safety state and hydrogen leak state by VIN in Redis.
|
||||
- Optionally keep short recent windows for operations screens.
|
||||
- Consume normalized Kafka envelopes.
|
||||
- Maintain latest state, location, and safety snapshots in Redis.
|
||||
- Serve latest-state APIs for operational screens.
|
||||
|
||||
Suggested Redis keys:
|
||||
|
||||
| Key | Value |
|
||||
|---|---|
|
||||
| `vehicle:state:{vin}` | Latest full state snapshot |
|
||||
| `vehicle:location:{vin}` | Latest location snapshot |
|
||||
| `vehicle:safety:{vin}` | Latest safety snapshot |
|
||||
| `vehicle:event:last:{vin}` | Latest event identity and timestamps |
|
||||
|
||||
Redis is for millisecond-level reads and recent state. It is not the historical
|
||||
source of truth.
|
||||
|
||||
Current module: `vehicle-state-service`.
|
||||
|
||||
Implemented boundaries:
|
||||
|
||||
- `VehicleStateEnvelopeIngestor` parses Kafka envelope bytes.
|
||||
- `VehicleStateUpdater` maps full-field telemetry snapshots to Redis JSON
|
||||
payloads.
|
||||
- `VehicleStateRepository` isolates storage from event parsing.
|
||||
- `VehicleStateController` exposes read-only latest state APIs.
|
||||
|
||||
### Statistics Service
|
||||
|
||||
The statistics service owns derived operational metrics.
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume normalized telemetry events from Kafka.
|
||||
- Maintain day-level vehicle point state needed for statistics.
|
||||
- Run scheduled and event-triggered calculations.
|
||||
- Store daily results in a statistics database.
|
||||
- Support per-vehicle calculation rules.
|
||||
|
||||
Initial daily mileage strategies:
|
||||
|
||||
| Strategy | Formula |
|
||||
|---|---|
|
||||
| `CURRENT_LAST_MINUS_PREVIOUS_LAST` | Current day last valid total mileage minus previous day last valid total mileage |
|
||||
| `DAY_MAX_MINUS_DAY_MIN` | Current day max valid total mileage minus current day min valid total mileage |
|
||||
|
||||
Hydrogen and electricity statistics use internal fields from the same event
|
||||
contract. Refueling and charging resets must be handled in statistics logic, not
|
||||
in protocol mappers.
|
||||
|
||||
Current module: `vehicle-stat-service`.
|
||||
|
||||
Implemented boundaries:
|
||||
|
||||
- `VehicleStatEnvelopeIngestor` parses Kafka envelope bytes.
|
||||
- `VehicleStatEventProcessor` extracts statistic source points from internal
|
||||
fields such as `total_mileage_km`.
|
||||
- `VehicleStatRepository` isolates point/result persistence from calculation.
|
||||
- `VehicleStatRuleRepository` isolates per-vehicle calculation rules.
|
||||
- `DailyVehicleStatService` calculates and saves day-level results.
|
||||
Redis state must be rebuildable from Kafka replay and should not be used as the
|
||||
source of truth for historical queries.
|
||||
|
||||
## Data Flow
|
||||
|
||||
Default production flow:
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
vehicle["Vehicle / Platform"] --> ingest["ingest service"]
|
||||
vehicle["Vehicles / Platforms"] --> ingest["Protocol apps"]
|
||||
ingest --> archive["sink-archive<br/>raw bytes"]
|
||||
ingest --> kafka["Kafka<br/>full-field telemetry events"]
|
||||
kafka --> history["event-history-service"]
|
||||
kafka --> state["vehicle-state-service"]
|
||||
kafka --> stat["vehicle-stat-service"]
|
||||
history --> parquet["Parquet files"]
|
||||
history --> duckdb["DuckDB query"]
|
||||
state --> redis["Redis hot state"]
|
||||
stat --> statdb["Statistics DB"]
|
||||
ingest --> kafka["Kafka<br/>event + raw envelopes"]
|
||||
kafka --> history["vehicle-history-app"]
|
||||
kafka --> analytics["vehicle-analytics-app"]
|
||||
history --> tdengine["TDengine<br/>raw_frames + locations"]
|
||||
analytics --> mysql["MySQL<br/>vehicle_stat_metric"]
|
||||
```
|
||||
|
||||
## Kafka Topic Boundaries
|
||||
|
||||
Suggested topics:
|
||||
|
||||
| Topic | Producer | Consumer |
|
||||
|---|---|---|
|
||||
| `vehicle.realtime` | protocol ingress | history, realtime state, statistics |
|
||||
| `vehicle.location` | protocol ingress | history, realtime state, statistics |
|
||||
| `vehicle.alarm` | protocol ingress | history, realtime state |
|
||||
| `vehicle.session` | protocol ingress | history |
|
||||
| `vehicle.media.meta` | protocol ingress | history, media tooling |
|
||||
| `vehicle.raw.archive` | optional archive publisher | replay or audit services |
|
||||
| `vehicle.dlq` | any consumer | operations and replay tooling |
|
||||
|
||||
All telemetry event topics use VIN as the partition key to preserve per-vehicle
|
||||
ordering.
|
||||
Optional latest-state flow, enabled only with `-Poptional-latest-state`, consumes
|
||||
the same Kafka envelopes and rebuilds Redis snapshots outside the default
|
||||
production deployment.
|
||||
|
||||
## Failure Handling
|
||||
|
||||
- Ingest must keep receiving traffic when history, Redis, or statistics services
|
||||
are down.
|
||||
- Kafka publish failures go through the existing sink circuit breaker and DLQ
|
||||
strategy.
|
||||
- History consumer commits Kafka offsets only after Parquet append succeeds.
|
||||
- Redis state consumer can replay from Kafka to rebuild hot state.
|
||||
- Statistics consumer stores checkpoints and can recompute a day from historical
|
||||
detail events when rules change.
|
||||
- Raw archive failures are logged and alerted, but parsed event publication
|
||||
remains independent.
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Full-Field Contract And Historical Detail
|
||||
|
||||
- Add a full-field telemetry contract to `ingest-api`.
|
||||
- Map GB/T 32960 parsed blocks into internal field snapshots.
|
||||
- Extend Kafka envelope schema for full-field events.
|
||||
- Make `event-file-store` store full-field telemetry records.
|
||||
- Add query and export APIs for date-range historical detail.
|
||||
|
||||
### Phase 2: Redis Hot State
|
||||
|
||||
- Add a Redis-backed realtime state module.
|
||||
- Consume full-field telemetry events.
|
||||
- Store latest state, location, and safety snapshots by VIN.
|
||||
- Provide query APIs for operations screens.
|
||||
|
||||
### Phase 3: Statistics Service
|
||||
|
||||
- Add a statistics module with rule configuration.
|
||||
- Consume full-field telemetry events.
|
||||
- Implement daily mileage strategies.
|
||||
- Add result persistence boundary and calculation tests.
|
||||
- Protocol apps continue accepting traffic when history, analytics, or Redis
|
||||
consumers are unavailable.
|
||||
- Kafka producer failures are handled by sink retry/circuit-breaker behavior and
|
||||
DLQ topics where configured.
|
||||
- History consumers should commit offsets only after TDengine writes succeed.
|
||||
- Analytics daily mileage state can recover from `vehicle_stat_metric` or Kafka
|
||||
replay.
|
||||
- Raw archive failures must be observable, but event publication and history
|
||||
indexing remain separate concerns.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- Ingest has no compile-time dependency on history query controllers, Redis
|
||||
state repositories, or statistics calculators.
|
||||
- Every service boundary communicates through Kafka events or explicit public
|
||||
APIs, not shared mutable state.
|
||||
- GB/T 32960 detail fields are traceable from internal field key back to
|
||||
protocol source path.
|
||||
- Historical query works across multiple days with ascending and descending
|
||||
order.
|
||||
- Redis state can be rebuilt from Kafka replay.
|
||||
- Daily mileage can be calculated by at least the two initial strategies.
|
||||
- Tests cover mapper output, Kafka serialization, Parquet query order, Redis
|
||||
state update, and mileage strategy calculation.
|
||||
- Active production deployment contains GB32960, JT808, Yutong MQTT,
|
||||
vehicle-history, and vehicle-analytics apps.
|
||||
- Xinda Push source and deployment bindings are removed.
|
||||
- JSATL12 is absent from the default Maven reactor and available through
|
||||
`optional-attachments`.
|
||||
- vehicle-state-service is absent from the default Maven reactor and available
|
||||
through `optional-latest-state`.
|
||||
- raw-archive-store prototype has been removed; raw bytes are written through
|
||||
`sink-archive`.
|
||||
- History APIs read from TDengine; file-based event indexes are not maintained.
|
||||
- Raw frame queries can return complete parsed JSON through `payloadJson.parsed`.
|
||||
- Location queries page over compact rows and reference raw records instead of
|
||||
duplicating full raw JSON.
|
||||
- JT808 daily mileage is stored only in `vehicle_stat_metric`.
|
||||
- Runtime state needed for 808 mileage is also in `vehicle_stat_metric`.
|
||||
- Build validation covers the active modules and their composition tests.
|
||||
|
||||
29
go/vehicle-gateway/Dockerfile
Normal file
29
go/vehicle-gateway/Dockerfile
Normal file
@@ -0,0 +1,29 @@
|
||||
FROM golang:1.26-bookworm AS build
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/gateway ./cmd/gateway \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/history-writer ./cmd/history-writer \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/stat-writer ./cmd/stat-writer \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/realtime-api ./cmd/realtime-api
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/gateway /app/gateway
|
||||
COPY --from=build /out/history-writer /app/history-writer
|
||||
COPY --from=build /out/stat-writer /app/stat-writer
|
||||
COPY --from=build /out/realtime-api /app/realtime-api
|
||||
|
||||
ENV TZ=Asia/Shanghai
|
||||
|
||||
EXPOSE 808 32960 20210
|
||||
|
||||
CMD ["/app/gateway"]
|
||||
223
go/vehicle-gateway/cmd/gateway/main.go
Normal file
223
go/vehicle-gateway/cmd/gateway/main.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/gateway"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-gateway")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
sink, err := buildSink(ctx, logger)
|
||||
if err != nil {
|
||||
logger.Error("build sink failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer sink.Close()
|
||||
resolver, closeResolver, err := buildIdentityResolver(ctx, logger)
|
||||
if err != nil {
|
||||
logger.Error("build identity resolver failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer closeResolver()
|
||||
|
||||
protocols := []gateway.TCPProtocol{
|
||||
{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: env("GB32960_TCP_ADDR", ":32960"),
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
Respond: gb32960.AutoResponse,
|
||||
},
|
||||
{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Addr: env("JT808_TCP_ADDR", ":808"),
|
||||
Extract: jt808.ExtractFrames,
|
||||
Parse: jt808.ParseFrame,
|
||||
Respond: jt808.NewAutoResponder(env("JT808_REGISTER_AUTH_CODE", "g7gps")).Respond,
|
||||
},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, len(protocols))
|
||||
for _, protocol := range protocols {
|
||||
server, err := gateway.NewTCPServer(gateway.TCPServerConfig{
|
||||
Protocol: protocol,
|
||||
Sink: sink,
|
||||
Resolver: resolver,
|
||||
Logger: logger,
|
||||
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
|
||||
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
|
||||
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := server.ListenAndServe(ctx); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}()
|
||||
}
|
||||
if envBool("YUTONG_MQTT_ENABLED", false) {
|
||||
client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{
|
||||
EndpointName: env("YUTONG_MQTT_ENDPOINT", env("YUTONG_MQTT_ENDPOINT_NAME", "yutong")),
|
||||
Broker: env("YUTONG_MQTT_URI", ""),
|
||||
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
|
||||
Username: env("YUTONG_MQTT_USERNAME", ""),
|
||||
Password: env("YUTONG_MQTT_PASSWORD", ""),
|
||||
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
|
||||
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
|
||||
CleanSession: envBool("YUTONG_MQTT_CLEAN_SESSION", false),
|
||||
KeepAlive: time.Duration(envInt("YUTONG_MQTT_KEEP_ALIVE_SECONDS", 20)) * time.Second,
|
||||
ConnectTimeout: time.Duration(envInt("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", 10)) * time.Second,
|
||||
TLSCACertPath: env("YUTONG_MQTT_TLS_CA_PEM", ""),
|
||||
TLSClientCertPath: env("YUTONG_MQTT_TLS_CLIENT_PEM", ""),
|
||||
TLSClientKeyPath: env("YUTONG_MQTT_TLS_CLIENT_KEY", ""),
|
||||
TLSHostnameVerification: envBool("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", true),
|
||||
Sink: sink,
|
||||
Resolver: resolver,
|
||||
Logger: logger,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("build yutong mqtt client failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := client.Start(ctx); err != nil {
|
||||
logger.Error("start yutong mqtt client failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
logger.Info("yutong mqtt client started")
|
||||
}
|
||||
|
||||
logger.Info("vehicle gateway started")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case err := <-errs:
|
||||
logger.Error("vehicle gateway listener failed", "error", err)
|
||||
stop()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.Resolver, func(), error) {
|
||||
dsn := env("IDENTITY_MYSQL_DSN", strings.TrimSpace(os.Getenv("MYSQL_DSN")))
|
||||
if strings.TrimSpace(dsn) == "" {
|
||||
logger.Warn("identity mysql dsn is empty; using noop identity resolver")
|
||||
return identity.NoopResolver{}, func() {}, nil
|
||||
}
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding")
|
||||
logger.Info("identity mysql resolver enabled", "table", table)
|
||||
return identity.NewMySQLResolver(db, table), func() { _ = db.Close() }, nil
|
||||
}
|
||||
|
||||
func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error) {
|
||||
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
|
||||
if len(brokers) == 0 {
|
||||
logger.Warn("KAFKA_BROKERS is empty; using log sink")
|
||||
return eventbus.NewLogSink(logger), nil
|
||||
}
|
||||
sink, err := eventbus.NewKafkaSink(eventbus.KafkaConfig{
|
||||
Brokers: brokers,
|
||||
RawTopics: map[envelope.Protocol]string{
|
||||
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1"),
|
||||
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1"),
|
||||
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1"),
|
||||
},
|
||||
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1"),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out eventbus.Sink = eventbus.NewRetryingSink(sink, eventbus.RetryConfig{
|
||||
Attempts: envInt("KAFKA_PUBLISH_ATTEMPTS", 3),
|
||||
Backoff: time.Duration(envInt("KAFKA_PUBLISH_BACKOFF_MS", 100)) * time.Millisecond,
|
||||
AttemptTimeout: time.Duration(envInt("KAFKA_PUBLISH_TIMEOUT_MS", 3000)) * time.Millisecond,
|
||||
})
|
||||
spoolDir := strings.TrimSpace(os.Getenv("KAFKA_SPOOL_DIR"))
|
||||
if spoolDir != "" {
|
||||
durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{Directory: spoolDir})
|
||||
interval := time.Duration(envInt("KAFKA_SPOOL_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond
|
||||
go durable.ReplayLoop(ctx, interval, func(err error) {
|
||||
logger.Warn("kafka spool replay failed", "error", err)
|
||||
})
|
||||
logger.Info("kafka durable spool enabled", "dir", spoolDir, "replay_interval_ms", interval.Milliseconds())
|
||||
out = durable
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
var out int
|
||||
for _, r := range value {
|
||||
if r < '0' || r > '9' {
|
||||
return fallback
|
||||
}
|
||||
out = out*10 + int(r-'0')
|
||||
}
|
||||
if out <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func envBool(key string, fallback bool) bool {
|
||||
value := strings.ToLower(strings.TrimSpace(os.Getenv(key)))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value == "1" || value == "true" || value == "yes" || value == "on"
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
123
go/vehicle-gateway/cmd/history-writer/main.go
Normal file
123
go/vehicle-gateway/cmd/history-writer/main.go
Normal file
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
_ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-history-writer")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
cfg := loadConfig()
|
||||
db, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
|
||||
if err != nil {
|
||||
logger.Error("tdengine open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
logger.Error("tdengine ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
writer := history.NewWriter(db)
|
||||
if cfg.EnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
|
||||
logger.Error("tdengine schema bootstrap failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
GroupID: cfg.KafkaGroup,
|
||||
GroupTopics: cfg.KafkaTopics,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
|
||||
logger.Info("history writer started",
|
||||
"driver", cfg.TDengineDriver,
|
||||
"group", cfg.KafkaGroup,
|
||||
"topics", strings.Join(cfg.KafkaTopics, ","))
|
||||
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := writer.AppendAll(ctx, env); err != nil {
|
||||
logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
KafkaBrokers []string
|
||||
KafkaTopics []string
|
||||
KafkaGroup string
|
||||
TDengineDriver string
|
||||
TDengineDSN string
|
||||
TDengineDatabase string
|
||||
EnsureSchema bool
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
return config{
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1")),
|
||||
KafkaGroup: env("KAFKA_GROUP", "go-history-writer"),
|
||||
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
|
||||
TDengineDSN: env("TDENGINE_DSN", ""),
|
||||
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
|
||||
EnsureSchema: env("TDENGINE_ENSURE_SCHEMA", "true") != "false",
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
198
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
198
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
@@ -0,0 +1,198 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/segmentio/kafka-go"
|
||||
_ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-realtime-api")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: env("REDIS_ADDR", "127.0.0.1:6379"),
|
||||
Username: env("REDIS_USERNAME", ""),
|
||||
Password: env("REDIS_PASSWORD", ""),
|
||||
DB: envInt("REDIS_DB", 0),
|
||||
})
|
||||
defer client.Close()
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
logger.Error("redis ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
repository := realtime.NewRepository(client, realtime.Config{
|
||||
OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", 600)) * time.Second,
|
||||
})
|
||||
if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
|
||||
go consumeKafka(ctx, logger, repository, brokers)
|
||||
} else {
|
||||
logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository))
|
||||
closeStats := func() {}
|
||||
if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" {
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
logger.Error("stats mysql open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
logger.Error("stats mysql ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
closeStats = func() { _ = db.Close() }
|
||||
mux.Handle("/api/stats/daily-metrics", stats.NewMetricHandler(stats.NewMetricRepository(db)))
|
||||
logger.Info("stats mysql query enabled")
|
||||
} else {
|
||||
mux.HandleFunc("/api/stats/daily-metrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "MYSQL_DSN is not configured"})
|
||||
})
|
||||
logger.Warn("MYSQL_DSN is empty; stats query api disabled")
|
||||
}
|
||||
defer closeStats()
|
||||
closeHistory := func() {}
|
||||
if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" {
|
||||
driver := env("TDENGINE_DRIVER", "taosWS")
|
||||
db, err := sql.Open(driver, dsn)
|
||||
if err != nil {
|
||||
logger.Error("history tdengine open failed", "driver", driver, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
logger.Error("history tdengine ping failed", "driver", driver, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
closeHistory = func() { _ = db.Close() }
|
||||
database := env("TDENGINE_DATABASE", history.DefaultDatabase)
|
||||
mux.Handle("/api/history/raw-frames", history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)))
|
||||
mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database)))
|
||||
mux.Handle("/api/history/mileage-points", history.NewMileagePointHandler(history.NewMileagePointRepository(db, database)))
|
||||
logger.Info("history query enabled", "driver", driver, "database", database)
|
||||
} else {
|
||||
historyUnavailable := func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"})
|
||||
}
|
||||
mux.HandleFunc("/api/history/raw-frames", historyUnavailable)
|
||||
mux.HandleFunc("/api/history/locations", historyUnavailable)
|
||||
mux.HandleFunc("/api/history/mileage-points", historyUnavailable)
|
||||
logger.Warn("TDENGINE_DSN is empty; history query api disabled")
|
||||
}
|
||||
defer closeHistory()
|
||||
|
||||
server := &http.Server{
|
||||
Addr: env("HTTP_ADDR", ":20210"),
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = server.Shutdown(shutdownCtx)
|
||||
}()
|
||||
|
||||
logger.Info("realtime api started", "addr", server.Addr)
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
logger.Error("http server failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func consumeKafka(ctx context.Context, logger interface {
|
||||
Info(string, ...any)
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, repository *realtime.Repository, brokers []string) {
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: brokers,
|
||||
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
|
||||
GroupTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.event.unified.v1")),
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
logger.Info("realtime kafka consumer started", "topics", env("KAFKA_TOPICS", "vehicle.event.unified.v1"))
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := repository.Update(ctx, env); err != nil {
|
||||
logger.Error("redis realtime update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
122
go/vehicle-gateway/cmd/stat-writer/main.go
Normal file
122
go/vehicle-gateway/cmd/stat-writer/main.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-stat-writer")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
cfg := loadConfig()
|
||||
db, err := sql.Open("mysql", cfg.MySQLDSN)
|
||||
if err != nil {
|
||||
logger.Error("mysql open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
logger.Error("mysql ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
writer := stats.NewWriter(db, cfg.Location)
|
||||
if cfg.EnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx); err != nil {
|
||||
logger.Error("mysql schema bootstrap failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
GroupID: cfg.KafkaGroup,
|
||||
GroupTopics: cfg.KafkaTopics,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
|
||||
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","))
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := writer.Append(ctx, env); err != nil {
|
||||
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
KafkaBrokers []string
|
||||
KafkaTopics []string
|
||||
KafkaGroup string
|
||||
MySQLDSN string
|
||||
EnsureSchema bool
|
||||
Location *time.Location
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
loc, err := time.LoadLocation(env("LOCAL_TZ", "Asia/Shanghai"))
|
||||
if err != nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
return config{
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1")),
|
||||
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
|
||||
MySQLDSN: env("MYSQL_DSN", ""),
|
||||
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",
|
||||
Location: loc,
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
30
go/vehicle-gateway/go.mod
Normal file
30
go/vehicle-gateway/go.mod
Normal file
@@ -0,0 +1,30 @@
|
||||
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/alicebob/miniredis/v2 v2.35.0
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/taosdata/driver-go/v3 v3.8.1
|
||||
golang.org/x/text v0.29.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.15.9 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
github.com/yuin/gopher-lua v1.1.1 // indirect
|
||||
golang.org/x/net v0.44.0 // indirect
|
||||
golang.org/x/sync v0.17.0 // indirect
|
||||
)
|
||||
73
go/vehicle-gateway/go.sum
Normal file
73
go/vehicle-gateway/go.sum
Normal file
@@ -0,0 +1,73 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE=
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
|
||||
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
||||
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/taosdata/driver-go/v3 v3.8.1 h1:kkd4ABsGiU+oXDbsw/sic985LKAvnpF9Gb/TEunTnLE=
|
||||
github.com/taosdata/driver-go/v3 v3.8.1/go.mod h1:S6OGOinfR0xxxaMGsvBi9cLkYxEIW1p6qqr8QJATTlg=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
|
||||
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
95
go/vehicle-gateway/internal/envelope/envelope.go
Normal file
95
go/vehicle-gateway/internal/envelope/envelope.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Protocol string
|
||||
|
||||
const (
|
||||
ProtocolGB32960 Protocol = "GB32960"
|
||||
ProtocolJT808 Protocol = "JT808"
|
||||
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||
)
|
||||
|
||||
type ParseStatus string
|
||||
|
||||
const (
|
||||
ParseOK ParseStatus = "OK"
|
||||
ParsePartial ParseStatus = "PARTIAL"
|
||||
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||
)
|
||||
|
||||
const (
|
||||
FieldSpeedKMH = "speed_kmh"
|
||||
FieldTotalMileageKM = "total_mileage_km"
|
||||
FieldLongitude = "longitude"
|
||||
FieldLatitude = "latitude"
|
||||
FieldSOCPercent = "soc_percent"
|
||||
)
|
||||
|
||||
type FrameEnvelope struct {
|
||||
EventID string `json:"event_id"`
|
||||
TraceID string `json:"trace_id"`
|
||||
Protocol Protocol `json:"protocol"`
|
||||
MessageID string `json:"message_id"`
|
||||
Sequence uint16 `json:"sequence"`
|
||||
VIN string `json:"vin,omitempty"`
|
||||
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
Parsed map[string]any `json:"parsed,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
ParseStatus ParseStatus `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) VehicleKey() string {
|
||||
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
return string(e.Protocol) + ":unknown"
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) StableEventID() string {
|
||||
if strings.TrimSpace(e.EventID) != "" {
|
||||
return e.EventID
|
||||
}
|
||||
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||
sum := sha256.Sum256([]byte(input))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) KafkaKey() []byte {
|
||||
return []byte(e.VehicleKey())
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||
if e.EventID == "" {
|
||||
e.EventID = e.StableEventID()
|
||||
}
|
||||
if e.ParseStatus == "" {
|
||||
e.ParseStatus = ParseOK
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
55
go/vehicle-gateway/internal/envelope/envelope_test.go
Normal file
55
go/vehicle-gateway/internal/envelope/envelope_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||
e := FrameEnvelope{
|
||||
Protocol: ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013307795425",
|
||||
Sequence: 1,
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7e02000000ff7e",
|
||||
}
|
||||
a := e.StableEventID()
|
||||
b := e.StableEventID()
|
||||
if a == "" || a != b {
|
||||
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeMarshalDefaults(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolGB32960, VIN: "LNBVIN00000000002", MessageID: "0x02"}
|
||||
payload, err := e.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalJSONBytes() error = %v", err)
|
||||
}
|
||||
var decoded FrameEnvelope
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if decoded.EventID == "" {
|
||||
t.Fatal("event id should be filled")
|
||||
}
|
||||
if decoded.ParseStatus != ParseOK {
|
||||
t.Fatalf("parse status = %q", decoded.ParseStatus)
|
||||
}
|
||||
}
|
||||
187
go/vehicle-gateway/internal/eventbus/durable_sink.go
Normal file
187
go/vehicle-gateway/internal/eventbus/durable_sink.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type DurableConfig struct {
|
||||
Directory string
|
||||
}
|
||||
|
||||
type DurableSink struct {
|
||||
delegate Sink
|
||||
dir string
|
||||
|
||||
mu sync.Mutex
|
||||
seq uint64
|
||||
rawPending map[string]struct{}
|
||||
}
|
||||
|
||||
type durableRecord struct {
|
||||
Kind string `json:"kind"`
|
||||
Envelope envelope.FrameEnvelope `json:"envelope"`
|
||||
}
|
||||
|
||||
func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
|
||||
if delegate == nil {
|
||||
panic("durable delegate sink must not be nil")
|
||||
}
|
||||
return &DurableSink{
|
||||
delegate: delegate,
|
||||
dir: strings.TrimSpace(cfg.Directory),
|
||||
rawPending: map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if err := s.delegate.PublishRaw(ctx, env); err == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.spool("raw", env); err != nil {
|
||||
return err
|
||||
}
|
||||
s.markRawPending(env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DurableSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if s.isRawPending(env) {
|
||||
return s.spool("unified", env)
|
||||
}
|
||||
if err := s.delegate.PublishUnified(ctx, env); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.spool("unified", env)
|
||||
}
|
||||
|
||||
func (s *DurableSink) ReplayOnce(ctx context.Context) error {
|
||||
files, err := filepath.Glob(filepath.Join(s.dir, "*.json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(files)
|
||||
for _, file := range files {
|
||||
record, err := readDurableRecord(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.publishRecord(ctx, record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(file); err != nil {
|
||||
return err
|
||||
}
|
||||
if record.Kind == "raw" {
|
||||
s.clearRawPending(record.Envelope)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DurableSink) ReplayLoop(ctx context.Context, interval time.Duration, onError func(error)) {
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := s.ReplayOnce(ctx); err != nil && onError != nil {
|
||||
onError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) Close() error {
|
||||
return s.delegate.Close()
|
||||
}
|
||||
|
||||
func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) error {
|
||||
switch record.Kind {
|
||||
case "raw":
|
||||
return s.delegate.PublishRaw(ctx, record.Envelope)
|
||||
case "unified":
|
||||
return s.delegate.PublishUnified(ctx, record.Envelope)
|
||||
default:
|
||||
return fmt.Errorf("unknown durable record kind %q", record.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
|
||||
if s.dir == "" {
|
||||
return fmt.Errorf("durable spool directory is empty")
|
||||
}
|
||||
if env.EventID == "" {
|
||||
env.EventID = env.StableEventID()
|
||||
}
|
||||
if env.ParseStatus == "" {
|
||||
env.ParseStatus = envelope.ParseOK
|
||||
}
|
||||
if err := os.MkdirAll(s.dir, 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := s.nextFileName(env, kind)
|
||||
path := filepath.Join(s.dir, name)
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, payload, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.seq++
|
||||
eventID := env.StableEventID()
|
||||
if len(eventID) > 12 {
|
||||
eventID = eventID[:12]
|
||||
}
|
||||
return fmt.Sprintf("%020d-%06d-%s-%s.json", time.Now().UnixNano(), s.seq, kind, eventID)
|
||||
}
|
||||
|
||||
func (s *DurableSink) markRawPending(env envelope.FrameEnvelope) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.rawPending[env.StableEventID()] = struct{}{}
|
||||
}
|
||||
|
||||
func (s *DurableSink) clearRawPending(env envelope.FrameEnvelope) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.rawPending, env.StableEventID())
|
||||
}
|
||||
|
||||
func (s *DurableSink) isRawPending(env envelope.FrameEnvelope) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, ok := s.rawPending[env.StableEventID()]
|
||||
return ok
|
||||
}
|
||||
|
||||
func readDurableRecord(path string) (durableRecord, error) {
|
||||
var record durableRecord
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return record, err
|
||||
}
|
||||
return record, json.Unmarshal(payload, &record)
|
||||
}
|
||||
153
go/vehicle-gateway/internal/eventbus/durable_sink_test.go
Normal file
153
go/vehicle-gateway/internal/eventbus/durable_sink_test.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
|
||||
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
|
||||
env := durableTestEnvelope()
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
|
||||
if delegate.rawCalls != 1 {
|
||||
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
|
||||
}
|
||||
if delegate.unifiedCalls != 0 {
|
||||
t.Fatalf("unified should not be delegated while raw is spooled, calls = %d", delegate.unifiedCalls)
|
||||
}
|
||||
files := spoolFiles(t, dir)
|
||||
if len(files) != 2 {
|
||||
t.Fatalf("spool files = %d, want 2: %#v", len(files), files)
|
||||
}
|
||||
assertSpoolKind(t, files[0], "raw")
|
||||
assertSpoolKind(t, files[1], "unified")
|
||||
}
|
||||
|
||||
func TestDurableSinkReplayPublishesInFileOrderAndDeletesFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
|
||||
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
|
||||
env := durableTestEnvelope()
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
delegate.rawErrors = nil
|
||||
|
||||
if err := sink.ReplayOnce(context.Background()); err != nil {
|
||||
t.Fatalf("ReplayOnce() error = %v", err)
|
||||
}
|
||||
|
||||
got := delegate.calls
|
||||
want := []string{"raw", "raw", "unified"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("calls = %#v, want %#v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("calls[%d] = %q, want %q; all=%#v", i, got[i], want[i], got)
|
||||
}
|
||||
}
|
||||
if files := spoolFiles(t, dir); len(files) != 0 {
|
||||
t.Fatalf("spool files after replay = %#v, want none", files)
|
||||
}
|
||||
}
|
||||
|
||||
func durableTestEnvelope() envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013079963379",
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
Sequence: 183,
|
||||
EventTimeMS: 1782903940000,
|
||||
ReceivedAtMS: 1782917751903,
|
||||
RawHex: "0200002201307996337900B7",
|
||||
}
|
||||
}
|
||||
|
||||
func spoolFiles(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
matches, err := filepath.Glob(filepath.Join(dir, "*.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob spool files: %v", err)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
func assertSpoolKind(t *testing.T, path string, kind string) {
|
||||
t.Helper()
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read spool file: %v", err)
|
||||
}
|
||||
if !containsString(string(payload), `"kind":"`+kind+`"`) {
|
||||
t.Fatalf("spool file %s payload = %s, want kind %q", path, payload, kind)
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(value string, needle string) bool {
|
||||
return len(needle) == 0 || (len(value) >= len(needle) && indexString(value, needle) >= 0)
|
||||
}
|
||||
|
||||
func indexString(value string, needle string) int {
|
||||
for i := 0; i+len(needle) <= len(value); i++ {
|
||||
if value[i:i+len(needle)] == needle {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
var errSpoolTest = errors.New("delegate unavailable")
|
||||
|
||||
type scriptedSink struct {
|
||||
rawErrors []error
|
||||
unifiedErrors []error
|
||||
rawCalls int
|
||||
unifiedCalls int
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (s *scriptedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
s.rawCalls++
|
||||
s.calls = append(s.calls, "raw")
|
||||
if len(s.rawErrors) > 0 {
|
||||
err := s.rawErrors[0]
|
||||
s.rawErrors = s.rawErrors[1:]
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scriptedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
s.unifiedCalls++
|
||||
s.calls = append(s.calls, "unified")
|
||||
if len(s.unifiedErrors) > 0 {
|
||||
err := s.unifiedErrors[0]
|
||||
s.unifiedErrors = s.unifiedErrors[1:]
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scriptedSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
94
go/vehicle-gateway/internal/eventbus/kafka_sink.go
Normal file
94
go/vehicle-gateway/internal/eventbus/kafka_sink.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Sink interface {
|
||||
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type KafkaSink struct {
|
||||
writer kafkaWriter
|
||||
rawTopics map[envelope.Protocol]string
|
||||
unifiedTopic string
|
||||
}
|
||||
|
||||
type kafkaWriter interface {
|
||||
WriteMessages(context.Context, ...kafka.Message) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type KafkaConfig struct {
|
||||
Brokers []string
|
||||
RawTopics map[envelope.Protocol]string
|
||||
UnifiedTopic string
|
||||
}
|
||||
|
||||
func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) {
|
||||
if len(cfg.Brokers) == 0 {
|
||||
return nil, errors.New("kafka brokers are required")
|
||||
}
|
||||
return newKafkaSinkWithWriter(&kafka.Writer{
|
||||
Addr: kafka.TCP(cfg.Brokers...),
|
||||
Balancer: &kafka.Hash{},
|
||||
AllowAutoTopicCreation: false,
|
||||
}, cfg), nil
|
||||
}
|
||||
|
||||
func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
|
||||
rawTopics := map[envelope.Protocol]string{
|
||||
envelope.ProtocolGB32960: "vehicle.raw.gb32960.v1",
|
||||
envelope.ProtocolJT808: "vehicle.raw.jt808.v1",
|
||||
envelope.ProtocolYutongMQTT: "vehicle.raw.yutong-mqtt.v1",
|
||||
}
|
||||
for protocol, topic := range cfg.RawTopics {
|
||||
if topic != "" {
|
||||
rawTopics[protocol] = topic
|
||||
}
|
||||
}
|
||||
unifiedTopic := cfg.UnifiedTopic
|
||||
if unifiedTopic == "" {
|
||||
unifiedTopic = "vehicle.event.unified.v1"
|
||||
}
|
||||
return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic}
|
||||
}
|
||||
|
||||
func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
topic, ok := s.rawTopics[env.Protocol]
|
||||
if !ok || topic == "" {
|
||||
return fmt.Errorf("raw topic not configured for protocol %s", env.Protocol)
|
||||
}
|
||||
return s.publish(ctx, topic, env)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.publish(ctx, s.unifiedTopic, env)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) Close() error {
|
||||
if s == nil || s.writer == nil {
|
||||
return nil
|
||||
}
|
||||
return s.writer.Close()
|
||||
}
|
||||
|
||||
func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.writer.WriteMessages(ctx, kafka.Message{
|
||||
Topic: topic,
|
||||
Key: env.KafkaKey(),
|
||||
Value: payload,
|
||||
})
|
||||
}
|
||||
77
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
Normal file
77
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestKafkaSinkRoutesRawTopics(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "013307795425", MessageID: "0x0200"}
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if len(writer.messages) != 1 {
|
||||
t.Fatalf("messages = %d", len(writer.messages))
|
||||
}
|
||||
msg := writer.messages[0]
|
||||
if msg.Topic != "vehicle.raw.jt808.v1" {
|
||||
t.Fatalf("topic = %q", msg.Topic)
|
||||
}
|
||||
if string(msg.Key) != "JT808:013307795425" {
|
||||
t.Fatalf("key = %q", string(msg.Key))
|
||||
}
|
||||
var decoded envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(msg.Value, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if decoded.EventID == "" || decoded.ParseStatus != envelope.ParseOK {
|
||||
t.Fatalf("unexpected payload defaults: %#v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKafkaSinkRoutesUnifiedTopic(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{UnifiedTopic: "custom.unified"})
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000001", MessageID: "0x02"}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
if writer.messages[0].Topic != "custom.unified" {
|
||||
t.Fatalf("topic = %q", writer.messages[0].Topic)
|
||||
}
|
||||
if string(writer.messages[0].Key) != "LNBVIN00000000001" {
|
||||
t.Fatalf("key = %q", string(writer.messages[0].Key))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKafkaSinkRejectsUnknownProtocol(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")})
|
||||
if err == nil {
|
||||
t.Fatal("expected unknown protocol error")
|
||||
}
|
||||
}
|
||||
|
||||
type recordingWriter struct {
|
||||
messages []kafka.Message
|
||||
}
|
||||
|
||||
func (w *recordingWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
|
||||
w.messages = append(w.messages, messages...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *recordingWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
33
go/vehicle-gateway/internal/eventbus/log_sink.go
Normal file
33
go/vehicle-gateway/internal/eventbus/log_sink.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type LogSink struct {
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewLogSink(logger *slog.Logger) *LogSink {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &LogSink{logger: logger}
|
||||
}
|
||||
|
||||
func (s *LogSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.logger.Info("raw envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID, "status", env.ParseStatus)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LogSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.logger.Info("unified envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LogSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
80
go/vehicle-gateway/internal/eventbus/retry_sink.go
Normal file
80
go/vehicle-gateway/internal/eventbus/retry_sink.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type RetryConfig struct {
|
||||
Attempts int
|
||||
Backoff time.Duration
|
||||
AttemptTimeout time.Duration
|
||||
}
|
||||
|
||||
type RetryingSink struct {
|
||||
delegate Sink
|
||||
cfg RetryConfig
|
||||
}
|
||||
|
||||
func NewRetryingSink(delegate Sink, cfg RetryConfig) *RetryingSink {
|
||||
if delegate == nil {
|
||||
panic("retry delegate sink must not be nil")
|
||||
}
|
||||
if cfg.Attempts <= 0 {
|
||||
cfg.Attempts = 1
|
||||
}
|
||||
return &RetryingSink{delegate: delegate, cfg: cfg}
|
||||
}
|
||||
|
||||
func (s *RetryingSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.withRetry(ctx, func(attemptCtx context.Context) error {
|
||||
return s.delegate.PublishRaw(attemptCtx, env)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *RetryingSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.withRetry(ctx, func(attemptCtx context.Context) error {
|
||||
return s.delegate.PublishUnified(attemptCtx, env)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *RetryingSink) Close() error {
|
||||
return s.delegate.Close()
|
||||
}
|
||||
|
||||
func (s *RetryingSink) withRetry(ctx context.Context, publish func(context.Context) error) error {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= s.cfg.Attempts; attempt++ {
|
||||
attemptCtx := ctx
|
||||
cancel := func() {}
|
||||
if s.cfg.AttemptTimeout > 0 {
|
||||
attemptCtx, cancel = context.WithTimeout(ctx, s.cfg.AttemptTimeout)
|
||||
}
|
||||
err := publish(attemptCtx)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
lastErr = err
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if attempt == s.cfg.Attempts {
|
||||
break
|
||||
}
|
||||
if s.cfg.Backoff > 0 {
|
||||
timer := time.NewTimer(s.cfg.Backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
80
go/vehicle-gateway/internal/eventbus/retry_sink_test.go
Normal file
80
go/vehicle-gateway/internal/eventbus/retry_sink_test.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestRetryingSinkRetriesRawUntilSuccess(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 1}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 3})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if delegate.rawCalls != 2 {
|
||||
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryingSinkReturnsLastErrorAfterAttempts(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 3}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 2})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if !errors.Is(err, errFlakyPublish) {
|
||||
t.Fatalf("PublishRaw() error = %v, want %v", err, errFlakyPublish)
|
||||
}
|
||||
if delegate.rawCalls != 2 {
|
||||
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryingSinkStopsWhenContextCanceledDuringBackoff(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 10}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 5, Backoff: time.Minute})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
err := sink.PublishRaw(ctx, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("PublishRaw() error = %v, want context canceled", err)
|
||||
}
|
||||
if delegate.rawCalls != 1 {
|
||||
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
var errFlakyPublish = errors.New("publish failed")
|
||||
|
||||
type flakySink struct {
|
||||
rawFailures int
|
||||
unifiedFailures int
|
||||
rawCalls int
|
||||
unifiedCalls int
|
||||
}
|
||||
|
||||
func (s *flakySink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
s.rawCalls++
|
||||
if s.rawCalls <= s.rawFailures {
|
||||
return errFlakyPublish
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *flakySink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
s.unifiedCalls++
|
||||
if s.unifiedCalls <= s.unifiedFailures {
|
||||
return errFlakyPublish
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *flakySink) Close() error {
|
||||
return nil
|
||||
}
|
||||
218
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
218
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
@@ -0,0 +1,218 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
|
||||
)
|
||||
|
||||
type MQTTClientConfig struct {
|
||||
EndpointName string
|
||||
Broker string
|
||||
ClientID string
|
||||
Username string
|
||||
Password string
|
||||
Topics []string
|
||||
QoS byte
|
||||
CleanSession bool
|
||||
KeepAlive time.Duration
|
||||
ConnectTimeout time.Duration
|
||||
TLSCACertPath string
|
||||
TLSClientCertPath string
|
||||
TLSClientKeyPath string
|
||||
TLSHostnameVerification bool
|
||||
Sink eventbus.Sink
|
||||
Resolver identity.Resolver
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
type MQTTClient struct {
|
||||
cfg MQTTClientConfig
|
||||
client mqtt.Client
|
||||
}
|
||||
|
||||
func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
|
||||
if strings.TrimSpace(cfg.Broker) == "" {
|
||||
return nil, errors.New("mqtt broker is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ClientID) == "" {
|
||||
return nil, errors.New("mqtt client id is required")
|
||||
}
|
||||
if len(cfg.Topics) == 0 {
|
||||
return nil, errors.New("mqtt topics are required")
|
||||
}
|
||||
if cfg.Sink == nil {
|
||||
return nil, errors.New("sink is required")
|
||||
}
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
if cfg.Resolver == nil {
|
||||
cfg.Resolver = identity.NoopResolver{}
|
||||
}
|
||||
if cfg.EndpointName == "" {
|
||||
cfg.EndpointName = "yutong"
|
||||
}
|
||||
return &MQTTClient{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) Start(ctx context.Context) error {
|
||||
opts, err := c.buildOptions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.client = mqtt.NewClient(opts)
|
||||
token := c.client.Connect()
|
||||
if token.Wait() && token.Error() != nil {
|
||||
return token.Error()
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
if c.client != nil && c.client.IsConnected() {
|
||||
c.client.Disconnect(250)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) buildOptions(ctx context.Context) (*mqtt.ClientOptions, error) {
|
||||
keepAlive := c.cfg.KeepAlive
|
||||
if keepAlive <= 0 {
|
||||
keepAlive = 20 * time.Second
|
||||
}
|
||||
connectTimeout := c.cfg.ConnectTimeout
|
||||
if connectTimeout <= 0 {
|
||||
connectTimeout = 10 * time.Second
|
||||
}
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker(c.cfg.Broker).
|
||||
SetClientID(c.cfg.ClientID).
|
||||
SetUsername(c.cfg.Username).
|
||||
SetPassword(c.cfg.Password).
|
||||
SetCleanSession(c.cfg.CleanSession).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectRetry(true).
|
||||
SetConnectRetryInterval(5 * time.Second).
|
||||
SetResumeSubs(true).
|
||||
SetOrderMatters(false).
|
||||
SetKeepAlive(keepAlive).
|
||||
SetConnectTimeout(connectTimeout)
|
||||
|
||||
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
|
||||
c.handleMessage(ctx, message.Topic(), message.Payload())
|
||||
})
|
||||
opts.OnConnect = func(client mqtt.Client) {
|
||||
for _, topic := range c.cfg.Topics {
|
||||
token := client.Subscribe(topic, c.cfg.QoS, nil)
|
||||
if token.Wait() && token.Error() != nil {
|
||||
c.cfg.Logger.Error("mqtt subscribe failed", "broker", c.cfg.Broker, "topic", topic, "error", token.Error())
|
||||
continue
|
||||
}
|
||||
c.cfg.Logger.Info("mqtt subscribed", "broker", c.cfg.Broker, "topic", topic, "qos", c.cfg.QoS)
|
||||
}
|
||||
}
|
||||
opts.OnConnectionLost = func(_ mqtt.Client, err error) {
|
||||
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
|
||||
}
|
||||
|
||||
tlsConfig, err := c.buildTLSConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tlsConfig != nil {
|
||||
opts.SetTLSConfig(tlsConfig)
|
||||
}
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
|
||||
caPath := strings.TrimSpace(c.cfg.TLSCACertPath)
|
||||
certPath := strings.TrimSpace(c.cfg.TLSClientCertPath)
|
||||
keyPath := strings.TrimSpace(c.cfg.TLSClientKeyPath)
|
||||
if caPath == "" && certPath == "" && keyPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
config := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: !c.cfg.TLSHostnameVerification,
|
||||
}
|
||||
if caPath != "" {
|
||||
caPEM, err := os.ReadFile(caPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read mqtt ca certificate: %w", err)
|
||||
}
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("parse mqtt ca certificate %s", caPath)
|
||||
}
|
||||
config.RootCAs = roots
|
||||
}
|
||||
if certPath != "" || keyPath != "" {
|
||||
if certPath == "" || keyPath == "" {
|
||||
return nil, errors.New("mqtt client certificate and key must be configured together")
|
||||
}
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load mqtt client certificate: %w", err)
|
||||
}
|
||||
config.Certificates = []tls.Certificate{cert}
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := yutongmqtt.ParseMessage(c.cfg.EndpointName, topic, payload, receivedAtMS)
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
MessageID: "MQTT",
|
||||
VehicleKeyHint: "unknown",
|
||||
SourceEndpoint: "mqtt://" + c.cfg.EndpointName + topic,
|
||||
EventTimeMS: receivedAtMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawText: string(payload),
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(payload)),
|
||||
ParseStatus: envelope.ParseBadFrame,
|
||||
ParseError: err.Error(),
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
} else {
|
||||
resolved, resolveErr := c.cfg.Resolver.Resolve(ctx, env)
|
||||
if resolveErr != nil {
|
||||
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
|
||||
env.ParseStatus = envelope.ParsePartial
|
||||
} else {
|
||||
env = resolved
|
||||
}
|
||||
}
|
||||
if err := c.cfg.Sink.PublishRaw(ctx, env); err != nil {
|
||||
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return
|
||||
}
|
||||
if err := c.cfg.Sink.PublishUnified(ctx, env); err != nil {
|
||||
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||
}
|
||||
}
|
||||
183
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
183
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "tcp://127.0.0.1:1883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
|
||||
"device":"LTEST000000000001",
|
||||
"time":"20260413100000",
|
||||
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
|
||||
}`))
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
|
||||
t.Fatalf("unexpected raw envelope: %#v", sink.raw[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "tcp://127.0.0.1:1883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
client.handleMessage(context.Background(), "/ytforward/shln/bad", []byte("{bad-json"))
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "ssl://mqtt.example.test:8883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
QoS: 1,
|
||||
Sink: &recordingSink{},
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
TLSCACertPath: caPath,
|
||||
TLSClientCertPath: certPath,
|
||||
TLSClientKeyPath: keyPath,
|
||||
TLSHostnameVerification: false,
|
||||
CleanSession: true,
|
||||
KeepAlive: 20 * time.Second,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
opts, err := client.buildOptions(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("buildOptions() error = %v", err)
|
||||
}
|
||||
if opts.TLSConfig == nil {
|
||||
t.Fatal("TLSConfig is nil")
|
||||
}
|
||||
if opts.TLSConfig.RootCAs == nil {
|
||||
t.Fatal("RootCAs is nil")
|
||||
}
|
||||
if len(opts.TLSConfig.Certificates) != 1 {
|
||||
t.Fatalf("client certificates = %d, want 1", len(opts.TLSConfig.Certificates))
|
||||
}
|
||||
if !opts.TLSConfig.InsecureSkipVerify {
|
||||
t.Fatal("InsecureSkipVerify should be true when hostname verification is disabled")
|
||||
}
|
||||
if !opts.CleanSession {
|
||||
t.Fatal("CleanSession should be true")
|
||||
}
|
||||
if opts.Order {
|
||||
t.Fatal("OrderMatters should be false so MQTT network handling is not blocked by Kafka/DB work")
|
||||
}
|
||||
if !opts.ResumeSubs {
|
||||
t.Fatal("ResumeSubs should be true to avoid subscribe failures during reconnect churn")
|
||||
}
|
||||
if got := opts.KeepAlive; got != 20 {
|
||||
t.Fatalf("KeepAlive = %d, want 20", got)
|
||||
}
|
||||
if got := opts.ConnectTimeout; got != 10*time.Second {
|
||||
t.Fatalf("ConnectTimeout = %v, want 10s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestTLSMaterial(t *testing.T, dir string) (string, string, string) {
|
||||
t.Helper()
|
||||
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate ca key: %v", err)
|
||||
}
|
||||
caTemplate := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "test-ca"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
}
|
||||
caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
|
||||
if err != nil {
|
||||
t.Fatalf("create ca cert: %v", err)
|
||||
}
|
||||
clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate client key: %v", err)
|
||||
}
|
||||
clientTemplate := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(2),
|
||||
Subject: pkix.Name{CommonName: "test-client"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
||||
}
|
||||
clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey)
|
||||
if err != nil {
|
||||
t.Fatalf("create client cert: %v", err)
|
||||
}
|
||||
caPath := filepath.Join(dir, "ca.pem")
|
||||
certPath := filepath.Join(dir, "client.pem")
|
||||
keyPath := filepath.Join(dir, "client-key.pem")
|
||||
writePEM(t, caPath, "CERTIFICATE", caDER)
|
||||
writePEM(t, certPath, "CERTIFICATE", clientDER)
|
||||
keyDER := x509.MarshalPKCS1PrivateKey(clientKey)
|
||||
writePEM(t, keyPath, "RSA PRIVATE KEY", keyDER)
|
||||
return caPath, certPath, keyPath
|
||||
}
|
||||
|
||||
func writePEM(t *testing.T, path, typ string, der []byte) {
|
||||
t.Helper()
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create %s: %v", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err := pem.Encode(file, &pem.Block{Type: typ, Bytes: der}); err != nil {
|
||||
t.Fatalf("write pem %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
238
go/vehicle-gateway/internal/gateway/tcp_server.go
Normal file
238
go/vehicle-gateway/internal/gateway/tcp_server.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
)
|
||||
|
||||
type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
|
||||
|
||||
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
|
||||
|
||||
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
|
||||
|
||||
type TCPProtocol struct {
|
||||
Protocol envelope.Protocol
|
||||
Addr string
|
||||
Extract FrameExtractor
|
||||
Parse FrameParser
|
||||
Respond FrameResponder
|
||||
}
|
||||
|
||||
type TCPServer struct {
|
||||
protocol TCPProtocol
|
||||
sink eventbus.Sink
|
||||
resolver identity.Resolver
|
||||
logger *slog.Logger
|
||||
readBufferSize int
|
||||
idleTimeout time.Duration
|
||||
maxConnections int
|
||||
}
|
||||
|
||||
type TCPServerConfig struct {
|
||||
Protocol TCPProtocol
|
||||
Sink eventbus.Sink
|
||||
Resolver identity.Resolver
|
||||
Logger *slog.Logger
|
||||
ReadBufferSize int
|
||||
IdleTimeout time.Duration
|
||||
MaxConnections int
|
||||
}
|
||||
|
||||
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
|
||||
if cfg.Protocol.Protocol == "" {
|
||||
return nil, errors.New("protocol is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Protocol.Addr) == "" {
|
||||
return nil, errors.New("listen addr is required")
|
||||
}
|
||||
if cfg.Protocol.Extract == nil {
|
||||
return nil, errors.New("frame extractor is required")
|
||||
}
|
||||
if cfg.Protocol.Parse == nil {
|
||||
return nil, errors.New("frame parser is required")
|
||||
}
|
||||
if cfg.Sink == nil {
|
||||
return nil, errors.New("sink is required")
|
||||
}
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
if cfg.Resolver == nil {
|
||||
cfg.Resolver = identity.NoopResolver{}
|
||||
}
|
||||
if cfg.ReadBufferSize <= 0 {
|
||||
cfg.ReadBufferSize = 32 * 1024
|
||||
}
|
||||
if cfg.IdleTimeout <= 0 {
|
||||
cfg.IdleTimeout = 2 * time.Minute
|
||||
}
|
||||
if cfg.MaxConnections <= 0 {
|
||||
cfg.MaxConnections = 10_000
|
||||
}
|
||||
return &TCPServer{
|
||||
protocol: cfg.Protocol,
|
||||
sink: cfg.Sink,
|
||||
resolver: cfg.Resolver,
|
||||
logger: cfg.Logger,
|
||||
readBufferSize: cfg.ReadBufferSize,
|
||||
idleTimeout: cfg.IdleTimeout,
|
||||
maxConnections: cfg.MaxConnections,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *TCPServer) ListenAndServe(ctx context.Context) error {
|
||||
var lc net.ListenConfig
|
||||
listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
_ = listener.Close()
|
||||
}()
|
||||
|
||||
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
|
||||
sem := make(chan struct{}, s.maxConnections)
|
||||
var wg sync.WaitGroup
|
||||
defer wg.Wait()
|
||||
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
s.handleConnection(ctx, conn)
|
||||
}()
|
||||
default:
|
||||
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
||||
defer conn.Close()
|
||||
source := conn.RemoteAddr().String()
|
||||
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
|
||||
log.Info("tcp connection opened")
|
||||
defer log.Info("tcp connection closed")
|
||||
|
||||
readBuffer := make([]byte, s.readBufferSize)
|
||||
var pending []byte
|
||||
for {
|
||||
_ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout))
|
||||
n, err := conn.Read(readBuffer)
|
||||
if n > 0 {
|
||||
pending = append(pending, readBuffer[:n]...)
|
||||
frames, remainder, extractErr := s.protocol.Extract(pending)
|
||||
if extractErr != nil {
|
||||
log.Warn("frame extraction failed", "error", extractErr)
|
||||
return
|
||||
}
|
||||
pending = remainder
|
||||
for _, frame := range frames {
|
||||
s.handleFrame(ctx, conn, frame, source)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
log.Warn("tcp connection idle timeout")
|
||||
return
|
||||
}
|
||||
log.Warn("tcp read failed", "error", err)
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string) {
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := s.protocol.Parse(raw, receivedAtMS, source)
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: s.protocol.Protocol,
|
||||
SourceEndpoint: source,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
EventTimeMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
ParseStatus: envelope.ParseBadFrame,
|
||||
ParseError: err.Error(),
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
} else {
|
||||
resolved, resolveErr := s.resolver.Resolve(ctx, env)
|
||||
if resolveErr != nil {
|
||||
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
|
||||
env.ParseStatus = envelope.ParsePartial
|
||||
} else {
|
||||
env = resolved
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.sink.PublishRaw(ctx, env); err != nil {
|
||||
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return
|
||||
}
|
||||
if err := s.sink.PublishUnified(ctx, env); err != nil {
|
||||
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if s.protocol.Respond == nil {
|
||||
return
|
||||
}
|
||||
response, ok, err := s.protocol.Respond(raw, env)
|
||||
if err != nil {
|
||||
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if !ok || len(response) == 0 {
|
||||
return
|
||||
}
|
||||
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := conn.Write(response); err != nil {
|
||||
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (p TCPProtocol) String() string {
|
||||
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
|
||||
}
|
||||
177
go/vehicle-gateway/internal/gateway/tcp_server_test.go
Normal file
177
go/vehicle-gateway/internal/gateway/tcp_server_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||
)
|
||||
|
||||
func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
|
||||
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Addr: ":0",
|
||||
Extract: jt808.ExtractFrames,
|
||||
Parse: jt808.ParseFrame,
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(frame); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].Phone != "013307795425" {
|
||||
t.Fatalf("phone = %q", sink.raw[0].Phone)
|
||||
}
|
||||
if sink.raw[0].Fields[envelope.FieldTotalMileageKM] != 10241.2 {
|
||||
t.Fatalf("total mileage = %#v", sink.raw[0].Fields[envelope.FieldTotalMileageKM])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
|
||||
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
good[len(good)-1] ^= 0xff
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(good); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||
}
|
||||
if sink.raw[0].ParseError == "" {
|
||||
t.Fatal("parse error should be recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
|
||||
frame := buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
|
||||
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
return []byte("ACK"), true, nil
|
||||
},
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(frame); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
buf := make([]byte, 3)
|
||||
if _, err := io.ReadFull(client, buf); err != nil {
|
||||
t.Fatalf("read response error = %v", err)
|
||||
}
|
||||
if string(buf) != "ACK" {
|
||||
t.Fatalf("response = %q", string(buf))
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer {
|
||||
t.Helper()
|
||||
server, err := NewTCPServer(TCPServerConfig{
|
||||
Protocol: protocol,
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
ReadBufferSize: 1024,
|
||||
IdleTimeout: time.Second,
|
||||
MaxConnections: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewTCPServer() error = %v", err)
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
func runPipe(t *testing.T, server *TCPServer) (net.Conn, <-chan struct{}) {
|
||||
t.Helper()
|
||||
client, srv := net.Pipe()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
server.handleConnection(context.Background(), srv)
|
||||
}()
|
||||
return client, done
|
||||
}
|
||||
|
||||
type recordingSink struct {
|
||||
raw []envelope.FrameEnvelope
|
||||
unified []envelope.FrameEnvelope
|
||||
}
|
||||
|
||||
func (s *recordingSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.raw = append(s.raw, env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *recordingSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.unified = append(s.unified, env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *recordingSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type testWriter struct {
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (w testWriter) Write(p []byte) (int, error) {
|
||||
w.t.Log(string(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func buildGBFrame(command byte, response byte, vin string, body []byte) []byte {
|
||||
frame := []byte{'#', '#', command, response}
|
||||
vinBytes := []byte(vin)
|
||||
if len(vinBytes) < 17 {
|
||||
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
|
||||
}
|
||||
frame = append(frame, vinBytes[:17]...)
|
||||
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
|
||||
frame = append(frame, body...)
|
||||
var bcc byte
|
||||
for _, value := range frame[2:] {
|
||||
bcc ^= value
|
||||
}
|
||||
return append(frame, bcc)
|
||||
}
|
||||
908
go/vehicle-gateway/internal/history/query.go
Normal file
908
go/vehicle-gateway/internal/history/query.go
Normal file
@@ -0,0 +1,908 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Queryer interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
type RawFrameQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
MessageID string
|
||||
OrderBy string
|
||||
IncludeTotal bool
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type RawFrameRow struct {
|
||||
TS string `json:"ts"`
|
||||
FrameID string `json:"frame_id"`
|
||||
EventID string `json:"event_id"`
|
||||
MessageID int64 `json:"message_id"`
|
||||
MessageIDHex string `json:"message_id_hex"`
|
||||
EventTime string `json:"event_time"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
RawSizeBytes int64 `json:"raw_size_bytes"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
ParsedJSON string `json:"parsed_json,omitempty"`
|
||||
FieldsJSON string `json:"fields_json,omitempty"`
|
||||
ParseStatus string `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type LocationQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type LocationRow struct {
|
||||
TS string `json:"ts"`
|
||||
EventID string `json:"event_id"`
|
||||
FrameID string `json:"frame_id"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
Longitude float64 `json:"longitude"`
|
||||
Latitude float64 `json:"latitude"`
|
||||
AltitudeM *float64 `json:"altitude_m,omitempty"`
|
||||
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
|
||||
DirectionDeg *int64 `json:"direction_deg,omitempty"`
|
||||
AlarmFlag *int64 `json:"alarm_flag,omitempty"`
|
||||
StatusFlag *int64 `json:"status_flag,omitempty"`
|
||||
TotalMileageKM *float64 `json:"total_mileage_km,omitempty"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type MileagePointQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type MileagePointRow struct {
|
||||
TS string `json:"ts"`
|
||||
EventID string `json:"event_id"`
|
||||
FrameID string `json:"frame_id"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
TotalMileageKM float64 `json:"total_mileage_km"`
|
||||
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type RawFrameRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
func NewRawFrameRepository(db Queryer, database string) *RawFrameRepository {
|
||||
if db == nil {
|
||||
panic("raw frame query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &RawFrameRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
type LocationRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
type MileagePointRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
func NewLocationRepository(db Queryer, database string) *LocationRepository {
|
||||
if db == nil {
|
||||
panic("location query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &LocationRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
func NewMileagePointRepository(db Queryer, database string) *MileagePointRepository {
|
||||
if db == nil {
|
||||
panic("mileage point query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &MileagePointRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) {
|
||||
query = normalizeRawFrameQuery(query)
|
||||
sqlText, args := buildRawFrameSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]RawFrameRow, 0)
|
||||
for rows.Next() {
|
||||
var row RawFrameRow
|
||||
var ts scanDateTime
|
||||
var eventTime scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.FrameID,
|
||||
&row.EventID,
|
||||
&row.MessageID,
|
||||
&eventTime,
|
||||
&receivedAt,
|
||||
&row.RawSizeBytes,
|
||||
&row.RawHex,
|
||||
&row.RawText,
|
||||
&row.ParsedJSON,
|
||||
&row.FieldsJSON,
|
||||
&row.ParseStatus,
|
||||
&row.ParseError,
|
||||
&row.SourceEndpoint,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.EventTime = eventTime.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.MessageIDHex = fmt.Sprintf("0x%04X", row.MessageID)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
|
||||
query = normalizeRawFrameQuery(query)
|
||||
sqlText, args := buildRawFrameCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]LocationRow, error) {
|
||||
query = normalizeLocationQuery(query)
|
||||
sqlText, args := buildLocationSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]LocationRow, 0)
|
||||
for rows.Next() {
|
||||
var row LocationRow
|
||||
var ts scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
var altitude sql.NullFloat64
|
||||
var speed sql.NullFloat64
|
||||
var direction sql.NullInt64
|
||||
var alarm sql.NullInt64
|
||||
var status sql.NullInt64
|
||||
var mileage sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.EventID,
|
||||
&row.FrameID,
|
||||
&receivedAt,
|
||||
&row.Longitude,
|
||||
&row.Latitude,
|
||||
&altitude,
|
||||
&speed,
|
||||
&direction,
|
||||
&alarm,
|
||||
&status,
|
||||
&mileage,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.AltitudeM = nullableFloat(altitude)
|
||||
row.SpeedKMH = nullableFloat(speed)
|
||||
row.DirectionDeg = nullableInt(direction)
|
||||
row.AlarmFlag = nullableInt(alarm)
|
||||
row.StatusFlag = nullableInt(status)
|
||||
row.TotalMileageKM = nullableFloat(mileage)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *LocationRepository) Count(ctx context.Context, query LocationQuery) (int64, error) {
|
||||
query = normalizeLocationQuery(query)
|
||||
sqlText, args := buildLocationCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) Query(ctx context.Context, query MileagePointQuery) ([]MileagePointRow, error) {
|
||||
query = normalizeMileagePointQuery(query)
|
||||
sqlText, args := buildMileagePointSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]MileagePointRow, 0)
|
||||
for rows.Next() {
|
||||
var row MileagePointRow
|
||||
var ts scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
var speed sql.NullFloat64
|
||||
var longitude sql.NullFloat64
|
||||
var latitude sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.EventID,
|
||||
&row.FrameID,
|
||||
&receivedAt,
|
||||
&row.TotalMileageKM,
|
||||
&speed,
|
||||
&longitude,
|
||||
&latitude,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.SpeedKMH = nullableFloat(speed)
|
||||
row.Longitude = nullableFloat(longitude)
|
||||
row.Latitude = nullableFloat(latitude)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) Count(ctx context.Context, query MileagePointQuery) (int64, error) {
|
||||
query = normalizeMileagePointQuery(query)
|
||||
sqlText, args := buildMileagePointCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func countRows(ctx context.Context, db Queryer, sqlText string, args ...any) (int64, error) {
|
||||
rows, err := db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var total int64
|
||||
if rows.Next() {
|
||||
if err := rows.Scan(&total); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return total, rows.Err()
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "raw_frames"
|
||||
}
|
||||
return r.database + ".raw_frames"
|
||||
}
|
||||
|
||||
func (r *LocationRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "vehicle_locations"
|
||||
}
|
||||
return r.database + ".vehicle_locations"
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "vehicle_mileage_points"
|
||||
}
|
||||
return r.database + ".vehicle_mileage_points"
|
||||
}
|
||||
|
||||
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.MessageID = strings.TrimSpace(query.MessageID)
|
||||
query.OrderBy = normalizeRawFrameOrderBy(query.OrderBy)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func normalizeLocationQuery(query LocationQuery) LocationQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func normalizeMileagePointQuery(query MileagePointQuery) MileagePointQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
|
||||
where := rawFrameWhere(query)
|
||||
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY " + rawFrameOrderColumn(query.OrderBy) + " DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := rawFrameWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
|
||||
where := locationWhere(query)
|
||||
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildLocationCountSQL(table string, query LocationQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := locationWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildMileagePointSQL(table string, query MileagePointQuery) (string, []any) {
|
||||
where := mileagePointWhere(query)
|
||||
sqlText := `SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildMileagePointCountSQL(table string, query MileagePointQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := mileagePointWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func rawFrameWhere(query RawFrameQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.MessageID != "" {
|
||||
if parsed, ok := parseMessageID(query.MessageID); ok {
|
||||
add("message_id = " + strconv.FormatInt(parsed, 10))
|
||||
}
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
func normalizeRawFrameOrderBy(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
switch value {
|
||||
case "receivedat", "received_at":
|
||||
return "receivedAt"
|
||||
default:
|
||||
return "ts"
|
||||
}
|
||||
}
|
||||
|
||||
func rawFrameOrderColumn(value string) string {
|
||||
if normalizeRawFrameOrderBy(value) == "receivedAt" {
|
||||
return "received_at"
|
||||
}
|
||||
return "ts"
|
||||
}
|
||||
|
||||
func locationWhere(query LocationQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
func mileagePointWhere(query MileagePointQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
type RawFrameHandler struct {
|
||||
repository *RawFrameRepository
|
||||
}
|
||||
|
||||
type LocationHandler struct {
|
||||
repository *LocationRepository
|
||||
}
|
||||
|
||||
type MileagePointHandler struct {
|
||||
repository *MileagePointRepository
|
||||
}
|
||||
|
||||
func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler {
|
||||
if repository == nil {
|
||||
panic("raw frame repository must not be nil")
|
||||
}
|
||||
return &RawFrameHandler{repository: repository}
|
||||
}
|
||||
|
||||
func NewLocationHandler(repository *LocationRepository) *LocationHandler {
|
||||
if repository == nil {
|
||||
panic("location repository must not be nil")
|
||||
}
|
||||
return &LocationHandler{repository: repository}
|
||||
}
|
||||
|
||||
func NewMileagePointHandler(repository *MileagePointRepository) *MileagePointHandler {
|
||||
if repository == nil {
|
||||
panic("mileage point repository must not be nil")
|
||||
}
|
||||
return &MileagePointHandler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/raw-frames" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseRawFrameQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
var total int64
|
||||
if query.IncludeTotal {
|
||||
total, err = h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !query.IncludeTotal {
|
||||
total = int64(len(rows))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/locations" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseLocationQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *MileagePointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/mileage-points" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseMileagePointQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return RawFrameQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return RawFrameQuery{}, err
|
||||
}
|
||||
query := RawFrameQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
MessageID: values.Get("messageId"),
|
||||
OrderBy: values.Get("orderBy"),
|
||||
IncludeTotal: values.Get("includeTotal") != "false",
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
if strings.TrimSpace(query.MessageID) != "" {
|
||||
if _, ok := parseMessageID(query.MessageID); !ok {
|
||||
return RawFrameQuery{}, errors.New("messageId must be decimal or hex like 0x0200")
|
||||
}
|
||||
}
|
||||
return normalizeRawFrameQuery(query), nil
|
||||
}
|
||||
|
||||
func parseMileagePointQuery(r *http.Request) (MileagePointQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return MileagePointQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return MileagePointQuery{}, err
|
||||
}
|
||||
query := MileagePointQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return MileagePointQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
return normalizeMileagePointQuery(query), nil
|
||||
}
|
||||
|
||||
func parseLocationQuery(r *http.Request) (LocationQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return LocationQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return LocationQuery{}, err
|
||||
}
|
||||
query := LocationQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
return normalizeLocationQuery(query), nil
|
||||
}
|
||||
|
||||
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil || value < min || value > max {
|
||||
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseMessageID(value string) (int64, bool) {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
if value == "" {
|
||||
return 0, false
|
||||
}
|
||||
if strings.HasPrefix(value, "0x") {
|
||||
parsed, err := strconv.ParseInt(strings.TrimPrefix(value, "0x"), 16, 32)
|
||||
return parsed, err == nil
|
||||
}
|
||||
parsed, err := strconv.ParseInt(value, 10, 32)
|
||||
return parsed, err == nil
|
||||
}
|
||||
|
||||
func validDateTime(value string) bool {
|
||||
value = normalizeDateTimeLiteral(value)
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
for _, layout := range []string{"2006-01-02", "2006-01-02 15:04:05", time.RFC3339} {
|
||||
if _, err := time.Parse(layout, value); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeDateTimeLiteral(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} {
|
||||
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
|
||||
return parsed.UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
}
|
||||
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
|
||||
return parsed.UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func safeIdentifier(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type scanDateTime struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDateTime) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatSQLTime(value any, layout string) string {
|
||||
switch typed := value.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case time.Time:
|
||||
return typed.Format(layout)
|
||||
case []byte:
|
||||
return strings.TrimSpace(string(typed))
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
|
||||
func nullableFloat(value sql.NullFloat64) *float64 {
|
||||
if !value.Valid {
|
||||
return nil
|
||||
}
|
||||
return &value.Float64
|
||||
}
|
||||
|
||||
func nullableInt(value sql.NullInt64) *int64 {
|
||||
if !value.Valid {
|
||||
return nil
|
||||
}
|
||||
return &value.Int64
|
||||
}
|
||||
|
||||
func writeHistoryError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
423
go/vehicle-gateway/internal/history/query_test.go
Normal file
423
go/vehicle-gateway/internal/history/query_test.go
Normal file
@@ -0,0 +1,423 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
"go_frame", "event-1", 0x0200,
|
||||
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
time.Date(2026, 7, 1, 23, 25, 37, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
64, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
|
||||
"OK", "", "222.66.200.68:29646", "JT808", "013079963379", "LKLG7C4E3NA774736", "013079963379", "9963379",
|
||||
))
|
||||
|
||||
repository := NewRawFrameRepository(db, "lingniu_vehicle_ts")
|
||||
rows, err := repository.Query(context.Background(), RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
DateFrom: "2026-07-01 00:00:00",
|
||||
DateTo: "2026-07-01 23:59:59",
|
||||
Limit: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query() error = %v", err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("row count = %d", len(rows))
|
||||
}
|
||||
if rows[0].MessageID != 512 || rows[0].MessageIDHex != "0x0200" || rows[0].ParsedJSON == "" {
|
||||
t.Fatalf("unexpected row: %#v", rows[0])
|
||||
}
|
||||
if rows[0].TS != "2026-07-01 23:25:36" {
|
||||
t.Fatalf("timestamp = %q", rows[0].TS)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(38))
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-01 22:28:25", "go_frame", "event-2", 2, "2026-07-01 22:28:25", "2026-07-01 22:28:25",
|
||||
128, "2323", "", `{"command":"REALTIME"}`, `{"total_mileage_km":53490.9}`,
|
||||
"OK", "", "8.134.95.166:53702", "GB32960", "LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"message_id_hex":"0x0002"`, `"total":38`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(11))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "go_frame", "event-3", 0x0200, "2026-07-02 00:18:22", "2026-07-02 00:22:43",
|
||||
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"total_mileage_km":8792.8}`,
|
||||
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"phone":"013307811350"`, `"total":11`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
|
||||
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
|
||||
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=NOT_EXISTS&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
|
||||
t.Fatalf("expected empty items array, got: %s", body)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocationHandlerReturnsLocationsByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(17))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
|
||||
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
|
||||
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
|
||||
"JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
|
||||
8792.8, 8.0, 121.07764, 30.585928,
|
||||
"JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewMileagePointHandler(NewMileagePointRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/mileage-points?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"total_mileage_km":8792.8`, `"speed_kmh":8`, `"total":19`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) {
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?dateFrom=2026-07-01T00:00:00&dateTo=2026-07-02T00:00:00", nil)
|
||||
|
||||
query, err := parseRawFrameQuery(request)
|
||||
if err != nil {
|
||||
t.Fatalf("parseRawFrameQuery() error = %v", err)
|
||||
}
|
||||
if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" {
|
||||
t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) {
|
||||
for raw, want := range map[string]string{
|
||||
"2026-07-01T00:00:00": "2026-06-30 16:00:00",
|
||||
"2026-07-01 00:00:00": "2026-06-30 16:00:00",
|
||||
"2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00",
|
||||
"2026-06-30T16:00:00Z": "2026-06-30 16:00:00",
|
||||
} {
|
||||
if got := normalizeDateTimeLiteral(raw); got != want {
|
||||
t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
|
||||
for raw, want := range map[string]int64{
|
||||
"512": 512,
|
||||
"0x0200": 512,
|
||||
"0X0100": 256,
|
||||
} {
|
||||
got, ok := parseMessageID(raw)
|
||||
if !ok || got != want {
|
||||
t.Fatalf("parseMessageID(%q) = %d,%v want %d,true", raw, got, ok, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
DateFrom: "2026-07-02 00:00:00",
|
||||
DateTo: "2026-07-02 23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"ts >= '2026-07-01 16:00:00'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildLocationSQL("lingniu_vehicle_ts.vehicle_locations", LocationQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
DateFrom: "2026-07-02 00:00:00",
|
||||
DateTo: "2026-07-02 23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"ts >= '2026-07-01 16:00:00'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
VIN: "VIN'1",
|
||||
MessageID: "0x0200",
|
||||
DateFrom: "2026-07-01T00:00:00",
|
||||
DateTo: "2026-07-01T23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"vin = 'VIN''1'",
|
||||
"message_id = 512",
|
||||
"ts >= '2026-06-30 16:00:00'",
|
||||
"ts <= '2026-07-01 15:59:59'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRawFrameSQLCanOrderByReceivedAt(t *testing.T) {
|
||||
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
OrderBy: "receivedAt",
|
||||
Limit: 1,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
if !strings.Contains(sqlText, "ORDER BY received_at DESC LIMIT 1 OFFSET 0") {
|
||||
t.Fatalf("sql should order by received_at: %s", sqlText)
|
||||
}
|
||||
}
|
||||
71
go/vehicle-gateway/internal/history/schema.go
Normal file
71
go/vehicle-gateway/internal/history/schema.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package history
|
||||
|
||||
const DefaultDatabase = "lingniu_vehicle_ts"
|
||||
|
||||
func SchemaStatements(database string) []string {
|
||||
if database == "" {
|
||||
database = DefaultDatabase
|
||||
}
|
||||
return []string{
|
||||
"CREATE DATABASE IF NOT EXISTS " + database + " KEEP 7300 DURATION 10 BUFFER 256",
|
||||
"USE " + database,
|
||||
`CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
event_id NCHAR(64),
|
||||
message_id INT,
|
||||
event_time TIMESTAMP,
|
||||
received_at TIMESTAMP,
|
||||
raw_size_bytes INT,
|
||||
raw_hex BINARY(16374),
|
||||
raw_text BINARY(16374),
|
||||
parsed_json BINARY(16374),
|
||||
fields_json BINARY(4096),
|
||||
parse_status NCHAR(16),
|
||||
parse_error BINARY(1024),
|
||||
source_endpoint NCHAR(128)
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
`CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg INT,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
total_mileage_km DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
}
|
||||
}
|
||||
339
go/vehicle-gateway/internal/history/writer.go
Normal file
339
go/vehicle-gateway/internal/history/writer.go
Normal file
@@ -0,0 +1,339 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Execer interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
exec Execer
|
||||
cache tableCache
|
||||
}
|
||||
|
||||
type tableCache struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]struct{}
|
||||
}
|
||||
|
||||
func NewWriter(exec Execer) *Writer {
|
||||
if exec == nil {
|
||||
panic("history execer must not be nil")
|
||||
}
|
||||
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
|
||||
}
|
||||
|
||||
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
||||
for _, statement := range SchemaStatements(database) {
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if err := w.AppendRawFrame(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.AppendLocation(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.AppendMileagePoint(ctx, env)
|
||||
}
|
||||
|
||||
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
table := tableName("raw", env)
|
||||
if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
||||
raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
|
||||
VALUES (%s)`, table, joinLiterals(rawValues(env))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
longitude, okLon := floatField(env, envelope.FieldLongitude)
|
||||
latitude, okLat := floatField(env, envelope.FieldLatitude)
|
||||
if !okLon || !okLat {
|
||||
return nil
|
||||
}
|
||||
table := tableName("loc", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_locations", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
||||
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
||||
VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
table := tableName("mil", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
|
||||
VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
|
||||
key := stable + "." + table
|
||||
w.cache.mu.Lock()
|
||||
_, ok := w.cache.seen[key]
|
||||
w.cache.mu.Unlock()
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
|
||||
table, stable,
|
||||
quote(string(env.Protocol)),
|
||||
quote(env.VehicleKey()),
|
||||
quote(env.VIN),
|
||||
quote(env.Phone),
|
||||
quote(env.DeviceID))
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
w.cache.mu.Lock()
|
||||
w.cache.seen[key] = struct{}{}
|
||||
w.cache.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawValues(env envelope.FrameEnvelope) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
eventTime := millis(env.EventTimeMS)
|
||||
return []any{
|
||||
received,
|
||||
frameID(env),
|
||||
env.StableEventID(),
|
||||
messageIDInt(env.MessageID),
|
||||
eventTime,
|
||||
received,
|
||||
rawSizeBytes(env),
|
||||
env.RawHex,
|
||||
env.RawText,
|
||||
jsonString(env.Parsed),
|
||||
jsonString(env.Fields),
|
||||
string(env.ParseStatus),
|
||||
env.ParseError,
|
||||
env.SourceEndpoint,
|
||||
}
|
||||
}
|
||||
|
||||
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
longitude,
|
||||
latitude,
|
||||
floatFieldOrNil(env, "altitude_m"),
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
intFieldOrNil(env, "direction_deg"),
|
||||
intFieldOrNil(env, "alarm_flag"),
|
||||
intFieldOrNil(env, "status_flag"),
|
||||
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
|
||||
}
|
||||
}
|
||||
|
||||
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
totalMileage,
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
floatFieldOrNil(env, envelope.FieldLongitude),
|
||||
floatFieldOrNil(env, envelope.FieldLatitude),
|
||||
}
|
||||
}
|
||||
|
||||
func frameID(env envelope.FrameEnvelope) string {
|
||||
return "go_" + env.StableEventID()
|
||||
}
|
||||
|
||||
func tableName(prefix string, env envelope.FrameEnvelope) string {
|
||||
return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
|
||||
}
|
||||
|
||||
func hash16(value string) string {
|
||||
sum := sha1.Sum([]byte(value))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
func messageIDInt(value string) int64 {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
value = strings.TrimPrefix(value, "0x")
|
||||
parsed, err := strconv.ParseInt(value, 16, 32)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
parsed, _ = strconv.ParseInt(value, 10, 32)
|
||||
return parsed
|
||||
}
|
||||
|
||||
func rawSizeBytes(env envelope.FrameEnvelope) int {
|
||||
rawHex := strings.TrimSpace(env.RawHex)
|
||||
if len(rawHex)%2 != 0 {
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
if rawHex != "" {
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
return len([]byte(env.RawText))
|
||||
}
|
||||
|
||||
func jsonString(value any) string {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||
if env.Fields == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
value, ok := floatField(env, key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
if env.Fields == nil {
|
||||
return nil
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return nil
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return typed
|
||||
case int64:
|
||||
return typed
|
||||
case uint16:
|
||||
return int64(typed)
|
||||
case uint32:
|
||||
return int64(typed)
|
||||
case float64:
|
||||
return int64(typed)
|
||||
case string:
|
||||
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
|
||||
if env.EventTimeMS > 0 {
|
||||
return millis(env.EventTimeMS)
|
||||
}
|
||||
return millis(env.ReceivedAtMS)
|
||||
}
|
||||
|
||||
func millis(value int64) time.Time {
|
||||
if value <= 0 {
|
||||
return time.UnixMilli(time.Now().UnixMilli()).UTC()
|
||||
}
|
||||
return time.UnixMilli(value).UTC()
|
||||
}
|
||||
|
||||
func quote(value string) string {
|
||||
return strings.ReplaceAll(value, "'", "''")
|
||||
}
|
||||
|
||||
func joinLiterals(values []any) string {
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, literal(value))
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
func literal(value any) string {
|
||||
if value == nil {
|
||||
return "NULL"
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case time.Time:
|
||||
return "'" + typed.UTC().Format("2006-01-02 15:04:05.000") + "'"
|
||||
case string:
|
||||
return "'" + quote(typed) + "'"
|
||||
case envelope.ParseStatus:
|
||||
return "'" + quote(string(typed)) + "'"
|
||||
case int:
|
||||
return strconv.FormatInt(int64(typed), 10)
|
||||
case int64:
|
||||
return strconv.FormatInt(typed, 10)
|
||||
case uint16:
|
||||
return strconv.FormatUint(uint64(typed), 10)
|
||||
case uint32:
|
||||
return strconv.FormatUint(uint64(typed), 10)
|
||||
case float64:
|
||||
return strconv.FormatFloat(typed, 'f', -1, 64)
|
||||
case float32:
|
||||
return strconv.FormatFloat(float64(typed), 'f', -1, 64)
|
||||
default:
|
||||
return "'" + quote(fmt.Sprint(typed)) + "'"
|
||||
}
|
||||
}
|
||||
146
go/vehicle-gateway/internal/history/writer_test.go
Normal file
146
go/vehicle-gateway/internal/history/writer_test.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
|
||||
statements := strings.Join(SchemaStatements("test_ts"), "\n")
|
||||
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} {
|
||||
if !strings.Contains(statements, want) {
|
||||
t.Fatalf("schema missing %q:\n%s", want, statements)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec)
|
||||
env := sampleEnvelope()
|
||||
|
||||
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||
t.Fatalf("AppendAll() error = %v", err)
|
||||
}
|
||||
|
||||
if got := countSQL(exec.calls, "USING raw_frames"); got != 1 {
|
||||
t.Fatalf("raw child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
|
||||
t.Fatalf("location child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 {
|
||||
t.Fatalf("mileage child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||
t.Fatalf("raw insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
|
||||
t.Fatalf("location insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 {
|
||||
t.Fatalf("mileage insert count = %d", got)
|
||||
}
|
||||
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
|
||||
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {
|
||||
t.Fatalf("raw insert should quote string literals: %s", rawInsert)
|
||||
}
|
||||
if strings.Contains(rawInsert, "VALUES (?,") {
|
||||
t.Fatalf("raw insert should not use placeholders for tdengine websocket plain insert: %s", rawInsert)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec)
|
||||
env := sampleEnvelope()
|
||||
env.Fields = map[string]any{}
|
||||
|
||||
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||
t.Fatalf("AppendAll() error = %v", err)
|
||||
}
|
||||
|
||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||
t.Fatalf("raw insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
|
||||
t.Fatalf("location insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
|
||||
t.Fatalf("mileage insert count = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawSizeBytesUsesRawTextWhenHexIsEmpty(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{RawText: `{"code":"0F80","data":{"speed":12}}`}
|
||||
if got, want := rawSizeBytes(env), len([]byte(env.RawText)); got != want {
|
||||
t.Fatalf("raw text size = %d, want %d", got, want)
|
||||
}
|
||||
|
||||
env.RawHex = "7E0200"
|
||||
if got, want := rawSizeBytes(env), 3; got != want {
|
||||
t.Fatalf("raw hex size = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func sampleEnvelope() envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Sequence: 1,
|
||||
VIN: "LNBVIN00000000001",
|
||||
Phone: "013307795425",
|
||||
SourceEndpoint: "127.0.0.1:18080",
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7E0200",
|
||||
Parsed: map[string]any{"message": "location"},
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 121.069881,
|
||||
envelope.FieldLatitude: 30.590151,
|
||||
envelope.FieldSpeedKMH: 23.0,
|
||||
envelope.FieldTotalMileageKM: 10241.2,
|
||||
"direction_deg": uint16(79),
|
||||
"alarm_flag": uint32(0),
|
||||
"status_flag": uint32(72),
|
||||
},
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
}
|
||||
|
||||
func countSQL(calls []execCall, pattern string) int {
|
||||
var count int
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call.query, pattern) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func findSQL(calls []execCall, pattern string) string {
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call.query, pattern) {
|
||||
return call.query
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type execCall struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type recordingExec struct {
|
||||
calls []execCall
|
||||
}
|
||||
|
||||
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||
return nil, nil
|
||||
}
|
||||
237
go/vehicle-gateway/internal/identity/resolver.go
Normal file
237
go/vehicle-gateway/internal/identity/resolver.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Resolver interface {
|
||||
Resolve(context.Context, envelope.FrameEnvelope) (envelope.FrameEnvelope, error)
|
||||
}
|
||||
|
||||
type NoopResolver struct{}
|
||||
|
||||
func (NoopResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||
return env, nil
|
||||
}
|
||||
|
||||
type MySQLResolver struct {
|
||||
db *sql.DB
|
||||
table string
|
||||
}
|
||||
|
||||
func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver {
|
||||
if db == nil {
|
||||
panic("identity db must not be nil")
|
||||
}
|
||||
table = strings.TrimSpace(table)
|
||||
if table == "" || !safeIdentifier(table) {
|
||||
table = "vehicle_identity_binding"
|
||||
}
|
||||
return &MySQLResolver{db: db, table: table}
|
||||
}
|
||||
|
||||
func safeIdentifier(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||
if strings.TrimSpace(env.VIN) != "" {
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
candidates := CandidateKeys(env)
|
||||
for _, candidate := range candidates {
|
||||
vin, err := r.lookup(ctx, candidate.Column, candidate.Value)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
return env, err
|
||||
}
|
||||
if strings.TrimSpace(vin) != "" {
|
||||
env.VIN = strings.TrimSpace(vin)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{
|
||||
"resolved": true,
|
||||
"source": candidate.Column,
|
||||
"value": candidate.Value,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
}
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) {
|
||||
query := "SELECT vin FROM " + r.table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' ORDER BY updated_at DESC LIMIT 1"
|
||||
var vin string
|
||||
err := r.db.QueryRowContext(ctx, query, value).Scan(&vin)
|
||||
return vin, err
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) trackJT808(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if env.Protocol != envelope.ProtocolJT808 || strings.TrimSpace(env.Phone) == "" || strings.TrimSpace(env.MessageID) == "" {
|
||||
return nil
|
||||
}
|
||||
registration := mapValue(env.Parsed, "registration")
|
||||
authentication := mapValue(env.Parsed, "authentication")
|
||||
deviceID := firstNonEmpty(env.DeviceID, textValue(registration, "device_id"))
|
||||
plate := firstNonEmpty(env.Plate, textValue(registration, "plate"))
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
if vin == "" {
|
||||
vin = "unknown"
|
||||
}
|
||||
firstRegisteredAt := nilTime()
|
||||
latestRegisteredAt := nilTime()
|
||||
if env.MessageID == "0x0100" {
|
||||
firstRegisteredAt = "CURRENT_TIMESTAMP"
|
||||
latestRegisteredAt = "CURRENT_TIMESTAMP"
|
||||
}
|
||||
latestAuthenticatedAt := nilTime()
|
||||
if env.MessageID == "0x0102" {
|
||||
latestAuthenticatedAt = "CURRENT_TIMESTAMP"
|
||||
}
|
||||
|
||||
query := `INSERT INTO jt808_registration
|
||||
(phone, device_id, plate, vin, province, city, manufacturer, device_type, plate_color,
|
||||
auth_token, auth_imei, auth_software_version, source_endpoint,
|
||||
first_registered_at, latest_registered_at, latest_authenticated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ` + firstRegisteredAt + `, ` + latestRegisteredAt + `, ` + latestAuthenticatedAt + `)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
device_id = IF(VALUES(device_id) <> '', VALUES(device_id), device_id),
|
||||
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
|
||||
vin = IF(VALUES(vin) <> 'unknown', VALUES(vin), vin),
|
||||
province = IF(VALUES(province) <> '', VALUES(province), province),
|
||||
city = IF(VALUES(city) <> '', VALUES(city), city),
|
||||
manufacturer = IF(VALUES(manufacturer) <> '', VALUES(manufacturer), manufacturer),
|
||||
device_type = IF(VALUES(device_type) <> '', VALUES(device_type), device_type),
|
||||
plate_color = IF(VALUES(plate_color) <> '', VALUES(plate_color), plate_color),
|
||||
auth_token = IF(VALUES(auth_token) <> '', VALUES(auth_token), auth_token),
|
||||
auth_imei = IF(VALUES(auth_imei) <> '', VALUES(auth_imei), auth_imei),
|
||||
auth_software_version = IF(VALUES(auth_software_version) <> '', VALUES(auth_software_version), auth_software_version),
|
||||
source_endpoint = IF(VALUES(source_endpoint) <> '', VALUES(source_endpoint), source_endpoint),
|
||||
latest_seen_at = CURRENT_TIMESTAMP,
|
||||
first_registered_at = COALESCE(first_registered_at, VALUES(first_registered_at)),
|
||||
latest_registered_at = COALESCE(VALUES(latest_registered_at), latest_registered_at),
|
||||
latest_authenticated_at = COALESCE(VALUES(latest_authenticated_at), latest_authenticated_at)`
|
||||
_, err := r.db.ExecContext(ctx, query,
|
||||
env.Phone,
|
||||
deviceID,
|
||||
plate,
|
||||
vin,
|
||||
textValue(registration, "province"),
|
||||
textValue(registration, "city"),
|
||||
textValue(registration, "manufacturer"),
|
||||
textValue(registration, "device_type"),
|
||||
textValue(registration, "plate_color"),
|
||||
textValue(authentication, "token"),
|
||||
textValue(authentication, "imei"),
|
||||
textValue(authentication, "software_version"),
|
||||
env.SourceEndpoint,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if vin == "unknown" {
|
||||
return nil
|
||||
}
|
||||
_, err = r.db.ExecContext(ctx, `UPDATE IGNORE `+r.table+`
|
||||
SET
|
||||
phone = CASE WHEN (phone IS NULL OR phone = '') AND ? <> '' THEN ? ELSE phone END,
|
||||
device_id = CASE WHEN (device_id IS NULL OR device_id = '') AND ? <> '' THEN ? ELSE device_id END,
|
||||
plate = CASE WHEN (plate IS NULL OR plate = '') AND ? <> '' THEN ? ELSE plate END
|
||||
WHERE vin = ?`, env.Phone, env.Phone, deviceID, deviceID, plate, plate, vin)
|
||||
return err
|
||||
}
|
||||
|
||||
type CandidateKey struct {
|
||||
Column string
|
||||
Value string
|
||||
}
|
||||
|
||||
func CandidateKeys(env envelope.FrameEnvelope) []CandidateKey {
|
||||
var out []CandidateKey
|
||||
add := func(column string, value string) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || strings.EqualFold(value, "unknown") {
|
||||
return
|
||||
}
|
||||
for _, existing := range out {
|
||||
if existing.Column == column && existing.Value == value {
|
||||
return
|
||||
}
|
||||
}
|
||||
out = append(out, CandidateKey{Column: column, Value: value})
|
||||
}
|
||||
add("phone", env.Phone)
|
||||
if trimmed := strings.TrimLeft(strings.TrimSpace(env.Phone), "0"); trimmed != "" {
|
||||
add("phone", trimmed)
|
||||
}
|
||||
add("device_id", env.DeviceID)
|
||||
add("plate", env.Plate)
|
||||
add("vin", env.VehicleKeyHint)
|
||||
return out
|
||||
}
|
||||
|
||||
func mapValue(values map[string]any, key string) map[string]any {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
nested, _ := values[key].(map[string]any)
|
||||
return nested
|
||||
}
|
||||
|
||||
func textValue(values map[string]any, key string) string {
|
||||
if values == nil {
|
||||
return ""
|
||||
}
|
||||
value, ok := values[key]
|
||||
if !ok || value == nil {
|
||||
return ""
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func nilTime() string {
|
||||
return "NULL"
|
||||
}
|
||||
171
go/vehicle-gateway/internal/identity/resolver_test.go
Normal file
171
go/vehicle-gateway/internal/identity/resolver_test.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestCandidateKeysPrioritizesPhoneDevicePlate(t *testing.T) {
|
||||
keys := CandidateKeys(envelope.FrameEnvelope{
|
||||
Phone: "013307795425",
|
||||
DeviceID: "D1",
|
||||
Plate: "豫A12345",
|
||||
VehicleKeyHint: "D1",
|
||||
})
|
||||
got := []string{}
|
||||
for _, key := range keys {
|
||||
got = append(got, key.Column+"="+key.Value)
|
||||
}
|
||||
want := []string{"phone=013307795425", "phone=13307795425", "device_id=D1", "plate=豫A12345", "vin=D1"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("candidate count = %d got=%#v", len(got), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateKeysAddsTrimmedPhoneVariant(t *testing.T) {
|
||||
keys := CandidateKeys(envelope.FrameEnvelope{
|
||||
Phone: "013079963379",
|
||||
})
|
||||
got := []string{}
|
||||
for _, key := range keys {
|
||||
got = append(got, key.Column+"="+key.Value)
|
||||
}
|
||||
want := []string{"phone=013079963379", "phone=13079963379"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("candidate count = %d got=%#v", len(got), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001"))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNBVIN00000000001" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "phone" {
|
||||
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverFallsBackToDeviceID(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
|
||||
WithArgs("D1").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000002"))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
DeviceID: "D1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNBVIN00000000002" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverTracksJT808RegistrationAndBackfillsBinding(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013079963379").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13079963379").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
|
||||
WithArgs("DEV0001").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("TEST123").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LKLG7C4E3NA774736"))
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE IGNORE vehicle_identity_binding").
|
||||
WithArgs("013079963379", "013079963379", "DEV0001", "DEV0001", "TEST123", "TEST123", "LKLG7C4E3NA774736").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0100",
|
||||
Phone: "013079963379",
|
||||
DeviceID: "DEV0001",
|
||||
Plate: "TEST123",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
Parsed: map[string]any{
|
||||
"registration": map[string]any{
|
||||
"province": uint16(16),
|
||||
"city": uint16(32),
|
||||
"manufacturer": "YUTNG",
|
||||
"device_type": "ZK6105CHEVNPG4",
|
||||
"device_id": "DEV0001",
|
||||
"plate_color": uint8(2),
|
||||
"plate": "TEST123",
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LKLG7C4E3NA774736" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) {
|
||||
t.Helper()
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
return db, mock
|
||||
}
|
||||
13
go/vehicle-gateway/internal/observability/logger.go
Normal file
13
go/vehicle-gateway/internal/observability/logger.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
// NewLogger returns a JSON logger with a stable service field so ECS logs from
|
||||
// different Go processes can be filtered without relying on container names.
|
||||
func NewLogger(service string) *slog.Logger {
|
||||
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||
return slog.New(handler).With("service", service)
|
||||
}
|
||||
840
go/vehicle-gateway/internal/protocol/gb32960/parser.go
Normal file
840
go/vehicle-gateway/internal/protocol/gb32960/parser.go
Normal file
@@ -0,0 +1,840 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFrameTooShort = errors.New("gb32960 frame too short")
|
||||
ErrBadStartSymbol = errors.New("gb32960 bad start symbol")
|
||||
ErrBodyLength = errors.New("gb32960 body length mismatch")
|
||||
ErrBCC = errors.New("gb32960 bcc mismatch")
|
||||
)
|
||||
|
||||
const (
|
||||
minFrameLen = 25
|
||||
headerLen = 24
|
||||
)
|
||||
|
||||
// ExtractFrames splits GB/T 32960 TCP streams using the protocol length field.
|
||||
// Returned frames include the start symbols and trailing BCC byte.
|
||||
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
|
||||
offset := 0
|
||||
for {
|
||||
start := indexStart(stream[offset:])
|
||||
if start < 0 {
|
||||
return frames, nil, nil
|
||||
}
|
||||
offset += start
|
||||
remaining := stream[offset:]
|
||||
if len(remaining) < headerLen {
|
||||
return frames, append([]byte(nil), remaining...), nil
|
||||
}
|
||||
bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
|
||||
frameLen := frameLength(remaining, bodyLen)
|
||||
if len(remaining) < frameLen {
|
||||
return frames, append([]byte(nil), remaining...), nil
|
||||
}
|
||||
frames = append(frames, append([]byte(nil), remaining[:frameLen]...))
|
||||
offset += frameLen
|
||||
if offset >= len(stream) {
|
||||
return frames, nil, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||
if len(raw) < minFrameLen {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
version, ok := version(raw[0], raw[1])
|
||||
if !ok {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
|
||||
}
|
||||
bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
|
||||
actualBodyLen := len(raw) - headerLen - 1
|
||||
if len(raw) != headerLen+bodyLen+1 && !isExtendedPlatformLogin(raw[2], bodyLen, actualBodyLen) {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
|
||||
}
|
||||
if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrBCC, got, want)
|
||||
}
|
||||
|
||||
command := raw[2]
|
||||
responseFlag := raw[3]
|
||||
vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
|
||||
body := raw[headerLen : headerLen+actualBodyLen]
|
||||
parsed := map[string]any{
|
||||
"header": map[string]any{
|
||||
"version": version,
|
||||
"command": fmt.Sprintf("0x%02X", command),
|
||||
"response_flag": fmt.Sprintf("0x%02X", responseFlag),
|
||||
"vin": vin,
|
||||
"encrypt": raw[21],
|
||||
"body_length": bodyLen,
|
||||
"actual_body_length": actualBodyLen,
|
||||
},
|
||||
}
|
||||
fields := map[string]any{}
|
||||
eventTimeMS := receivedAtMS
|
||||
|
||||
if command == 0x02 || command == 0x03 {
|
||||
eventTime, units := parseDataBody(version, body, fields)
|
||||
if !eventTime.IsZero() {
|
||||
eventTimeMS = eventTime.UnixMilli()
|
||||
fields["device_time"] = eventTime.Format(time.RFC3339)
|
||||
parsed["device_time"] = eventTime.Format(time.RFC3339)
|
||||
}
|
||||
parsed["data_units"] = units
|
||||
} else if command == 0x05 {
|
||||
login := parsePlatformLogin(body)
|
||||
parsed["platform_login"] = login
|
||||
if deviceTime, ok := login["login_time"].(string); ok && deviceTime != "" {
|
||||
fields["device_time"] = deviceTime
|
||||
parsed["device_time"] = deviceTime
|
||||
}
|
||||
if account, ok := login["username"].(string); ok && account != "" {
|
||||
fields["platform_account"] = account
|
||||
}
|
||||
}
|
||||
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
MessageID: fmt.Sprintf("0x%02X", command),
|
||||
VIN: vin,
|
||||
SourceEndpoint: sourceEndpoint,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
|
||||
if len(body) < 6 {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
eventTime := parseGBTime(body[:6])
|
||||
cursor := 6
|
||||
var units []map[string]any
|
||||
for cursor < len(body) {
|
||||
unitType := body[cursor]
|
||||
cursor++
|
||||
switch unitType {
|
||||
case 0x01:
|
||||
size := 20
|
||||
if version == "V2025" {
|
||||
size = 18
|
||||
}
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x01", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseVehicleData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x01", "name": "vehicle", "value": unit})
|
||||
fields["vehicle_status"] = unit["vehicle_status"]
|
||||
fields["charge_status"] = unit["charge_status"]
|
||||
fields["running_mode"] = unit["running_mode"]
|
||||
fields[envelope.FieldSpeedKMH] = unit["speed_kmh"]
|
||||
fields[envelope.FieldTotalMileageKM] = unit["total_mileage_km"]
|
||||
fields[envelope.FieldSOCPercent] = unit["soc_percent"]
|
||||
cursor += size
|
||||
case 0x05:
|
||||
if len(body[cursor:]) < 9 {
|
||||
units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parsePositionData(body[cursor : cursor+9])
|
||||
units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
|
||||
fields["position_status"] = unit["position_status"]
|
||||
fields[envelope.FieldLongitude] = unit["longitude"]
|
||||
fields[envelope.FieldLatitude] = unit["latitude"]
|
||||
cursor += 9
|
||||
case 0x02:
|
||||
if len(body[cursor:]) < 1 {
|
||||
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
size := 1 + int(body[cursor])*12
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseDriveMotorData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x02", "name": "drive_motor", "value": unit})
|
||||
cursor += size
|
||||
case 0x03:
|
||||
if len(body[cursor:]) < 8 {
|
||||
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
probeCount := int(binary.BigEndian.Uint16(body[cursor+6 : cursor+8]))
|
||||
size := 8 + probeCount + 10
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseFuelCellData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x03", "name": "fuel_cell", "value": unit})
|
||||
fields["fuel_cell_hydrogen_consumption_kg_per_100km"] = unit["hydrogen_consumption_kg_per_100km"]
|
||||
fields["fuel_cell_voltage_v"] = unit["fuel_cell_voltage_v"]
|
||||
fields["fuel_cell_current_a"] = unit["fuel_cell_current_a"]
|
||||
cursor += size
|
||||
case 0x04:
|
||||
if len(body[cursor:]) < 5 {
|
||||
units = append(units, map[string]any{"type": "0x04", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseEngineData(body[cursor : cursor+5])
|
||||
units = append(units, map[string]any{"type": "0x04", "name": "engine", "value": unit})
|
||||
cursor += 5
|
||||
case 0x06:
|
||||
if len(body[cursor:]) < 14 {
|
||||
units = append(units, map[string]any{"type": "0x06", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseExtremeData(body[cursor : cursor+14])
|
||||
units = append(units, map[string]any{"type": "0x06", "name": "extreme", "value": unit})
|
||||
cursor += 14
|
||||
case 0x07:
|
||||
size, ok := alarmDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x07", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseAlarmData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x07", "name": "alarm", "value": unit})
|
||||
cursor += size
|
||||
case 0x08:
|
||||
size, ok := voltageDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x08", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseVoltageData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x08", "name": "voltage", "value": unit})
|
||||
cursor += size
|
||||
case 0x09:
|
||||
size, ok := temperatureDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x09", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseTemperatureData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x09", "name": "temperature", "value": unit})
|
||||
cursor += size
|
||||
case 0x30:
|
||||
size, ok := gdFCStackDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x30", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCStackData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x30", "name": "gd_fc_stack", "value": unit})
|
||||
if summaries, ok := unit["summaries"].([]map[string]any); ok && len(summaries) > 0 {
|
||||
fields["gd_fc_stack_hydrogen_inlet_pressure_kpa"] = summaries[0]["hydrogen_inlet_pressure_kpa"]
|
||||
fields["gd_fc_stack_water_outlet_temp_c"] = summaries[0]["stack_water_outlet_temp_c"]
|
||||
fields["gd_fc_stack_cell_count"] = summaries[0]["cell_count"]
|
||||
fields["gd_fc_stack_frame_cell_count"] = summaries[0]["frame_cell_count"]
|
||||
}
|
||||
cursor += size
|
||||
case 0x31:
|
||||
size, ok := gdFCAuxiliaryDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x31", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCAuxiliaryData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x31", "name": "gd_fc_auxiliary", "value": unit})
|
||||
cursor += size
|
||||
case 0x32:
|
||||
if len(body[cursor:]) < 9 {
|
||||
units = append(units, map[string]any{"type": "0x32", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCDcDcData(body[cursor : cursor+9])
|
||||
units = append(units, map[string]any{"type": "0x32", "name": "gd_fc_dcdc", "value": unit})
|
||||
fields["gd_fc_dcdc_controller_temp_c"] = unit["controller_temp_c"]
|
||||
cursor += 9
|
||||
case 0x33:
|
||||
if len(body[cursor:]) < 5 {
|
||||
units = append(units, map[string]any{"type": "0x33", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCAirConditionerData(body[cursor : cursor+5])
|
||||
units = append(units, map[string]any{"type": "0x33", "name": "gd_fc_air_conditioner", "value": unit})
|
||||
cursor += 5
|
||||
case 0x34:
|
||||
if len(body[cursor:]) < 7 {
|
||||
units = append(units, map[string]any{"type": "0x34", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCVehicleInfoData(body[cursor : cursor+7])
|
||||
units = append(units, map[string]any{"type": "0x34", "name": "gd_fc_vehicle_info", "value": unit})
|
||||
fields["gd_fc_vehicle_hydrogen_mass_kg"] = unit["hydrogen_mass_kg"]
|
||||
cursor += 7
|
||||
case 0x80:
|
||||
if len(body[cursor:]) < 11 {
|
||||
units = append(units, map[string]any{"type": "0x80", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCDemoExtensionData(body[cursor : cursor+11])
|
||||
units = append(units, map[string]any{"type": "0x80", "name": "gd_fc_demo_extension", "value": unit})
|
||||
fields["gd_fc_demo_stack_temp_c"] = unit["stack_temp_c"]
|
||||
cursor += 11
|
||||
case 0x83:
|
||||
size, ok := gdFCVendorTLVDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x83", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCVendorTLVData(0x83, body[cursor:cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x83", "name": "gd_fc_vendor_tlv", "value": unit})
|
||||
cursor += size
|
||||
default:
|
||||
units = append(units, map[string]any{
|
||||
"type": fmt.Sprintf("0x%02X", unitType),
|
||||
"raw_tail": strings.ToUpper(hex.EncodeToString(body[cursor:])),
|
||||
"parse": "unknown_unit",
|
||||
"byte_size": len(body) - cursor,
|
||||
})
|
||||
return eventTime, units
|
||||
}
|
||||
}
|
||||
return eventTime, units
|
||||
}
|
||||
|
||||
func frameLength(frameStart []byte, bodyLen int) int {
|
||||
standardLen := headerLen + bodyLen + 1
|
||||
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
|
||||
return standardLen
|
||||
}
|
||||
// Some field platform-login frames declare the standard 41-byte body but
|
||||
// carry a 32-byte password, making the actual body 53 bytes. Only consume
|
||||
// the extended form when its own BCC verifies so the splitter does not eat
|
||||
// bytes from the next frame.
|
||||
extendedLen := headerLen + 53 + 1
|
||||
if len(frameStart) < extendedLen {
|
||||
return standardLen
|
||||
}
|
||||
extended := frameStart[:extendedLen]
|
||||
if bcc(extended[2:len(extended)-1]) == extended[len(extended)-1] {
|
||||
return extendedLen
|
||||
}
|
||||
return standardLen
|
||||
}
|
||||
|
||||
func isExtendedPlatformLogin(command byte, declaredBodyLen int, actualBodyLen int) bool {
|
||||
return command == 0x05 && declaredBodyLen == 41 && actualBodyLen == 53
|
||||
}
|
||||
|
||||
func parsePlatformLogin(body []byte) map[string]any {
|
||||
login := map[string]any{
|
||||
"raw_body_length": len(body),
|
||||
}
|
||||
if len(body) < 9 {
|
||||
login["error"] = "truncated"
|
||||
return login
|
||||
}
|
||||
loginTime := parseGBTime(body[:6])
|
||||
if !loginTime.IsZero() {
|
||||
login["login_time"] = loginTime.Format(time.RFC3339)
|
||||
}
|
||||
login["serial_no"] = binary.BigEndian.Uint16(body[6:8])
|
||||
if len(body) >= 21 {
|
||||
login["username"] = trimASCII(body[8:20])
|
||||
}
|
||||
if len(body) > 21 {
|
||||
passwordEnd := len(body) - 1
|
||||
login["password"] = trimASCII(body[20:passwordEnd])
|
||||
login["encrypt_rule"] = body[passwordEnd]
|
||||
}
|
||||
return login
|
||||
}
|
||||
|
||||
func parseVehicleData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"vehicle_status": int(data[0]),
|
||||
"charge_status": int(data[1]),
|
||||
"running_mode": int(data[2]),
|
||||
"speed_kmh": float64(binary.BigEndian.Uint16(data[3:5])) / 10,
|
||||
"total_mileage_km": float64(binary.BigEndian.Uint32(data[5:9])) / 10,
|
||||
"total_voltage_v": float64(binary.BigEndian.Uint16(data[9:11])) / 10,
|
||||
"total_current_a": float64(binary.BigEndian.Uint16(data[11:13]))/10 - 1000,
|
||||
"soc_percent": int(data[13]),
|
||||
"dc_dc_status": int(data[14]),
|
||||
"gear": int(data[15]),
|
||||
"insulation_kohm": binary.BigEndian.Uint16(data[16:18]),
|
||||
"accelerator_pct": int(data[18]),
|
||||
"brake_pct": int(data[19]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseDriveMotorData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
motors := make([]map[string]any, 0, count)
|
||||
cursor := 1
|
||||
for i := 0; i < count; i++ {
|
||||
motors = append(motors, map[string]any{
|
||||
"serial_no": int(data[cursor]),
|
||||
"state": int(data[cursor+1]),
|
||||
"controller_temperature_c": int(data[cursor+2]) - 40,
|
||||
"speed_rpm": int(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) - 20000,
|
||||
"torque_nm": float64(int(binary.BigEndian.Uint16(data[cursor+5:cursor+7]))-20000) / 10,
|
||||
"motor_temperature_c": int(data[cursor+7]) - 40,
|
||||
"controller_voltage_v": float64(binary.BigEndian.Uint16(data[cursor+8:cursor+10])) / 10,
|
||||
"controller_current_a": float64(binary.BigEndian.Uint16(data[cursor+10:cursor+12]))/10 - 1000,
|
||||
})
|
||||
cursor += 12
|
||||
}
|
||||
return map[string]any{
|
||||
"count": count,
|
||||
"motors": motors,
|
||||
}
|
||||
}
|
||||
|
||||
func parseFuelCellData(data []byte) map[string]any {
|
||||
probeCount := int(binary.BigEndian.Uint16(data[6:8]))
|
||||
probes := make([]int, 0, probeCount)
|
||||
cursor := 8
|
||||
for i := 0; i < probeCount; i++ {
|
||||
probes = append(probes, int(data[cursor])-40)
|
||||
cursor++
|
||||
}
|
||||
out := map[string]any{
|
||||
"fuel_cell_voltage_v": float64(binary.BigEndian.Uint16(data[0:2])) / 10,
|
||||
"fuel_cell_current_a": float64(binary.BigEndian.Uint16(data[2:4])) / 10,
|
||||
"hydrogen_consumption_kg_per_100km": float64(binary.BigEndian.Uint16(data[4:6])) / 100,
|
||||
"temperature_probe_count": probeCount,
|
||||
"temperature_probe_values_c": probes,
|
||||
"max_hydrogen_temperature_c": float64(binary.BigEndian.Uint16(data[cursor:cursor+2]))/10 - 40,
|
||||
"max_hydrogen_temperature_probe_id": int(data[cursor+2]),
|
||||
"max_hydrogen_concentration_fraction": float64(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) * 0.000001,
|
||||
"max_hydrogen_concentration_probe_id": int(data[cursor+5]),
|
||||
"max_hydrogen_pressure_mpa": float64(binary.BigEndian.Uint16(data[cursor+6:cursor+8])) / 10,
|
||||
"max_hydrogen_pressure_probe_id": int(data[cursor+8]),
|
||||
"dc_dc_status": int(data[cursor+9]),
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseEngineData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"engine_status": int(data[0]),
|
||||
"crank_speed_rpm": binary.BigEndian.Uint16(data[1:3]),
|
||||
"fuel_rate": binary.BigEndian.Uint16(data[3:5]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseExtremeData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"max_voltage_subsystem_no": int(data[0]),
|
||||
"max_voltage_cell_no": int(data[1]),
|
||||
"max_voltage_v": float64(binary.BigEndian.Uint16(data[2:4])) / 1000,
|
||||
"min_voltage_subsystem_no": int(data[4]),
|
||||
"min_voltage_cell_no": int(data[5]),
|
||||
"min_voltage_v": float64(binary.BigEndian.Uint16(data[6:8])) / 1000,
|
||||
"max_temp_subsystem_no": int(data[8]),
|
||||
"max_temp_probe_no": int(data[9]),
|
||||
"max_temp_c": int(data[10]) - 40,
|
||||
"min_temp_subsystem_no": int(data[11]),
|
||||
"min_temp_probe_no": int(data[12]),
|
||||
"min_temp_c": int(data[13]) - 40,
|
||||
}
|
||||
}
|
||||
|
||||
func alarmDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 5 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 5
|
||||
for i := 0; i < 4; i++ {
|
||||
if len(data[cursor:]) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
count := int(data[cursor])
|
||||
cursor++
|
||||
cursor += count * 4
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseAlarmData(data []byte) map[string]any {
|
||||
cursor := 5
|
||||
readList := func() []string {
|
||||
count := int(data[cursor])
|
||||
cursor++
|
||||
values := make([]string, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
values = append(values, fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[cursor:cursor+4])))
|
||||
cursor += 4
|
||||
}
|
||||
return values
|
||||
}
|
||||
return map[string]any{
|
||||
"max_alarm_level": int(data[0]),
|
||||
"general_alarm_flag": fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[1:5])),
|
||||
"battery_faults": readList(),
|
||||
"motor_faults": readList(),
|
||||
"engine_faults": readList(),
|
||||
"other_faults": readList(),
|
||||
}
|
||||
}
|
||||
|
||||
func voltageDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 10 {
|
||||
return 0, false
|
||||
}
|
||||
frameCellCount := int(data[cursor+9])
|
||||
cursor += 10 + frameCellCount*2
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseVoltageData(data []byte) map[string]any {
|
||||
subCount := int(data[0])
|
||||
cursor := 1
|
||||
totalVoltage := 0.0
|
||||
maxCell := 0.0
|
||||
minCell := 0.0
|
||||
cellSeen := false
|
||||
for i := 0; i < subCount; i++ {
|
||||
totalVoltage += float64(binary.BigEndian.Uint16(data[cursor+1:cursor+3])) / 10
|
||||
cellCount := int(binary.BigEndian.Uint16(data[cursor+5 : cursor+7]))
|
||||
frameCellCount := int(data[cursor+9])
|
||||
cursor += 10
|
||||
for c := 0; c < frameCellCount; c++ {
|
||||
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
|
||||
if !cellSeen || voltage > maxCell {
|
||||
maxCell = voltage
|
||||
}
|
||||
if !cellSeen || voltage < minCell {
|
||||
minCell = voltage
|
||||
}
|
||||
cellSeen = true
|
||||
cursor += 2
|
||||
}
|
||||
_ = cellCount
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": subCount,
|
||||
"total_voltage_v": totalVoltage,
|
||||
"max_cell_v": maxCell,
|
||||
"min_cell_v": minCell,
|
||||
}
|
||||
}
|
||||
|
||||
func temperatureDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 3 {
|
||||
return 0, false
|
||||
}
|
||||
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
|
||||
cursor += 3 + probeCount
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseTemperatureData(data []byte) map[string]any {
|
||||
subCount := int(data[0])
|
||||
cursor := 1
|
||||
maxTemp := 0
|
||||
minTemp := 0
|
||||
seen := false
|
||||
for i := 0; i < subCount; i++ {
|
||||
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
|
||||
cursor += 3
|
||||
for p := 0; p < probeCount; p++ {
|
||||
temp := int(data[cursor]) - 40
|
||||
if !seen || temp > maxTemp {
|
||||
maxTemp = temp
|
||||
}
|
||||
if !seen || temp < minTemp {
|
||||
minTemp = temp
|
||||
}
|
||||
seen = true
|
||||
cursor++
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": subCount,
|
||||
"max_temp_c": maxTemp,
|
||||
"min_temp_c": minTemp,
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCStackDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 22 {
|
||||
return 0, false
|
||||
}
|
||||
frameCellCount := int(data[cursor+21])
|
||||
cursor += 22 + frameCellCount*2
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseGdFCStackData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
cursor := 1
|
||||
summaries := make([]map[string]any, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
frameCellCount := int(data[cursor+21])
|
||||
summary := map[string]any{
|
||||
"engine_work_state": int(data[cursor]),
|
||||
"stack_water_outlet_temp_c": nullableTempOffset40(data[cursor+1]),
|
||||
"hydrogen_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+2:cursor+4], 0.1, -100),
|
||||
"air_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+4:cursor+6], 0.1, -100),
|
||||
"air_inlet_temp_c": nullableTempOffset40(data[cursor+6]),
|
||||
"max_cell_voltage_id": nullableU16(data[cursor+7 : cursor+9]),
|
||||
"min_cell_voltage_id": nullableU16(data[cursor+9 : cursor+11]),
|
||||
"max_cell_voltage_v": nullableScaledU16(data[cursor+11:cursor+13], 0.001),
|
||||
"min_cell_voltage_v": nullableScaledU16(data[cursor+13:cursor+15], 0.001),
|
||||
"avg_cell_voltage_v": nullableScaledU16(data[cursor+15:cursor+17], 0.001),
|
||||
"cell_count": nullableU16(data[cursor+17 : cursor+19]),
|
||||
"frame_cell_start": nullableU16(data[cursor+19 : cursor+21]),
|
||||
"frame_cell_count": frameCellCount,
|
||||
}
|
||||
cursor += 22
|
||||
maxCell := 0.0
|
||||
minCell := 0.0
|
||||
seen := false
|
||||
for c := 0; c < frameCellCount; c++ {
|
||||
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
|
||||
if !seen || voltage > maxCell {
|
||||
maxCell = voltage
|
||||
}
|
||||
if !seen || voltage < minCell {
|
||||
minCell = voltage
|
||||
}
|
||||
seen = true
|
||||
cursor += 2
|
||||
}
|
||||
if seen {
|
||||
summary["frame_max_cell_voltage_v"] = maxCell
|
||||
summary["frame_min_cell_voltage_v"] = minCell
|
||||
}
|
||||
summaries = append(summaries, summary)
|
||||
}
|
||||
return map[string]any{
|
||||
"stack_count": count,
|
||||
"summaries": summaries,
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCAuxiliaryDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
size := 1 + int(data[0])*16
|
||||
return size, len(data) >= size
|
||||
}
|
||||
|
||||
func parseGdFCAuxiliaryData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
cursor := 1
|
||||
subsystems := make([]map[string]any, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
subsystems = append(subsystems, map[string]any{
|
||||
"air_compressor_motor_voltage_v": nullableScaledU16(data[cursor:cursor+2], 0.1),
|
||||
"air_compressor_power_kw": nullableU16WithOffset(data[cursor+2:cursor+4], -5),
|
||||
"hydrogen_pump_motor_voltage_v": nullableScaledU16(data[cursor+4:cursor+6], 0.2),
|
||||
"hydrogen_pump_power_kw": nullableU16WithOffset(data[cursor+6:cursor+8], -5),
|
||||
"water_pump_voltage_v": nullableScaledU16(data[cursor+8:cursor+10], 0.1),
|
||||
"ptc_voltage_v": nullableScaledU16(data[cursor+10:cursor+12], 0.1),
|
||||
"ptc_power_kw": nullableU16WithOffset(data[cursor+12:cursor+14], -5),
|
||||
"low_voltage_battery_voltage_v": nullableScaledU16(data[cursor+14:cursor+16], 0.1),
|
||||
})
|
||||
cursor += 16
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": count,
|
||||
"subsystems": subsystems,
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCDcDcData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"input_voltage_v": nullableScaledU16(data[0:2], 0.1),
|
||||
"input_current_a": nullableScaledU16(data[2:4], 0.1),
|
||||
"output_voltage_v": nullableScaledU16(data[4:6], 0.1),
|
||||
"output_current_a": nullableScaledU16(data[6:8], 0.1),
|
||||
"controller_temp_c": nullableTempOffset40(data[8]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCAirConditionerData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"status": int(data[0]),
|
||||
"power_kw": nullableU16WithOffset(data[1:3], -5),
|
||||
"compressor_input_voltage_v": nullableScaledU16(data[3:5], 0.1),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCVehicleInfoData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"collision_alarm": int(data[0]),
|
||||
"ambient_temp_c": nullableU16WithOffset(data[1:3], -40),
|
||||
"ambient_pressure_kpa": nullableScaledU16WithOffset(data[3:5], 0.1, -100),
|
||||
"hydrogen_mass_kg": nullableScaledU16(data[5:7], 0.1),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCDemoExtensionData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"declared_length": int(binary.BigEndian.Uint16(data[0:2])),
|
||||
"stack_temp_c": nullableTempOffset40(data[2]),
|
||||
"air_compressor_voltage_v": nullableScaledU16(data[3:5], 0.1),
|
||||
"air_compressor_current_a": nullableScaledU16WithOffset(data[5:7], 0.1, -1000),
|
||||
"hydrogen_pump_voltage_v": nullableScaledU16(data[7:9], 0.1),
|
||||
"hydrogen_pump_current_a": nullableScaledU16WithOffset(data[9:11], 0.1, -1000),
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCVendorTLVDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 2 {
|
||||
return 0, false
|
||||
}
|
||||
size := 2 + int(binary.BigEndian.Uint16(data[0:2]))
|
||||
return size, len(data) >= size
|
||||
}
|
||||
|
||||
func parseGdFCVendorTLVData(typeCode byte, data []byte) map[string]any {
|
||||
length := int(binary.BigEndian.Uint16(data[0:2]))
|
||||
return map[string]any{
|
||||
"type": fmt.Sprintf("0x%02X", typeCode),
|
||||
"declared_length": length,
|
||||
"payload_hex": strings.ToUpper(hex.EncodeToString(data[2:])),
|
||||
}
|
||||
}
|
||||
|
||||
func nullableU16(data []byte) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return int(value)
|
||||
}
|
||||
|
||||
func nullableU16WithOffset(data []byte, offset int) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return int(value) + offset
|
||||
}
|
||||
|
||||
func nullableScaledU16(data []byte, scale float64) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return float64(value) * scale
|
||||
}
|
||||
|
||||
func nullableScaledU16WithOffset(data []byte, scale float64, offset float64) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return float64(value)*scale + offset
|
||||
}
|
||||
|
||||
func nullableTempOffset40(value byte) any {
|
||||
if value == 0xfe || value == 0xff {
|
||||
return nil
|
||||
}
|
||||
return int(value) - 40
|
||||
}
|
||||
|
||||
func parsePositionData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"position_status": int(data[0]),
|
||||
"longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
|
||||
"latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
func indexStart(data []byte) int {
|
||||
for i := 0; i+1 < len(data); i++ {
|
||||
if _, ok := version(data[i], data[i+1]); ok {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func version(a byte, b byte) (string, bool) {
|
||||
switch {
|
||||
case a == '#' && b == '#':
|
||||
return "V2016", true
|
||||
case a == '$' && b == '$':
|
||||
return "V2025", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func parseGBTime(data []byte) time.Time {
|
||||
if len(data) != 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
year := 2000 + int(data[0])
|
||||
month := time.Month(data[1])
|
||||
day := int(data[2])
|
||||
hour := int(data[3])
|
||||
minute := int(data[4])
|
||||
second := int(data[5])
|
||||
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func trimASCII(data []byte) string {
|
||||
return strings.TrimRight(string(data), "\x00 ")
|
||||
}
|
||||
|
||||
func bcc(data []byte) byte {
|
||||
var out byte
|
||||
for _, value := range data {
|
||||
out ^= value
|
||||
}
|
||||
return out
|
||||
}
|
||||
258
go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
Normal file
258
go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
Normal file
@@ -0,0 +1,258 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
|
||||
|
||||
func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
|
||||
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39})
|
||||
second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
|
||||
stream := append([]byte{0x00, 0x01}, first...)
|
||||
stream = append(stream, second[:len(second)-3]...)
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 {
|
||||
t.Fatalf("expected one complete frame, got %d", len(frames))
|
||||
}
|
||||
if string(remainder) != string(second[:len(second)-3]) {
|
||||
t.Fatalf("expected partial second frame as remainder, got %s", hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x02" {
|
||||
t.Fatalf("unexpected envelope header: %#v", env)
|
||||
}
|
||||
if env.VIN != "LNBSCB3D4R1234567" {
|
||||
t.Fatalf("unexpected vin: %q", env.VIN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameRejectsBadBCC(t *testing.T) {
|
||||
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
frame[len(frame)-1] ^= 0xff
|
||||
|
||||
if _, err := ParseFrame(frame, 1782745114999, ""); err == nil {
|
||||
t.Fatal("expected bad bcc error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFramesSupportsExtendedPlatformLogin(t *testing.T) {
|
||||
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x1f, 0x00, 0x01}
|
||||
body = append(body, fixedASCII("Hyundai", 12)...)
|
||||
body = append(body, fixedASCII("f2e3445d7cda409fb4f278f6fb890734", 32)...)
|
||||
body = append(body, 0x01)
|
||||
frame := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
|
||||
|
||||
frames, remainder, err := ExtractFrames(append(frame, []byte{0x23, 0x23, 0x05}...))
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 3 {
|
||||
t.Fatalf("unexpected split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
if len(frames[0]) != headerLen+53+1 {
|
||||
t.Fatalf("extended frame len = %d", len(frames[0]))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "8.134.95.166:39376")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x05" || env.ParseStatus != envelope.ParseOK {
|
||||
t.Fatalf("unexpected env: %#v", env)
|
||||
}
|
||||
header := env.Parsed["header"].(map[string]any)
|
||||
if header["body_length"] != 41 || header["actual_body_length"] != 53 {
|
||||
t.Fatalf("unexpected length header: %#v", header)
|
||||
}
|
||||
login := env.Parsed["platform_login"].(map[string]any)
|
||||
if login["username"] != "Hyundai" || login["password"] != "f2e3445d7cda409fb4f278f6fb890734" {
|
||||
t.Fatalf("unexpected platform login: %#v", login)
|
||||
}
|
||||
if login["login_time"] != "2026-07-02T00:38:31+08:00" {
|
||||
t.Fatalf("unexpected login time: %#v", login)
|
||||
}
|
||||
if env.Fields["platform_account"] != "Hyundai" {
|
||||
t.Fatalf("platform account not exposed: %#v", env.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponseEchoesRawVINAndOriginalTime(t *testing.T) {
|
||||
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x0f, 0x00, 0x01}
|
||||
body = append(body, fixedASCII("Hyundai", 12)...)
|
||||
body = append(body, fixedASCII("ack-test-password", 32)...)
|
||||
body = append(body, 0x01)
|
||||
request := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
|
||||
env, err := ParseFrame(request, 1782914969584, "8.134.95.166:39376")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := AutoResponse(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("AutoResponse() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
if response[2] != 0x05 || response[3] != 0x01 {
|
||||
t.Fatalf("unexpected response header: %x", response[:4])
|
||||
}
|
||||
if hex.EncodeToString(response[4:21]) != "0000000000000000000000000000000000" {
|
||||
t.Fatalf("raw vin not echoed: %x", response[4:21])
|
||||
}
|
||||
if hex.EncodeToString(response[24:30]) != "1a070200260f" {
|
||||
t.Fatalf("timestamp body not preserved: %x", response[24:30])
|
||||
}
|
||||
if got, want := bcc(response[2:len(response)-1]), response[len(response)-1]; got != want {
|
||||
t.Fatalf("response bcc got=0x%02x want=0x%02x", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
|
||||
body := []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39}
|
||||
body = append(body, 0x01)
|
||||
body = append(body,
|
||||
0x01, // vehicle status
|
||||
0x03, // charge status
|
||||
0x02, // running mode
|
||||
0x01, 0x2c, // speed: 30.0km/h
|
||||
0x00, 0x01, 0x86, 0xa0, // total mileage: 10000.0km
|
||||
0x15, 0xe5, // total voltage: 560.5V
|
||||
0x27, 0x10, // total current: 0A after -1000 offset
|
||||
85, 0x01, 0x00,
|
||||
0x27, 0x10,
|
||||
0x00, 0x00)
|
||||
body = append(body, 0x05)
|
||||
body = append(body,
|
||||
0x00,
|
||||
0x07, 0x36, 0x50, 0x40, // longitude: 121.000000
|
||||
0x01, 0xd2, 0x4f, 0x00) // latitude: 30.560000
|
||||
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", body)
|
||||
|
||||
env, err := ParseFrame(frame, 1782745114999, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 30.0)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10000.0)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121.0)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.56)
|
||||
if env.Fields[envelope.FieldSOCPercent] != 85 {
|
||||
t.Fatalf("unexpected soc: %#v", env.Fields[envelope.FieldSOCPercent])
|
||||
}
|
||||
if env.EventTimeMS == 1782745114999 {
|
||||
t.Fatal("event time should come from GB32960 device timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testing.T) {
|
||||
frame, err := hex.DecodeString(realFuelCellFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frame, 1782914969584, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LB9A32A21R0LS1707" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 53490.9)
|
||||
assertFloatField(t, env, "fuel_cell_hydrogen_consumption_kg_per_100km", 1.8)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 120.800326)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 31.634907)
|
||||
assertFloatField(t, env, "gd_fc_stack_hydrogen_inlet_pressure_kpa", 97.0)
|
||||
assertFloatField(t, env, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
|
||||
assertIntField(t, env, "gd_fc_demo_stack_temp_c", 65)
|
||||
|
||||
units, ok := env.Parsed["data_units"].([]map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("data_units missing: %#v", env.Parsed["data_units"])
|
||||
}
|
||||
gotTypes := make([]string, 0, len(units))
|
||||
for _, unit := range units {
|
||||
gotTypes = append(gotTypes, fmt.Sprint(unit["type"]))
|
||||
}
|
||||
wantTypes := []string{"0x01", "0x02", "0x03", "0x04", "0x05", "0x06", "0x07", "0x08", "0x09", "0x30", "0x31", "0x32", "0x33", "0x34", "0x80", "0x83"}
|
||||
if fmt.Sprint(gotTypes) != fmt.Sprint(wantTypes) {
|
||||
t.Fatalf("unit types = %v, want %v", gotTypes, wantTypes)
|
||||
}
|
||||
if units[1]["name"] != "drive_motor" {
|
||||
t.Fatalf("unexpected motor unit: %#v", units[1])
|
||||
}
|
||||
if units[2]["name"] != "fuel_cell" {
|
||||
t.Fatalf("unexpected fuel cell unit: %#v", units[2])
|
||||
}
|
||||
if units[9]["name"] != "gd_fc_stack" {
|
||||
t.Fatalf("unexpected gd stack unit: %#v", units[9])
|
||||
}
|
||||
if units[10]["name"] != "gd_fc_auxiliary" {
|
||||
t.Fatalf("unexpected gd auxiliary unit: %#v", units[10])
|
||||
}
|
||||
if units[15]["name"] != "gd_fc_vendor_tlv" {
|
||||
t.Fatalf("unexpected gd vendor tlv unit: %#v", units[15])
|
||||
}
|
||||
tlv := units[15]["value"].(map[string]any)
|
||||
if tlv["declared_length"] != 36 {
|
||||
t.Fatalf("unexpected vendor tlv: %#v", tlv)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if math.Abs(got-want) > 0.000001 {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntField(t *testing.T, env envelope.FrameEnvelope, key string, want int) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(int)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not int: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func buildFrame(command byte, response byte, vin string, body []byte) []byte {
|
||||
return buildFrameWithDeclaredLength(command, response, vin, body, len(body))
|
||||
}
|
||||
|
||||
func buildFrameWithDeclaredLength(command byte, response byte, vin string, body []byte, declaredBodyLen int) []byte {
|
||||
frame := []byte{'#', '#', command, response}
|
||||
vinBytes := []byte(vin)
|
||||
if len(vinBytes) < 17 {
|
||||
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
|
||||
}
|
||||
frame = append(frame, vinBytes[:17]...)
|
||||
frame = append(frame, 0x01, byte(declaredBodyLen>>8), byte(declaredBodyLen))
|
||||
frame = append(frame, body...)
|
||||
return append(frame, bcc(frame[2:]))
|
||||
}
|
||||
|
||||
func fixedASCII(value string, size int) []byte {
|
||||
out := make([]byte, size)
|
||||
copy(out, []byte(value))
|
||||
return out
|
||||
}
|
||||
80
go/vehicle-gateway/internal/protocol/gb32960/response.go
Normal file
80
go/vehicle-gateway/internal/protocol/gb32960/response.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrResponseFrameTooShort = errors.New("gb32960 response frame too short")
|
||||
|
||||
const (
|
||||
responseSuccess = byte(0x01)
|
||||
responseCommand = byte(0xfe)
|
||||
encryptNone = byte(0x01)
|
||||
)
|
||||
|
||||
// AutoResponse builds the protocol ACK for accepted upstream GB/T 32960 frames.
|
||||
// The ACK is written only after the caller has durably published the frame.
|
||||
func AutoResponse(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return nil, false, nil
|
||||
}
|
||||
if len(raw) < headerLen+1 {
|
||||
return nil, false, ErrResponseFrameTooShort
|
||||
}
|
||||
if raw[3] != responseCommand {
|
||||
return nil, false, nil
|
||||
}
|
||||
command := raw[2]
|
||||
if !shouldRespond(command) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
body := []byte(nil)
|
||||
if timestamp := responseTime(raw, command); !timestamp.IsZero() {
|
||||
body = encodeGBTime(timestamp)
|
||||
}
|
||||
return buildResponse(raw[0], command, responseSuccess, raw[4:21], body), true, nil
|
||||
}
|
||||
|
||||
func shouldRespond(command byte) bool {
|
||||
switch command {
|
||||
case 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func responseTime(raw []byte, command byte) time.Time {
|
||||
bodyLen := len(raw) - headerLen - 1
|
||||
if bodyLen >= 6 {
|
||||
return parseGBTime(raw[headerLen : headerLen+6])
|
||||
}
|
||||
return time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func encodeGBTime(t time.Time) []byte {
|
||||
local := t.In(time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
return []byte{
|
||||
byte(local.Year() - 2000),
|
||||
byte(local.Month()),
|
||||
byte(local.Day()),
|
||||
byte(local.Hour()),
|
||||
byte(local.Minute()),
|
||||
byte(local.Second()),
|
||||
}
|
||||
}
|
||||
|
||||
func buildResponse(start byte, command byte, responseFlag byte, vin17 []byte, body []byte) []byte {
|
||||
out := make([]byte, 0, headerLen+len(body)+1)
|
||||
out = append(out, start, start, command, responseFlag)
|
||||
out = append(out, vin17[:17]...)
|
||||
out = append(out, encryptNone, 0, 0)
|
||||
binary.BigEndian.PutUint16(out[22:24], uint16(len(body)))
|
||||
out = append(out, body...)
|
||||
return append(out, bcc(out[2:]))
|
||||
}
|
||||
600
go/vehicle-gateway/internal/protocol/jt808/parser.go
Normal file
600
go/vehicle-gateway/internal/protocol/jt808/parser.go
Normal file
@@ -0,0 +1,600 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFrameTooShort = errors.New("jt808 frame too short")
|
||||
ErrChecksum = errors.New("jt808 checksum mismatch")
|
||||
ErrEscape = errors.New("jt808 invalid escape sequence")
|
||||
)
|
||||
|
||||
type Location struct {
|
||||
AlarmFlag uint32
|
||||
StatusFlag uint32
|
||||
Latitude float64
|
||||
Longitude float64
|
||||
AltitudeM uint16
|
||||
SpeedKMH float64
|
||||
DirectionDeg uint16
|
||||
Time time.Time
|
||||
TotalMileageKM *float64
|
||||
Additional []AdditionalItem
|
||||
}
|
||||
|
||||
type AdditionalItem struct {
|
||||
ID byte
|
||||
Length int
|
||||
ValueHex string
|
||||
Parsed any
|
||||
}
|
||||
|
||||
// ExtractFrames splits JT/T 808 TCP streams by 0x7e delimiters and unescapes
|
||||
// complete frame payloads. Returned frames exclude the leading and trailing 0x7e.
|
||||
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
|
||||
start := -1
|
||||
for i, value := range stream {
|
||||
if value != 0x7e {
|
||||
continue
|
||||
}
|
||||
if start < 0 {
|
||||
start = i
|
||||
continue
|
||||
}
|
||||
if i == start+1 {
|
||||
start = i
|
||||
continue
|
||||
}
|
||||
frame, err := unescape(stream[start+1 : i])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
frames = append(frames, frame)
|
||||
start = i
|
||||
}
|
||||
if start >= 0 && start < len(stream)-1 {
|
||||
return frames, append([]byte(nil), stream[start:]...), nil
|
||||
}
|
||||
return frames, nil, nil
|
||||
}
|
||||
|
||||
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||
if len(raw) < 13 {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
if got, want := checksum(raw[:len(raw)-1]), raw[len(raw)-1]; got != want {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrChecksum, got, want)
|
||||
}
|
||||
|
||||
messageID := binary.BigEndian.Uint16(raw[0:2])
|
||||
props := binary.BigEndian.Uint16(raw[2:4])
|
||||
bodySize := props & 0x03ff
|
||||
versioned := props&0x4000 != 0
|
||||
headerLen := 12
|
||||
phoneStart := 4
|
||||
phoneEnd := 10
|
||||
serialStart := 10
|
||||
packageInfo := map[string]any(nil)
|
||||
protocolVersion := byte(0)
|
||||
if versioned {
|
||||
headerLen = 17
|
||||
if len(raw) < headerLen {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: versioned header len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
protocolVersion = raw[4]
|
||||
phoneStart = 5
|
||||
phoneEnd = 15
|
||||
serialStart = 15
|
||||
}
|
||||
subpackage := props&0x2000 != 0
|
||||
if subpackage {
|
||||
headerLen += 4
|
||||
}
|
||||
if len(raw) < headerLen+int(bodySize)+1 {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
|
||||
}
|
||||
if subpackage {
|
||||
packageInfo = map[string]any{
|
||||
"total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]),
|
||||
"index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]),
|
||||
}
|
||||
}
|
||||
|
||||
phoneBCD := raw[phoneStart:phoneEnd]
|
||||
phone := bcdString(phoneBCD)
|
||||
phoneTrimZero := strings.TrimLeft(phone, "0")
|
||||
if versioned && phoneTrimZero != "" {
|
||||
phone = phoneTrimZero
|
||||
}
|
||||
body := raw[headerLen : headerLen+int(bodySize)]
|
||||
parsed := map[string]any{
|
||||
"header": map[string]any{
|
||||
"message_id": fmt.Sprintf("0x%04X", messageID),
|
||||
"protocol_version": protocolVersion,
|
||||
"versioned": versioned,
|
||||
"properties": props,
|
||||
"body_size": bodySize,
|
||||
"phone_bcd_hex": strings.ToUpper(hex.EncodeToString(phoneBCD)),
|
||||
"phone": phone,
|
||||
"phone_trim_zero": phoneTrimZero,
|
||||
"sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
|
||||
"subpackage": subpackage,
|
||||
"package_info": packageInfo,
|
||||
"encryption_flags": (props >> 10) & 0x07,
|
||||
},
|
||||
}
|
||||
fields := map[string]any{}
|
||||
eventTimeMS := receivedAtMS
|
||||
|
||||
if messageID == 0x0200 {
|
||||
location, err := parseLocation(body)
|
||||
if err != nil {
|
||||
return envelope.FrameEnvelope{}, err
|
||||
}
|
||||
parsed["location"] = locationToMap(location)
|
||||
fields[envelope.FieldLatitude] = location.Latitude
|
||||
fields[envelope.FieldLongitude] = location.Longitude
|
||||
fields[envelope.FieldSpeedKMH] = location.SpeedKMH
|
||||
fields["altitude_m"] = location.AltitudeM
|
||||
fields["direction_deg"] = location.DirectionDeg
|
||||
fields["alarm_flag"] = location.AlarmFlag
|
||||
fields["status_flag"] = location.StatusFlag
|
||||
if !location.Time.IsZero() {
|
||||
eventTimeMS = location.Time.UnixMilli()
|
||||
fields["device_time"] = location.Time.Format(time.RFC3339)
|
||||
}
|
||||
if location.TotalMileageKM != nil {
|
||||
fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
|
||||
}
|
||||
} else if messageID == 0x0100 {
|
||||
registration, err := parseRegistration(body, versioned)
|
||||
if err != nil {
|
||||
return envelope.FrameEnvelope{}, err
|
||||
}
|
||||
parsed["registration"] = registration
|
||||
if deviceID, ok := registration["device_id"].(string); ok {
|
||||
fields["device_id"] = deviceID
|
||||
}
|
||||
if plate, ok := registration["plate"].(string); ok {
|
||||
fields["plate"] = plate
|
||||
}
|
||||
} else if messageID == 0x0102 {
|
||||
authentication := parseAuthentication(body, versioned)
|
||||
parsed["authentication"] = authentication
|
||||
if token, ok := authentication["token"].(string); ok {
|
||||
fields["auth_token"] = token
|
||||
}
|
||||
}
|
||||
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: fmt.Sprintf("0x%04X", messageID),
|
||||
Sequence: binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
|
||||
Phone: phone,
|
||||
SourceEndpoint: sourceEndpoint,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
if registration, ok := parsed["registration"].(map[string]any); ok {
|
||||
env.DeviceID, _ = registration["device_id"].(string)
|
||||
env.Plate, _ = registration["plate"].(string)
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func parseRegistration(body []byte, versioned bool) (map[string]any, error) {
|
||||
schemaVersion := "2011_2013"
|
||||
manufacturerSize := 5
|
||||
deviceTypeSize := 20
|
||||
deviceIDSize := 7
|
||||
if versioned {
|
||||
schemaVersion = "2019"
|
||||
manufacturerSize = 11
|
||||
deviceTypeSize = 30
|
||||
deviceIDSize = 30
|
||||
}
|
||||
minLen := 2 + 2 + manufacturerSize + deviceTypeSize + deviceIDSize + 1
|
||||
if len(body) < minLen {
|
||||
return nil, fmt.Errorf("%w: registration body len=%d", ErrFrameTooShort, len(body))
|
||||
}
|
||||
manufacturerStart := 4
|
||||
deviceTypeStart := manufacturerStart + manufacturerSize
|
||||
deviceIDStart := deviceTypeStart + deviceTypeSize
|
||||
plateColorStart := deviceIDStart + deviceIDSize
|
||||
registration := map[string]any{
|
||||
"schema_version": schemaVersion,
|
||||
"province": binary.BigEndian.Uint16(body[0:2]),
|
||||
"city": binary.BigEndian.Uint16(body[2:4]),
|
||||
"manufacturer": fixedText(body[manufacturerStart:deviceTypeStart]),
|
||||
"device_type": fixedText(body[deviceTypeStart:deviceIDStart]),
|
||||
"device_id": fixedText(body[deviceIDStart:plateColorStart]),
|
||||
"plate_color": body[plateColorStart],
|
||||
"plate": freeText(body[plateColorStart+1:]),
|
||||
}
|
||||
return registration, nil
|
||||
}
|
||||
|
||||
func parseAuthentication(body []byte, versioned bool) map[string]any {
|
||||
if versioned {
|
||||
if out, ok := parseVersionedAuthentication(body); ok {
|
||||
return out
|
||||
}
|
||||
if out, ok := parseDelimitedVersionedAuthentication(body); ok {
|
||||
return out
|
||||
}
|
||||
}
|
||||
return map[string]any{"token": freeText(body)}
|
||||
}
|
||||
|
||||
func parseVersionedAuthentication(body []byte) (map[string]any, bool) {
|
||||
if len(body) < 17 {
|
||||
return nil, false
|
||||
}
|
||||
tokenLen := int(body[0])
|
||||
imeiStart := 1 + tokenLen
|
||||
softwareLenStart := imeiStart + 15
|
||||
if tokenLen <= 0 || len(body) < softwareLenStart+1 {
|
||||
return nil, false
|
||||
}
|
||||
softwareLen := int(body[softwareLenStart])
|
||||
softwareStart := softwareLenStart + 1
|
||||
if len(body) < softwareStart+softwareLen {
|
||||
return nil, false
|
||||
}
|
||||
return map[string]any{
|
||||
"schema_version": "2019",
|
||||
"token": freeText(body[1:imeiStart]),
|
||||
"imei": fixedText(body[imeiStart:softwareLenStart]),
|
||||
"software_version": freeText(body[softwareStart : softwareStart+softwareLen]),
|
||||
}, true
|
||||
}
|
||||
|
||||
func parseDelimitedVersionedAuthentication(body []byte) (map[string]any, bool) {
|
||||
parts := bytes.Split(body, []byte{0xff, 0xff})
|
||||
if len(parts) < 2 {
|
||||
return nil, false
|
||||
}
|
||||
out := map[string]any{
|
||||
"schema_version": "2019_delimited",
|
||||
"token": freeText(parts[0]),
|
||||
"imei": freeText(parts[1]),
|
||||
}
|
||||
if len(parts) > 2 {
|
||||
out["software_version"] = freeText(parts[2])
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
func parseLocation(body []byte) (Location, error) {
|
||||
if len(body) < 28 {
|
||||
return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
|
||||
}
|
||||
status := binary.BigEndian.Uint32(body[4:8])
|
||||
latitude := float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000
|
||||
longitude := float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000
|
||||
if status&(1<<2) != 0 {
|
||||
latitude = -latitude
|
||||
}
|
||||
if status&(1<<3) != 0 {
|
||||
longitude = -longitude
|
||||
}
|
||||
location := Location{
|
||||
AlarmFlag: binary.BigEndian.Uint32(body[0:4]),
|
||||
StatusFlag: status,
|
||||
Latitude: latitude,
|
||||
Longitude: longitude,
|
||||
AltitudeM: binary.BigEndian.Uint16(body[16:18]),
|
||||
SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10,
|
||||
DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
|
||||
Time: parseBCDTime(body[22:28]),
|
||||
}
|
||||
location.Additional = parseAdditional(body[28:], &location)
|
||||
return location, nil
|
||||
}
|
||||
|
||||
func parseAdditional(data []byte, location *Location) []AdditionalItem {
|
||||
var out []AdditionalItem
|
||||
for len(data) >= 2 {
|
||||
id := data[0]
|
||||
size := int(data[1])
|
||||
data = data[2:]
|
||||
if len(data) < size {
|
||||
out = append(out, AdditionalItem{
|
||||
ID: id,
|
||||
Length: size,
|
||||
ValueHex: strings.ToUpper(hex.EncodeToString(data)),
|
||||
Parsed: "truncated",
|
||||
})
|
||||
return out
|
||||
}
|
||||
value := data[:size]
|
||||
item := AdditionalItem{
|
||||
ID: id,
|
||||
Length: size,
|
||||
ValueHex: strings.ToUpper(hex.EncodeToString(value)),
|
||||
}
|
||||
switch {
|
||||
case id == 0x01 && size == 4:
|
||||
rawMileage := binary.BigEndian.Uint32(value)
|
||||
mileage := float64(rawMileage) / 10
|
||||
location.TotalMileageKM = &mileage
|
||||
item.Parsed = map[string]any{
|
||||
"name": envelope.FieldTotalMileageKM,
|
||||
"raw_value_tenth_km": rawMileage,
|
||||
"value": mileage,
|
||||
"unit": "km",
|
||||
}
|
||||
case id == 0x02 && size == 2:
|
||||
rawFuel := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "fuel_l",
|
||||
"raw_value_tenth_l": rawFuel,
|
||||
"value": float64(rawFuel) / 10,
|
||||
"unit": "L",
|
||||
}
|
||||
case id == 0x03 && size == 2:
|
||||
rawSpeed := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "recorder_speed_kmh",
|
||||
"raw_value_tenth_kmh": rawSpeed,
|
||||
"value": float64(rawSpeed) / 10,
|
||||
"unit": "km/h",
|
||||
}
|
||||
case id == 0x04 && size == 2:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "manual_alarm_event_id",
|
||||
"value": binary.BigEndian.Uint16(value),
|
||||
}
|
||||
case id == 0x05 && size == 30:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "tire_pressure",
|
||||
"values": parseTirePressure(value),
|
||||
}
|
||||
case id == 0x06 && size == 2:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "carriage_temperature_c",
|
||||
"value": int16(binary.BigEndian.Uint16(value)),
|
||||
"unit": "C",
|
||||
}
|
||||
case id == 0x11 && (size == 1 || size == 5):
|
||||
parsed := map[string]any{
|
||||
"name": "overspeed_alarm",
|
||||
"location_type": value[0],
|
||||
}
|
||||
if size == 5 {
|
||||
parsed["area_id"] = binary.BigEndian.Uint32(value[1:5])
|
||||
}
|
||||
item.Parsed = parsed
|
||||
case id == 0x12 && size == 6:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "area_route_alarm",
|
||||
"location_type": value[0],
|
||||
"area_id": binary.BigEndian.Uint32(value[1:5]),
|
||||
"direction": value[5],
|
||||
}
|
||||
case id == 0x13 && size == 7:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "road_section_driving_time_alarm",
|
||||
"road_section_id": binary.BigEndian.Uint32(value[0:4]),
|
||||
"driving_time_seconds": binary.BigEndian.Uint16(value[4:6]),
|
||||
"result": value[6],
|
||||
"result_description": roadSectionDrivingTimeResult(value[6]),
|
||||
"driving_time_result_source": "jt808_0x0200_additional_0x13",
|
||||
}
|
||||
case id == 0x25 && size == 4:
|
||||
rawStatus := binary.BigEndian.Uint32(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "extended_vehicle_signal_status",
|
||||
"value": rawStatus,
|
||||
"bits": parseExtendedVehicleSignal(rawStatus),
|
||||
}
|
||||
case id == 0x2a && size == 2:
|
||||
rawStatus := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "io_status",
|
||||
"value": rawStatus,
|
||||
"bits": parseIOStatus(rawStatus),
|
||||
}
|
||||
case id == 0x2b && size == 4:
|
||||
analog := binary.BigEndian.Uint32(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "analog",
|
||||
"ad0": analog & 0xffff,
|
||||
"ad1": analog >> 16,
|
||||
"value": analog,
|
||||
}
|
||||
case id == 0x30 && size == 1:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "network_signal_strength",
|
||||
"value": value[0],
|
||||
}
|
||||
case id == 0x31 && size == 1:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "gnss_satellite_count",
|
||||
"value": value[0],
|
||||
}
|
||||
}
|
||||
out = append(out, item)
|
||||
data = data[size:]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func locationToMap(location Location) map[string]any {
|
||||
out := map[string]any{
|
||||
"alarm_flag": location.AlarmFlag,
|
||||
"status_flag": location.StatusFlag,
|
||||
"latitude": location.Latitude,
|
||||
"longitude": location.Longitude,
|
||||
"altitude_m": location.AltitudeM,
|
||||
"speed_kmh": location.SpeedKMH,
|
||||
"direction_deg": location.DirectionDeg,
|
||||
"additional": additionalToMaps(location.Additional),
|
||||
}
|
||||
if !location.Time.IsZero() {
|
||||
out["device_time"] = location.Time.Format(time.RFC3339)
|
||||
}
|
||||
if location.TotalMileageKM != nil {
|
||||
out[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func additionalToMaps(items []AdditionalItem) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(items))
|
||||
for _, item := range items {
|
||||
value := map[string]any{
|
||||
"id": fmt.Sprintf("0x%02X", item.ID),
|
||||
"length": item.Length,
|
||||
"value_hex": item.ValueHex,
|
||||
}
|
||||
if item.Parsed != nil {
|
||||
value["parsed"] = item.Parsed
|
||||
}
|
||||
out = append(out, value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseTirePressure(data []byte) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(data))
|
||||
for index, value := range data {
|
||||
out = append(out, map[string]any{
|
||||
"position": index,
|
||||
"value": value,
|
||||
"valid": value != 0xff,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseExtendedVehicleSignal(value uint32) map[string]bool {
|
||||
return map[string]bool{
|
||||
"low_beam": value&0x0001 != 0,
|
||||
"high_beam": value&0x0002 != 0,
|
||||
"right_turn": value&0x0004 != 0,
|
||||
"left_turn": value&0x0008 != 0,
|
||||
"brake": value&0x0010 != 0,
|
||||
"reverse_gear": value&0x0020 != 0,
|
||||
"fog_light": value&0x0040 != 0,
|
||||
"clearance_light": value&0x0080 != 0,
|
||||
"horn": value&0x0100 != 0,
|
||||
"air_conditioner": value&0x0200 != 0,
|
||||
"neutral": value&0x0400 != 0,
|
||||
"retarder": value&0x0800 != 0,
|
||||
"abs": value&0x1000 != 0,
|
||||
"heater": value&0x2000 != 0,
|
||||
"clutch": value&0x4000 != 0,
|
||||
}
|
||||
}
|
||||
|
||||
func parseIOStatus(value uint16) map[string]bool {
|
||||
return map[string]bool{
|
||||
"deep_sleep": value&0x0001 != 0,
|
||||
"sleep": value&0x0002 != 0,
|
||||
}
|
||||
}
|
||||
|
||||
func roadSectionDrivingTimeResult(value byte) string {
|
||||
if value == 0 {
|
||||
return "insufficient"
|
||||
}
|
||||
if value == 1 {
|
||||
return "too_long"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func bcdString(data []byte) string {
|
||||
out := make([]byte, 0, len(data)*2)
|
||||
for _, value := range data {
|
||||
out = append(out, '0'+((value>>4)&0x0f), '0'+(value&0x0f))
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func fixedText(data []byte) string {
|
||||
return strings.Trim(freeText(data), "\x00 ")
|
||||
}
|
||||
|
||||
func freeText(data []byte) string {
|
||||
data = bytes.Trim(data, "\x00 ")
|
||||
if len(data) == 0 {
|
||||
return ""
|
||||
}
|
||||
if utf8.Valid(data) {
|
||||
return string(data)
|
||||
}
|
||||
decoded, err := simplifiedchinese.GBK.NewDecoder().Bytes(data)
|
||||
if err == nil && utf8.Valid(decoded) {
|
||||
return string(decoded)
|
||||
}
|
||||
return strings.ToValidUTF8(string(data), "")
|
||||
}
|
||||
|
||||
func parseBCDTime(data []byte) time.Time {
|
||||
if len(data) != 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
year := 2000 + bcdByte(data[0])
|
||||
month := time.Month(bcdByte(data[1]))
|
||||
day := bcdByte(data[2])
|
||||
hour := bcdByte(data[3])
|
||||
minute := bcdByte(data[4])
|
||||
second := bcdByte(data[5])
|
||||
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func bcdByte(value byte) int {
|
||||
return int(value>>4)*10 + int(value&0x0f)
|
||||
}
|
||||
|
||||
func unescape(data []byte) ([]byte, error) {
|
||||
out := make([]byte, 0, len(data))
|
||||
for i := 0; i < len(data); i++ {
|
||||
if data[i] != 0x7d {
|
||||
out = append(out, data[i])
|
||||
continue
|
||||
}
|
||||
if i+1 >= len(data) {
|
||||
return nil, ErrEscape
|
||||
}
|
||||
i++
|
||||
switch data[i] {
|
||||
case 0x01:
|
||||
out = append(out, 0x7d)
|
||||
case 0x02:
|
||||
out = append(out, 0x7e)
|
||||
default:
|
||||
return nil, ErrEscape
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func checksum(data []byte) byte {
|
||||
var out byte
|
||||
for _, value := range data {
|
||||
out ^= value
|
||||
}
|
||||
return out
|
||||
}
|
||||
503
go/vehicle-gateway/internal/protocol/jt808/parser_test.go
Normal file
503
go/vehicle-gateway/internal/protocol/jt808/parser_test.go
Normal file
@@ -0,0 +1,503 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
|
||||
const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E"
|
||||
const fullAdditionalLocationFrame = "7e020000800123456789017fff000004000000080006eeb6ad02633df701380003006320070719235901040000000b02020016030200210402002c051e3737370000000000000000000000000000000000000000000000000000001105420000004212064d0000004d4d1307000000580058582504000000632a02000a2b040000001430011e31012806020001927e"
|
||||
|
||||
func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
|
||||
stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 {
|
||||
t.Fatalf("expected one complete frame, got %d", len(frames))
|
||||
}
|
||||
if hex.EncodeToString(remainder) != "7e02000000" {
|
||||
t.Fatalf("expected partial remainder, got %s", hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolJT808 {
|
||||
t.Fatalf("protocol = %q", env.Protocol)
|
||||
}
|
||||
if env.MessageID != "0x0200" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "013307795425" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.Sequence != 1 {
|
||||
t.Fatalf("sequence = %d", env.Sequence)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.590151)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121.069881)
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 23.0)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10241.2)
|
||||
if env.Fields["direction_deg"] != uint16(79) {
|
||||
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
|
||||
}
|
||||
|
||||
location, ok := env.Parsed["location"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("parsed location missing: %#v", env.Parsed)
|
||||
}
|
||||
additional, ok := location["additional"].([]map[string]any)
|
||||
if !ok || len(additional) == 0 {
|
||||
t.Fatalf("additional fields missing: %#v", location["additional"])
|
||||
}
|
||||
if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
|
||||
t.Fatalf("unexpected first additional item: %#v", additional[0])
|
||||
}
|
||||
parsed, ok := additional[0]["parsed"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("mileage parsed field missing: %#v", additional[0])
|
||||
}
|
||||
if parsed["raw_value_tenth_km"] != uint32(102412) || parsed["value"] != 10241.2 {
|
||||
t.Fatalf("unexpected mileage parsed value: %#v", parsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
|
||||
stream, err := hex.DecodeString(real2019LocationFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782914967713, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0200" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "14894135060" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.Sequence != 1995 {
|
||||
t.Fatalf("sequence = %d", env.Sequence)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.570155)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 117.178187)
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 33.4)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 7868.9)
|
||||
if env.Fields["altitude_m"] != uint16(13) {
|
||||
t.Fatalf("altitude = %#v", env.Fields["altitude_m"])
|
||||
}
|
||||
if env.Fields["direction_deg"] != uint16(232) {
|
||||
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
|
||||
}
|
||||
|
||||
header, ok := env.Parsed["header"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("parsed header missing: %#v", env.Parsed)
|
||||
}
|
||||
if header["versioned"] != true || header["protocol_version"] != byte(1) {
|
||||
t.Fatalf("unexpected versioned header: %#v", header)
|
||||
}
|
||||
if header["phone_bcd_hex"] != "00000000014894135060" {
|
||||
t.Fatalf("phone bcd = %#v", header["phone_bcd_hex"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesCommonLocationAdditionalItems(t *testing.T) {
|
||||
stream, err := hex.DecodeString(fullAdditionalLocationFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782914967713, "127.0.0.1:808")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Phone != "012345678901" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.1)
|
||||
|
||||
location := env.Parsed["location"].(map[string]any)
|
||||
additional := location["additional"].([]map[string]any)
|
||||
byID := map[string]map[string]any{}
|
||||
for _, item := range additional {
|
||||
byID[item["id"].(string)] = item["parsed"].(map[string]any)
|
||||
}
|
||||
if byID["0x01"]["raw_value_tenth_km"] != uint32(11) || byID["0x01"]["value"] != 1.1 {
|
||||
t.Fatalf("mileage parsed = %#v", byID["0x01"])
|
||||
}
|
||||
if byID["0x05"]["name"] != "tire_pressure" {
|
||||
t.Fatalf("tire pressure missing: %#v", byID["0x05"])
|
||||
}
|
||||
tireValues := byID["0x05"]["values"].([]map[string]any)
|
||||
if tireValues[0]["value"] != byte(0x37) || tireValues[0]["valid"] != true {
|
||||
t.Fatalf("tire pressure first value = %#v", tireValues[0])
|
||||
}
|
||||
if byID["0x11"]["location_type"] != byte(0x42) || byID["0x11"]["area_id"] != uint32(0x42) {
|
||||
t.Fatalf("overspeed parsed = %#v", byID["0x11"])
|
||||
}
|
||||
if byID["0x12"]["location_type"] != byte(0x4d) || byID["0x12"]["area_id"] != uint32(0x4d) || byID["0x12"]["direction"] != byte(0x4d) {
|
||||
t.Fatalf("area route parsed = %#v", byID["0x12"])
|
||||
}
|
||||
if byID["0x13"]["road_section_id"] != uint32(0x58) || byID["0x13"]["driving_time_seconds"] != uint16(0x58) {
|
||||
t.Fatalf("road section parsed = %#v", byID["0x13"])
|
||||
}
|
||||
ioBits := byID["0x2A"]["bits"].(map[string]bool)
|
||||
if ioBits["deep_sleep"] || !ioBits["sleep"] {
|
||||
t.Fatalf("io bits = %#v", ioBits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameRejectsTruncatedSubpackageHeader(t *testing.T) {
|
||||
payload := []byte{0x02, 0x00, 0x20, 0x00}
|
||||
payload = append(payload, encodeBCD("013307795425", 6)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, 1)
|
||||
payload = append(payload, checksum(payload))
|
||||
|
||||
_, err := ParseFrame(payload, 1782914967713, "115.231.168.135:43625")
|
||||
if err == nil {
|
||||
t.Fatal("expected truncated subpackage header to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLocationAppliesSouthWestStatusBits(t *testing.T) {
|
||||
body := make([]byte, 28)
|
||||
binary.BigEndian.PutUint32(body[4:8], (1<<2)|(1<<3))
|
||||
binary.BigEndian.PutUint32(body[8:12], 12345678)
|
||||
binary.BigEndian.PutUint32(body[12:16], 98765432)
|
||||
|
||||
location, err := parseLocation(body)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLocation() error = %v", err)
|
||||
}
|
||||
if location.Latitude != -12.345678 {
|
||||
t.Fatalf("latitude = %v", location.Latitude)
|
||||
}
|
||||
if location.Longitude != -98.765432 {
|
||||
t.Fatalf("longitude = %v", location.Longitude)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0001", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST123")...)
|
||||
|
||||
env, err := ParseFrame(buildFrame(0x0100, "013079963379", 7, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0100" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "013079963379" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.DeviceID != "DEV0001" {
|
||||
t.Fatalf("device id = %q", env.DeviceID)
|
||||
}
|
||||
if env.Plate != "TEST123" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
registration, ok := env.Parsed["registration"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("registration missing: %#v", env.Parsed)
|
||||
}
|
||||
if registration["manufacturer"] != "YUTNG" || registration["device_type"] != "ZK6105CHEVNPG4" {
|
||||
t.Fatalf("registration fields = %#v", registration)
|
||||
}
|
||||
if registration["plate_color"] != uint8(2) {
|
||||
t.Fatalf("plate color = %#v", registration["plate_color"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesJT8082019RegistrationIdentity(t *testing.T) {
|
||||
body := make([]byte, 0, 84)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTONG00001", 11)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4-2019", 30)
|
||||
body = appendFixedASCII(body, "DEV201900000000000000000000001", 30)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST2019")...)
|
||||
|
||||
env, err := ParseFrame(buildVersionedFrame(0x0100, "14894135060", 11, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Phone != "14894135060" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.DeviceID != "DEV201900000000000000000000001" {
|
||||
t.Fatalf("device id = %q", env.DeviceID)
|
||||
}
|
||||
if env.Plate != "TEST2019" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
registration, ok := env.Parsed["registration"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("registration missing: %#v", env.Parsed)
|
||||
}
|
||||
if registration["schema_version"] != "2019" || registration["manufacturer"] != "YUTONG00001" {
|
||||
t.Fatalf("registration fields = %#v", registration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0002", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte{0xBB, 0xA6, 'A', '5', '3', '3', '0', '1'}...)
|
||||
|
||||
env, err := ParseFrame(buildFrame(0x0100, "013079963380", 9, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Plate != "沪A53301" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesJT8082019Authentication(t *testing.T) {
|
||||
body := []byte{5}
|
||||
body = append(body, []byte("TOKEN")...)
|
||||
body = appendFixedASCII(body, "123456789012345", 15)
|
||||
body = append(body, 4)
|
||||
body = append(body, []byte("v1.2")...)
|
||||
|
||||
env, err := ParseFrame(buildVersionedFrame(0x0102, "14894135060", 12, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
auth, ok := env.Parsed["authentication"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("authentication missing: %#v", env.Parsed)
|
||||
}
|
||||
if auth["token"] != "TOKEN" || auth["imei"] != "123456789012345" || auth["software_version"] != "v1.2" {
|
||||
t.Fatalf("authentication = %#v", auth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesAuthenticationToken(t *testing.T) {
|
||||
env, err := ParseFrame(buildFrame(0x0102, "064646848757", 8, []byte("g7gps")), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0102" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
auth, ok := env.Parsed["authentication"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("authentication missing: %#v", env.Parsed)
|
||||
}
|
||||
if auth["token"] != "g7gps" {
|
||||
t.Fatalf("token = %#v", auth["token"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponderBuildsRegisterAck(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0001", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST123")...)
|
||||
request := buildFrame(0x0100, "013079963379", 9, body)
|
||||
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := NewAutoResponder("g7gps").Respond(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("Respond() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(response)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
payload := frames[0]
|
||||
if binary.BigEndian.Uint16(payload[0:2]) != 0x8100 {
|
||||
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
|
||||
}
|
||||
if hex.EncodeToString(payload[4:10]) != "013079963379" {
|
||||
t.Fatalf("phone bcd = %x", payload[4:10])
|
||||
}
|
||||
if binary.BigEndian.Uint16(payload[12:14]) != 9 || payload[14] != 0 || string(payload[15:20]) != "g7gps" {
|
||||
t.Fatalf("unexpected register ack body: %x", payload[12:len(payload)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponderBuildsVersionedGeneralAck(t *testing.T) {
|
||||
request := buildVersionedFrame(0x0200, "14894135060", 12, make([]byte, 28))
|
||||
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := NewAutoResponder("").Respond(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("Respond() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(response)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
payload := frames[0]
|
||||
if binary.BigEndian.Uint16(payload[0:2]) != 0x8001 {
|
||||
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
|
||||
}
|
||||
if binary.BigEndian.Uint16(payload[2:4])&0x4000 == 0 || payload[4] != 1 {
|
||||
t.Fatalf("versioned header not preserved: %x", payload[:5])
|
||||
}
|
||||
if hex.EncodeToString(payload[5:15]) != "00000000014894135060" {
|
||||
t.Fatalf("phone bcd = %x", payload[5:15])
|
||||
}
|
||||
bodyStart := 17
|
||||
if binary.BigEndian.Uint16(payload[bodyStart:bodyStart+2]) != 12 ||
|
||||
binary.BigEndian.Uint16(payload[bodyStart+2:bodyStart+4]) != 0x0200 ||
|
||||
payload[bodyStart+4] != 0 {
|
||||
t.Fatalf("unexpected general ack body: %x", payload[bodyStart:len(payload)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
|
||||
payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
|
||||
frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
|
||||
frame = append(frame, 0x7e)
|
||||
|
||||
frames, remainder, err := ExtractFrames(frame)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
if hex.EncodeToString(frames[0][:len(payload)]) != hex.EncodeToString(payload) {
|
||||
t.Fatalf("frame was not unescaped: %s", hex.EncodeToString(frames[0]))
|
||||
}
|
||||
}
|
||||
|
||||
func buildFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
|
||||
payload := make([]byte, 0, 12+len(body)+1)
|
||||
payload = binary.BigEndian.AppendUint16(payload, messageID)
|
||||
payload = binary.BigEndian.AppendUint16(payload, uint16(len(body)))
|
||||
payload = append(payload, encodeBCD(phone, 6)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return payload
|
||||
}
|
||||
|
||||
func buildVersionedFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
|
||||
payload := make([]byte, 0, 17+len(body)+1)
|
||||
payload = binary.BigEndian.AppendUint16(payload, messageID)
|
||||
payload = binary.BigEndian.AppendUint16(payload, 0x4000|uint16(len(body)))
|
||||
payload = append(payload, 1)
|
||||
payload = append(payload, encodeBCD(phone, 10)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return payload
|
||||
}
|
||||
|
||||
func appendFixedASCII(out []byte, value string, size int) []byte {
|
||||
start := len(out)
|
||||
out = append(out, make([]byte, size)...)
|
||||
copy(out[start:], []byte(value))
|
||||
return out
|
||||
}
|
||||
|
||||
func encodeBCD(value string, size int) []byte {
|
||||
out := make([]byte, size)
|
||||
if len(value)%2 == 1 {
|
||||
value = "0" + value
|
||||
}
|
||||
if len(value) > size*2 {
|
||||
value = value[len(value)-size*2:]
|
||||
}
|
||||
for len(value) < size*2 {
|
||||
value = "0" + value
|
||||
}
|
||||
for i := 0; i < size; i++ {
|
||||
out[i] = (value[i*2]-'0')<<4 | (value[i*2+1] - '0')
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func escape(payload []byte) []byte {
|
||||
var out []byte
|
||||
for _, value := range payload {
|
||||
switch value {
|
||||
case 0x7e:
|
||||
out = append(out, 0x7d, 0x02)
|
||||
case 0x7d:
|
||||
out = append(out, 0x7d, 0x01)
|
||||
default:
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
130
go/vehicle-gateway/internal/protocol/jt808/response.go
Normal file
130
go/vehicle-gateway/internal/protocol/jt808/response.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrResponseHeaderTooShort = errors.New("jt808 response header too short")
|
||||
|
||||
const (
|
||||
msgTerminalGeneralResponse = uint16(0x0001)
|
||||
msgTerminalRegister = uint16(0x0100)
|
||||
msgPlatformGeneralResponse = uint16(0x8001)
|
||||
msgPlatformRegisterAck = uint16(0x8100)
|
||||
)
|
||||
|
||||
type AutoResponder struct {
|
||||
authCode string
|
||||
}
|
||||
|
||||
func NewAutoResponder(authCode string) AutoResponder {
|
||||
if authCode == "" {
|
||||
authCode = "g7gps"
|
||||
}
|
||||
return AutoResponder{authCode: authCode}
|
||||
}
|
||||
|
||||
func (r AutoResponder) Respond(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return nil, false, nil
|
||||
}
|
||||
header, err := parseResponseHeader(raw)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if header.messageID == msgTerminalGeneralResponse {
|
||||
return nil, false, nil
|
||||
}
|
||||
if header.messageID == msgTerminalRegister {
|
||||
body := make([]byte, 3, 3+len(r.authCode))
|
||||
binary.BigEndian.PutUint16(body[0:2], header.sequence)
|
||||
body[2] = 0
|
||||
body = append(body, []byte(r.authCode)...)
|
||||
return encodeResponse(msgPlatformRegisterAck, header, body), true, nil
|
||||
}
|
||||
body := make([]byte, 5)
|
||||
binary.BigEndian.PutUint16(body[0:2], header.sequence)
|
||||
binary.BigEndian.PutUint16(body[2:4], header.messageID)
|
||||
body[4] = 0
|
||||
return encodeResponse(msgPlatformGeneralResponse, header, body), true, nil
|
||||
}
|
||||
|
||||
type responseHeader struct {
|
||||
messageID uint16
|
||||
versioned bool
|
||||
protocolVersion byte
|
||||
phoneBCD []byte
|
||||
sequence uint16
|
||||
}
|
||||
|
||||
func parseResponseHeader(raw []byte) (responseHeader, error) {
|
||||
if len(raw) < 12 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
props := binary.BigEndian.Uint16(raw[2:4])
|
||||
versioned := props&0x4000 != 0
|
||||
phoneStart := 4
|
||||
phoneLen := 6
|
||||
protocolVersion := byte(0)
|
||||
if versioned {
|
||||
if len(raw) < 17 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
protocolVersion = raw[4]
|
||||
phoneStart = 5
|
||||
phoneLen = 10
|
||||
}
|
||||
sequenceStart := phoneStart + phoneLen
|
||||
if len(raw) < sequenceStart+2 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
phone := append([]byte(nil), raw[phoneStart:sequenceStart]...)
|
||||
return responseHeader{
|
||||
messageID: binary.BigEndian.Uint16(raw[0:2]),
|
||||
versioned: versioned,
|
||||
protocolVersion: protocolVersion,
|
||||
phoneBCD: phone,
|
||||
sequence: binary.BigEndian.Uint16(raw[sequenceStart : sequenceStart+2]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func encodeResponse(messageID uint16, header responseHeader, body []byte) []byte {
|
||||
props := uint16(len(body)) & 0x03ff
|
||||
if header.versioned {
|
||||
props |= 0x4000
|
||||
}
|
||||
payload := make([]byte, 0, 4+1+len(header.phoneBCD)+2+len(body)+1)
|
||||
payload = appendU16(payload, messageID)
|
||||
payload = appendU16(payload, props)
|
||||
if header.versioned {
|
||||
payload = append(payload, header.protocolVersion)
|
||||
}
|
||||
payload = append(payload, header.phoneBCD...)
|
||||
payload = appendU16(payload, header.sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return frame(payload)
|
||||
}
|
||||
|
||||
func appendU16(out []byte, value uint16) []byte {
|
||||
return append(out, byte(value>>8), byte(value))
|
||||
}
|
||||
|
||||
func frame(payload []byte) []byte {
|
||||
out := []byte{0x7e}
|
||||
for _, value := range payload {
|
||||
switch value {
|
||||
case 0x7e:
|
||||
out = append(out, 0x7d, 0x02)
|
||||
case 0x7d:
|
||||
out = append(out, 0x7d, 0x01)
|
||||
default:
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
out = append(out, 0x7e)
|
||||
return out
|
||||
}
|
||||
201
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
201
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package yutongmqtt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrInvalidJSON = errors.New("yutong mqtt invalid json")
|
||||
|
||||
func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS int64) (envelope.FrameEnvelope, error) {
|
||||
root := map[string]any{}
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&root); err != nil {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %v", ErrInvalidJSON, err)
|
||||
}
|
||||
|
||||
data := objectValue(root, "data")
|
||||
if data == nil {
|
||||
data = root
|
||||
}
|
||||
deviceID := firstText(root, "device", "deviceId", "terminalId", "terminalID", "imei", "IMEI", "mqttDeviceId")
|
||||
externalVIN := firstText(root, "vin", "VIN", "externalVin", "vehicleVin")
|
||||
vin := externalVIN
|
||||
if vin == "" && looksLikeVIN(deviceID) {
|
||||
vin = deviceID
|
||||
}
|
||||
plate := firstText(root, "plateNo", "carNo", "plate", "vehicleNo")
|
||||
phone := firstText(root, "sim", "phone", "mobile")
|
||||
eventTimeMS := parseTimeMS(firstText(root, "time", "deviceTime", "eventTime", "gpsTime"), receivedAtMS)
|
||||
fields := fieldsFromData(data)
|
||||
|
||||
parsed := map[string]any{
|
||||
"endpoint": endpoint,
|
||||
"topic": topic,
|
||||
"root": root,
|
||||
"data": data,
|
||||
}
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
MessageID: "MQTT",
|
||||
VIN: vin,
|
||||
VehicleKeyHint: deviceID,
|
||||
Phone: phone,
|
||||
DeviceID: deviceID,
|
||||
Plate: plate,
|
||||
SourceEndpoint: "mqtt://" + endpoint + topic,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawText: string(payload),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func fieldsFromData(data map[string]any) map[string]any {
|
||||
fields := map[string]any{}
|
||||
if speed, ok := firstFloat(data, "METER_SPEED", "speed", "speed_kmh"); ok {
|
||||
fields[envelope.FieldSpeedKMH] = speed
|
||||
}
|
||||
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
|
||||
fields[envelope.FieldTotalMileageKM] = normalizeTotalMileageKM(mileage)
|
||||
}
|
||||
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
|
||||
fields[envelope.FieldSOCPercent] = soc
|
||||
}
|
||||
if longitude, ok := firstCoord(data, "LONGITUDE", "longitude", "lng"); ok {
|
||||
fields[envelope.FieldLongitude] = longitude
|
||||
}
|
||||
if latitude, ok := firstCoord(data, "LATITUDE", "latitude", "lat"); ok {
|
||||
fields[envelope.FieldLatitude] = latitude
|
||||
}
|
||||
if direction, ok := firstFloat(data, "GPSDirection", "direction", "direction_deg"); ok {
|
||||
fields["direction_deg"] = direction
|
||||
}
|
||||
if gear, ok := firstFloat(data, "ON_GEAR", "gear"); ok {
|
||||
fields["gear"] = gear
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func objectValue(root map[string]any, key string) map[string]any {
|
||||
value, ok := root[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
typed, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return typed
|
||||
}
|
||||
|
||||
func firstText(root map[string]any, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value, ok := root[key]; ok && value != nil {
|
||||
text := strings.TrimSpace(fmt.Sprint(value))
|
||||
if text != "" && text != "<nil>" {
|
||||
return text
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstFloat(root map[string]any, keys ...string) (float64, bool) {
|
||||
for _, key := range keys {
|
||||
value, ok := root[key]
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
if parsed, ok := toFloat(value); ok {
|
||||
return parsed, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func firstCoord(root map[string]any, keys ...string) (float64, bool) {
|
||||
value, ok := firstFloat(root, keys...)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if value == 0 || int64(value) == 999999 {
|
||||
return 0, false
|
||||
}
|
||||
if value > 1000 || value < -1000 {
|
||||
value = value / 1_000_000
|
||||
}
|
||||
return value, true
|
||||
}
|
||||
|
||||
func toFloat(value any) (float64, bool) {
|
||||
switch typed := value.(type) {
|
||||
case json.Number:
|
||||
parsed, err := typed.Float64()
|
||||
return parsed, err == nil
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func parseTimeMS(raw string, fallback int64) int64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
for _, layout := range []string{
|
||||
"2006-01-02 15:04:05.000",
|
||||
"2006-01-02 15:04:05",
|
||||
} {
|
||||
if parsed, err := time.ParseInLocation(layout, raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
}
|
||||
if len(raw) == 14 {
|
||||
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
}
|
||||
if parsed, err := time.Parse(time.RFC3339, raw); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
if epoch, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
||||
if epoch > 10_000_000_000 {
|
||||
return epoch
|
||||
}
|
||||
return epoch * 1000
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func looksLikeVIN(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
return len(value) == 17
|
||||
}
|
||||
|
||||
func normalizeTotalMileageKM(value float64) float64 {
|
||||
return value / 1000
|
||||
}
|
||||
112
go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go
Normal file
112
go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package yutongmqtt
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device": "LTEST000000000001",
|
||||
"time": "20260413100000",
|
||||
"plateNo": "豫A12345",
|
||||
"data": {
|
||||
"METER_SPEED": 52.3,
|
||||
"TOTAL_MILEAGE": 123456.7,
|
||||
"BATTERY_CAPACITY_SOC": 70,
|
||||
"LONGITUDE": 116397128,
|
||||
"LATITUDE": 39916527,
|
||||
"GPSDirection": 88
|
||||
}
|
||||
}`)
|
||||
|
||||
env, err := ParseMessage("endpoint-a", "/ytforward/shln/dev1", payload, 1782745114999)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolYutongMQTT {
|
||||
t.Fatalf("protocol = %q", env.Protocol)
|
||||
}
|
||||
if env.VIN != "LTEST000000000001" || env.DeviceID != "LTEST000000000001" {
|
||||
t.Fatalf("identity fields = vin:%q device:%q", env.VIN, env.DeviceID)
|
||||
}
|
||||
if env.Plate != "豫A12345" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123.4567)
|
||||
assertFloatField(t, env, envelope.FieldSOCPercent, 70)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
|
||||
assertFloatField(t, env, "direction_deg", 88)
|
||||
if env.EventTimeMS == 1782745114999 {
|
||||
t.Fatal("event time should come from device time")
|
||||
}
|
||||
if env.RawText == "" || env.Parsed["data"] == nil {
|
||||
t.Fatalf("raw/parsed missing: %#v", env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
|
||||
payload := []byte(`{"device":"YT-DEVICE-001","time":"20260413100000","data":{"METER_SPEED":"12.3"}}`)
|
||||
env, err := ParseMessage("endpoint-a", "/ytforward/shln/YT-DEVICE-001", payload, 1782745114999)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if env.VIN != "" {
|
||||
t.Fatalf("non-vin device should not be treated as vin: %q", env.VIN)
|
||||
}
|
||||
if env.VehicleKey() != "YT-DEVICE-001" {
|
||||
t.Fatalf("vehicle key = %q", env.VehicleKey())
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
|
||||
}
|
||||
|
||||
func TestParseMessageNormalizesProductionMileageMeters(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device":"LMRKH9AC3R1004101",
|
||||
"time":"2026-07-02 01:02:04.085",
|
||||
"data":{"TOTAL_MILEAGE":56905000,"METER_SPEED":19}
|
||||
}`)
|
||||
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 56905)
|
||||
if env.EventTimeMS != 1782925324085 {
|
||||
t.Fatalf("event time = %d, want 1782925324085", env.EventTimeMS)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageTreatsSmallYutongMileageAsMeters(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device":"LMRKH9AC3R1004101",
|
||||
"time":"2026-07-02 01:02:04.085",
|
||||
"data":{"TOTAL_MILEAGE":1024,"METER_SPEED":19}
|
||||
}`)
|
||||
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.024)
|
||||
}
|
||||
|
||||
func TestParseMessageRejectsMalformedJSON(t *testing.T) {
|
||||
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
|
||||
if !errors.Is(err, ErrInvalidJSON) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
repository *Repository
|
||||
}
|
||||
|
||||
func NewHandler(repository *Repository) *Handler {
|
||||
if repository == nil {
|
||||
panic("realtime repository must not be nil")
|
||||
}
|
||||
return &Handler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
path := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/realtime/vehicles/"), "/")
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" {
|
||||
writeError(w, http.StatusNotFound, "vehicle not found")
|
||||
return
|
||||
}
|
||||
vin := parts[0]
|
||||
switch {
|
||||
case len(parts) == 1:
|
||||
snapshot, err := h.repository.GetMerged(r.Context(), vin)
|
||||
writeResult(w, snapshot, err)
|
||||
case len(parts) == 2 && parts[1] == "online":
|
||||
status, err := h.repository.IsOnline(r.Context(), vin)
|
||||
writeResult(w, status, err)
|
||||
case len(parts) == 3 && parts[1] == "protocols":
|
||||
snapshot, err := h.repository.GetProtocol(r.Context(), vin, envelope.Protocol(strings.ToUpper(parts[2])))
|
||||
writeResult(w, snapshot, err)
|
||||
default:
|
||||
writeError(w, http.StatusNotFound, "route not found")
|
||||
}
|
||||
}
|
||||
|
||||
func writeResult(w http.ResponseWriter, value any, err error) {
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
41
go/vehicle-gateway/internal/realtime/model.go
Normal file
41
go/vehicle-gateway/internal/realtime/model.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Snapshot struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Protocol envelope.Protocol `json:"protocol,omitempty"`
|
||||
Protocols []envelope.Protocol `json:"protocols,omitempty"`
|
||||
EventID string `json:"event_id,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
|
||||
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||
}
|
||||
|
||||
type OnlineStatus struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Online bool `json:"online"`
|
||||
LastSeenMS int64 `json:"last_seen_ms"`
|
||||
Protocols []envelope.Protocol `json:"protocols,omitempty"`
|
||||
TTLSeconds int64 `json:"ttl_seconds"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
OnlineTTL time.Duration
|
||||
}
|
||||
|
||||
func (c Config) ttl() time.Duration {
|
||||
if c.OnlineTTL <= 0 {
|
||||
return 10 * time.Minute
|
||||
}
|
||||
return c.OnlineTTL
|
||||
}
|
||||
232
go/vehicle-gateway/internal/realtime/repository.go
Normal file
232
go/vehicle-gateway/internal/realtime/repository.go
Normal file
@@ -0,0 +1,232 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
client *redis.Client
|
||||
cfg Config
|
||||
}
|
||||
|
||||
func NewRepository(client *redis.Client, cfg Config) *Repository {
|
||||
if client == nil {
|
||||
panic("redis client must not be nil")
|
||||
}
|
||||
return &Repository{client: client, cfg: cfg}
|
||||
}
|
||||
|
||||
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
vehicleKey := strings.TrimSpace(env.VehicleKey())
|
||||
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
|
||||
return nil
|
||||
}
|
||||
nowMS := time.Now().UnixMilli()
|
||||
eventMS := env.EventTimeMS
|
||||
if eventMS <= 0 {
|
||||
eventMS = env.ReceivedAtMS
|
||||
}
|
||||
protocolSnapshot := Snapshot{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
EventID: env.StableEventID(),
|
||||
EventTimeMS: eventMS,
|
||||
ReceivedAtMS: env.ReceivedAtMS,
|
||||
SourceEndpoint: env.SourceEndpoint,
|
||||
Fields: cloneFields(env.Fields),
|
||||
FieldTimesMS: fieldTimes(env.Fields, eventMS),
|
||||
UpdatedAtMS: nowMS,
|
||||
}
|
||||
if err := r.setJSON(ctx, protocolKey(vehicleKey, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
protocols, err := r.addProtocol(ctx, vehicleKey, env.Protocol)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
merged, err := r.GetMerged(ctx, vehicleKey)
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
return err
|
||||
}
|
||||
if merged.VehicleKey == "" {
|
||||
merged = Snapshot{VehicleKey: vehicleKey, VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
|
||||
} else if merged.VIN == "" && vin != "" {
|
||||
merged.VIN = vin
|
||||
}
|
||||
mergeFields(&merged, env.Fields, eventMS)
|
||||
if eventMS >= merged.EventTimeMS {
|
||||
merged.EventTimeMS = eventMS
|
||||
merged.EventID = env.StableEventID()
|
||||
merged.ReceivedAtMS = env.ReceivedAtMS
|
||||
merged.SourceEndpoint = env.SourceEndpoint
|
||||
}
|
||||
merged.Protocols = protocols
|
||||
merged.UpdatedAtMS = nowMS
|
||||
if err := r.setJSON(ctx, mergedKey(vehicleKey), merged, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
online := OnlineStatus{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Online: true,
|
||||
LastSeenMS: env.ReceivedAtMS,
|
||||
Protocols: protocols,
|
||||
TTLSeconds: int64(r.cfg.ttl().Seconds()),
|
||||
}
|
||||
if err := r.setJSON(ctx, onlineKey(vehicleKey), online, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vehicleKey}).Err()
|
||||
}
|
||||
|
||||
func (r *Repository) GetMerged(ctx context.Context, vehicleKey string) (Snapshot, error) {
|
||||
return r.getSnapshot(ctx, mergedKey(vehicleKey))
|
||||
}
|
||||
|
||||
func (r *Repository) GetProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) (Snapshot, error) {
|
||||
return r.getSnapshot(ctx, protocolKey(vehicleKey, protocol))
|
||||
}
|
||||
|
||||
func (r *Repository) IsOnline(ctx context.Context, vehicleKey string) (OnlineStatus, error) {
|
||||
vehicleKey = strings.TrimSpace(vehicleKey)
|
||||
var status OnlineStatus
|
||||
payload, err := r.client.Get(ctx, onlineKey(vehicleKey)).Bytes()
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}, nil
|
||||
}
|
||||
return status, err
|
||||
}
|
||||
if err := json.Unmarshal(payload, &status); err != nil {
|
||||
return status, err
|
||||
}
|
||||
ttl := r.client.TTL(ctx, onlineKey(vehicleKey)).Val()
|
||||
status.Online = ttl > 0
|
||||
status.TTLSeconds = int64(ttl.Seconds())
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func (r *Repository) getSnapshot(ctx context.Context, key string) (Snapshot, error) {
|
||||
var snapshot Snapshot
|
||||
payload, err := r.client.Get(ctx, key).Bytes()
|
||||
if err != nil {
|
||||
return snapshot, err
|
||||
}
|
||||
return snapshot, json.Unmarshal(payload, &snapshot)
|
||||
}
|
||||
|
||||
func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl time.Duration) error {
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return r.client.Set(ctx, key, payload, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Repository) addProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
|
||||
key := protocolsKey(vehicleKey)
|
||||
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = r.client.Expire(ctx, key, r.cfg.ttl()).Err()
|
||||
values, err := r.client.SMembers(ctx, key).Result()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(values)
|
||||
out := make([]envelope.Protocol, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, envelope.Protocol(value))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func mergeFields(snapshot *Snapshot, fields map[string]any, eventMS int64) {
|
||||
if snapshot.Fields == nil {
|
||||
snapshot.Fields = map[string]any{}
|
||||
}
|
||||
if snapshot.FieldTimesMS == nil {
|
||||
snapshot.FieldTimesMS = map[string]int64{}
|
||||
}
|
||||
for key, value := range fields {
|
||||
if key == envelope.FieldTotalMileageKM && !positiveNumber(value) {
|
||||
continue
|
||||
}
|
||||
if eventMS >= snapshot.FieldTimesMS[key] {
|
||||
snapshot.Fields[key] = value
|
||||
snapshot.FieldTimesMS[key] = eventMS
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func positiveNumber(value any) bool {
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed > 0
|
||||
case float32:
|
||||
return typed > 0
|
||||
case int:
|
||||
return typed > 0
|
||||
case int64:
|
||||
return typed > 0
|
||||
case uint16:
|
||||
return typed > 0
|
||||
case uint32:
|
||||
return typed > 0
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return err == nil && parsed > 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func cloneFields(fields map[string]any) map[string]any {
|
||||
if len(fields) == 0 {
|
||||
return map[string]any{}
|
||||
}
|
||||
out := make(map[string]any, len(fields))
|
||||
for key, value := range fields {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
|
||||
out := make(map[string]int64, len(fields))
|
||||
for key := range fields {
|
||||
out[key] = eventMS
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergedKey(vehicleKey string) string {
|
||||
return "vehicle:latest:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
|
||||
func protocolKey(vehicleKey string, protocol envelope.Protocol) string {
|
||||
return "vehicle:latest:" + strings.TrimSpace(vehicleKey) + ":" + string(protocol)
|
||||
}
|
||||
|
||||
func onlineKey(vehicleKey string) string {
|
||||
return "vehicle:online:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
|
||||
func protocolsKey(vehicleKey string) string {
|
||||
return "vehicle:protocols:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
250
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
250
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
@@ -0,0 +1,250 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 121.1,
|
||||
envelope.FieldTotalMileageKM: 10.5,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 900,
|
||||
ReceivedAtMS: 1200,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 120.0,
|
||||
envelope.FieldLatitude: 30.5,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
merged, err := repo.GetMerged(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.Fields[envelope.FieldLongitude] != 121.1 {
|
||||
t.Fatalf("older longitude overwrote newer value: %#v", merged.Fields)
|
||||
}
|
||||
if merged.Fields[envelope.FieldLatitude] != 30.5 {
|
||||
t.Fatalf("new latitude missing: %#v", merged.Fields)
|
||||
}
|
||||
if got := protocolNames(merged.Protocols); strings.Join(got, ",") != "GB32960,JT808" {
|
||||
t.Fatalf("merged protocols = %#v", got)
|
||||
}
|
||||
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProtocol() error = %v", err)
|
||||
}
|
||||
if protocol.Fields[envelope.FieldTotalMileageKM] != 10.5 {
|
||||
t.Fatalf("protocol snapshot = %#v", protocol)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryOnlineStatus(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
status, err := repo.IsOnline(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if status.Online {
|
||||
t.Fatal("empty vin should be offline")
|
||||
}
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
status, err = repo.IsOnline(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if !status.Online || status.LastSeenMS != 1100 {
|
||||
t.Fatalf("online status = %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryUpdatesPhoneOnlyVehicleKey(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811170",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldSpeedKMH: 23.0,
|
||||
envelope.FieldTotalMileageKM: 10003.7,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
vehicleKey := "JT808:013307811170"
|
||||
merged, err := repo.GetMerged(ctx, vehicleKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.VehicleKey != vehicleKey || merged.VIN != "" {
|
||||
t.Fatalf("unexpected identity: %#v", merged)
|
||||
}
|
||||
if merged.Fields[envelope.FieldTotalMileageKM] != 10003.7 {
|
||||
t.Fatalf("merged fields = %#v", merged.Fields)
|
||||
}
|
||||
status, err := repo.IsOnline(ctx, vehicleKey)
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if !status.Online || status.VehicleKey != vehicleKey {
|
||||
t.Fatalf("online status = %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryDoesNotOverwritePositiveMileageWithZero(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 12345.6,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 2000,
|
||||
ReceivedAtMS: 2100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldSpeedKMH: 22.0,
|
||||
envelope.FieldTotalMileageKM: 0,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
merged, err := repo.GetMerged(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.Fields[envelope.FieldTotalMileageKM] != 12345.6 {
|
||||
t.Fatalf("zero mileage overwrote positive value: %#v", merged.Fields)
|
||||
}
|
||||
if merged.Fields[envelope.FieldSpeedKMH] != 22.0 {
|
||||
t.Fatalf("new speed should still merge: %#v", merged.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReturnsMergedSnapshot(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/VIN001", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
NewHandler(repo).ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !stringsContains(rec.Body.String(), `"vin":"VIN001"`) {
|
||||
t.Fatalf("unexpected body: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReturnsPhoneOnlyVehicleKeySnapshot(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811170",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/JT808:013307811170", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
NewHandler(repo).ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !stringsContains(rec.Body.String(), `"vehicle_key":"JT808:013307811170"`) {
|
||||
t.Fatalf("unexpected body: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRepository(t *testing.T) (*Repository, func()) {
|
||||
t.Helper()
|
||||
server, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("miniredis.Run() error = %v", err)
|
||||
}
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
return NewRepository(client, Config{OnlineTTL: time.Minute}), func() {
|
||||
_ = client.Close()
|
||||
server.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func stringsContains(value string, pattern string) bool {
|
||||
return strings.Contains(value, pattern)
|
||||
}
|
||||
|
||||
func protocolNames(protocols []envelope.Protocol) []string {
|
||||
names := make([]string, 0, len(protocols))
|
||||
for _, protocol := range protocols {
|
||||
names = append(names, string(protocol))
|
||||
}
|
||||
return names
|
||||
}
|
||||
275
go/vehicle-gateway/internal/stats/daily_metric.go
Normal file
275
go/vehicle-gateway/internal/stats/daily_metric.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const (
|
||||
MetricDailyMileageKM = "daily_mileage_km"
|
||||
MetricDailyTotalMileageKM = "daily_total_mileage_km"
|
||||
)
|
||||
|
||||
type Execer interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
exec Execer
|
||||
loc *time.Location
|
||||
}
|
||||
|
||||
type MetricSample struct {
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Protocol envelope.Protocol
|
||||
StatDate string
|
||||
MetricKey string
|
||||
MetricValue float64
|
||||
TotalMileageKM float64
|
||||
}
|
||||
|
||||
func NewWriter(exec Execer, loc *time.Location) *Writer {
|
||||
if exec == nil {
|
||||
panic("stats execer must not be nil")
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
return &Writer{exec: exec, loc: loc}
|
||||
}
|
||||
|
||||
func (w *Writer) EnsureSchema(ctx context.Context) error {
|
||||
if _, err := w.exec.ExecContext(ctx, DailyMetricTableSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
if db, ok := w.exec.(metadataDB); ok {
|
||||
return migrateDailyMetricTable(ctx, db)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
samples, err := SamplesFromEnvelope(env, w.loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, sample := range samples {
|
||||
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL,
|
||||
sample.VehicleKey,
|
||||
sample.VIN,
|
||||
sample.StatDate,
|
||||
string(sample.Protocol),
|
||||
sample.MetricKey,
|
||||
sample.MetricValue,
|
||||
sample.TotalMileageKM,
|
||||
sample.TotalMileageKM); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
vehicleKey := strings.TrimSpace(env.VehicleKey())
|
||||
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
|
||||
return nil, nil
|
||||
}
|
||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
if totalMileage <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
eventMS := env.EventTimeMS
|
||||
if eventMS <= 0 {
|
||||
eventMS = env.ReceivedAtMS
|
||||
}
|
||||
if eventMS <= 0 {
|
||||
return nil, errors.New("event or received time is required")
|
||||
}
|
||||
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
|
||||
return []MetricSample{
|
||||
{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
StatDate: statDate,
|
||||
MetricKey: MetricDailyMileageKM,
|
||||
MetricValue: 0,
|
||||
TotalMileageKM: totalMileage,
|
||||
},
|
||||
{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
StatDate: statDate,
|
||||
MetricKey: MetricDailyTotalMileageKM,
|
||||
MetricValue: totalMileage,
|
||||
TotalMileageKM: totalMileage,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
const upsertDailyMetricSQL = `
|
||||
INSERT INTO vehicle_daily_metric
|
||||
(vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
|
||||
first_total_mileage_km = CASE
|
||||
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
|
||||
THEN VALUES(first_total_mileage_km)
|
||||
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
END,
|
||||
latest_total_mileage_km = CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
END,
|
||||
metric_value = CASE
|
||||
WHEN metric_key = 'daily_mileage_km'
|
||||
THEN GREATEST(
|
||||
CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE latest_total_mileage_km
|
||||
END,
|
||||
VALUES(latest_total_mileage_km)
|
||||
)
|
||||
- CASE
|
||||
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
|
||||
THEN VALUES(first_total_mileage_km)
|
||||
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
END
|
||||
WHEN metric_key = 'daily_total_mileage_km'
|
||||
THEN GREATEST(
|
||||
CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE latest_total_mileage_km
|
||||
END,
|
||||
VALUES(latest_total_mileage_km)
|
||||
)
|
||||
ELSE VALUES(metric_value)
|
||||
END,
|
||||
sample_count = sample_count + 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`
|
||||
|
||||
type metadataDB interface {
|
||||
Execer
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}
|
||||
|
||||
func migrateDailyMetricTable(ctx context.Context, db metadataDB) error {
|
||||
columnExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND column_name = 'vehicle_key'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !columnExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD COLUMN vehicle_key VARCHAR(96) NOT NULL DEFAULT '' AFTER id`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `UPDATE vehicle_daily_metric SET vehicle_key = vin WHERE vehicle_key = ''`); err != nil {
|
||||
return err
|
||||
}
|
||||
vehicleIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'uk_daily_metric_vehicle'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !vehicleIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key)`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
oldIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'uk_daily_metric'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if oldIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric DROP INDEX uk_daily_metric`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
vinIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'idx_vin'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !vinIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD KEY idx_vin (vin)`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func informationSchemaExists(ctx context.Context, db metadataDB, query string) (bool, error) {
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, query).Scan(&count); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||
if env.Fields == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
162
go/vehicle-gateway/internal/stats/daily_metric_test.go
Normal file
162
go/vehicle-gateway/internal/stats/daily_metric_test.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "LNBVIN00000000001",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 8, 30, 0, 0, loc).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 10241.2,
|
||||
},
|
||||
}
|
||||
|
||||
samples, err := SamplesFromEnvelope(env, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 2 {
|
||||
t.Fatalf("sample count = %d", len(samples))
|
||||
}
|
||||
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
|
||||
t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
|
||||
}
|
||||
if samples[0].VehicleKey != "LNBVIN00000000001" {
|
||||
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
|
||||
}
|
||||
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 {
|
||||
t.Fatalf("unexpected daily total sample: %#v", samples[1])
|
||||
}
|
||||
if samples[0].StatDate != "2026-07-01" {
|
||||
t.Fatalf("stat date = %q", samples[0].StatDate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811254",
|
||||
EventTimeMS: time.Date(2026, 7, 2, 0, 3, 0, 0, loc).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 10985.7,
|
||||
},
|
||||
}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 2 {
|
||||
t.Fatalf("sample count = %d", len(samples))
|
||||
}
|
||||
if samples[0].VehicleKey != "JT808:013307811254" {
|
||||
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
|
||||
}
|
||||
if samples[0].VIN != "" {
|
||||
t.Fatalf("vin should stay empty for unresolved JT808 identity, got %q", samples[0].VIN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples without vehicle key, got %#v", samples)
|
||||
}
|
||||
|
||||
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "LNBVIN00000000002",
|
||||
Fields: map[string]any{},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples without mileage, got %#v", samples)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeSkipsNonPositiveMileage(t *testing.T) {
|
||||
for _, value := range []any{0, 0.0, -1.0, "0"} {
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "LNBVIN00000000001",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: value,
|
||||
},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope(%#v) error = %v", value, err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples for non-positive mileage %#v, got %#v", value, samples)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
if err := writer.EnsureSchema(context.Background()); err != nil {
|
||||
t.Fatalf("EnsureSchema() error = %v", err)
|
||||
}
|
||||
if err := writer.Append(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "LNBVIN00000000002",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: "10000.0",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") {
|
||||
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
|
||||
}
|
||||
if !strings.Contains(exec.calls[0].query, "vehicle_key") {
|
||||
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
|
||||
}
|
||||
if len(exec.calls) != 3 {
|
||||
t.Fatalf("exec calls = %d", len(exec.calls))
|
||||
}
|
||||
if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") {
|
||||
t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query)
|
||||
}
|
||||
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
|
||||
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
|
||||
}
|
||||
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
|
||||
t.Fatalf("first upsert arg should be vehicle_key, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type execCall struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type recordingExec struct {
|
||||
calls []execCall
|
||||
}
|
||||
|
||||
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||
return nil, nil
|
||||
}
|
||||
326
go/vehicle-gateway/internal/stats/query.go
Normal file
326
go/vehicle-gateway/internal/stats/query.go
Normal file
@@ -0,0 +1,326 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Queryer interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
type MetricQuery struct {
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Protocol string
|
||||
MetricKey string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type MetricRow struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
StatDate string `json:"stat_date"`
|
||||
Protocol string `json:"protocol"`
|
||||
MetricKey string `json:"metric_key"`
|
||||
MetricValue float64 `json:"metric_value"`
|
||||
MetricUnit string `json:"metric_unit"`
|
||||
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
|
||||
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
|
||||
SampleCount int64 `json:"sample_count"`
|
||||
CalculationMethod string `json:"calculation_method"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type MetricRepository struct {
|
||||
db Queryer
|
||||
}
|
||||
|
||||
func NewMetricRepository(db Queryer) *MetricRepository {
|
||||
if db == nil {
|
||||
panic("metric query db must not be nil")
|
||||
}
|
||||
return &MetricRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]MetricRow, error) {
|
||||
query = normalizeMetricQuery(query)
|
||||
sqlText, args := buildMetricSQL(query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]MetricRow, 0)
|
||||
for rows.Next() {
|
||||
var row MetricRow
|
||||
var statDate scanDate
|
||||
var createdAt scanDateTime
|
||||
var updatedAt scanDateTime
|
||||
var first sql.NullFloat64
|
||||
var latest sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&statDate,
|
||||
&row.Protocol,
|
||||
&row.MetricKey,
|
||||
&row.MetricValue,
|
||||
&row.MetricUnit,
|
||||
&first,
|
||||
&latest,
|
||||
&row.SampleCount,
|
||||
&row.CalculationMethod,
|
||||
&createdAt,
|
||||
&updatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.StatDate = statDate.String
|
||||
row.CreatedAt = createdAt.String
|
||||
row.UpdatedAt = updatedAt.String
|
||||
if first.Valid {
|
||||
row.FirstTotalMileageKM = &first.Float64
|
||||
}
|
||||
if latest.Valid {
|
||||
row.LatestTotalMileageKM = &latest.Float64
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, error) {
|
||||
query = normalizeMetricQuery(query)
|
||||
sqlText, args := buildMetricCountSQL(query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var total int64
|
||||
if rows.Next() {
|
||||
if err := rows.Scan(&total); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return total, rows.Err()
|
||||
}
|
||||
|
||||
func normalizeMetricQuery(query MetricQuery) MetricQuery {
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.MetricKey = strings.TrimSpace(query.MetricKey)
|
||||
query.DateFrom = strings.TrimSpace(query.DateFrom)
|
||||
query.DateTo = strings.TrimSpace(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 50
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func buildMetricSQL(query MetricQuery) (string, []any) {
|
||||
where, args := buildMetricWhere(query)
|
||||
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric`
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
|
||||
args = append(args, query.Limit, query.Offset)
|
||||
return sqlText, args
|
||||
}
|
||||
|
||||
func buildMetricCountSQL(query MetricQuery) (string, []any) {
|
||||
where, args := buildMetricWhere(query)
|
||||
sqlText := `SELECT COUNT(*) FROM vehicle_daily_metric`
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, args
|
||||
}
|
||||
|
||||
func buildMetricWhere(query MetricQuery) ([]string, []any) {
|
||||
var where []string
|
||||
var args []any
|
||||
add := func(clause string, value any) {
|
||||
where = append(where, clause)
|
||||
args = append(args, value)
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = ?", query.VIN)
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = ?", query.VehicleKey)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = ?", query.Protocol)
|
||||
}
|
||||
if query.MetricKey != "" {
|
||||
add("metric_key = ?", query.MetricKey)
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("stat_date >= ?", query.DateFrom)
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("stat_date <= ?", query.DateTo)
|
||||
}
|
||||
return where, args
|
||||
}
|
||||
|
||||
type MetricHandler struct {
|
||||
repository *MetricRepository
|
||||
}
|
||||
|
||||
func NewMetricHandler(repository *MetricRepository) *MetricHandler {
|
||||
if repository == nil {
|
||||
panic("metric repository must not be nil")
|
||||
}
|
||||
return &MetricHandler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeMetricError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/stats/daily-metrics" {
|
||||
writeMetricError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseMetricQuery(r)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func parseMetricQuery(r *http.Request) (MetricQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit")
|
||||
if err != nil {
|
||||
return MetricQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return MetricQuery{}, err
|
||||
}
|
||||
query := MetricQuery{
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Protocol: values.Get("protocol"),
|
||||
MetricKey: values.Get("metricKey"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDate(query.DateFrom) || !validDate(query.DateTo) {
|
||||
return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD")
|
||||
}
|
||||
return normalizeMetricQuery(query), nil
|
||||
}
|
||||
|
||||
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil || value < min || value > max {
|
||||
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func validDate(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
if len(value) != len("2006-01-02") {
|
||||
return false
|
||||
}
|
||||
for i, r := range value {
|
||||
switch i {
|
||||
case 4, 7:
|
||||
if r != '-' {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func writeMetricError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
|
||||
type scanDate struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDate) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02")
|
||||
return nil
|
||||
}
|
||||
|
||||
type scanDateTime struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDateTime) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatSQLTime(value any, layout string) string {
|
||||
switch typed := value.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case time.Time:
|
||||
return typed.Format(layout)
|
||||
case []byte:
|
||||
return strings.TrimSpace(string(typed))
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
185
go/vehicle-gateway/internal/stats/query_test.go
Normal file
185
go/vehicle-gateway/internal/stats/query_test.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km",
|
||||
12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
))
|
||||
|
||||
repository := NewMetricRepository(db)
|
||||
rows, err := repository.Query(context.Background(), MetricQuery{
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
Protocol: "JT808",
|
||||
DateFrom: "2026-07-01",
|
||||
DateTo: "2026-07-01",
|
||||
Limit: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query() error = %v", err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("row count = %d", len(rows))
|
||||
}
|
||||
if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 {
|
||||
t.Fatalf("unexpected row: %#v", rows[0])
|
||||
}
|
||||
if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
|
||||
t.Fatalf("vehicle key = %q", rows[0].VehicleKey)
|
||||
}
|
||||
if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" {
|
||||
t.Fatalf("unexpected time formatting: %#v", rows[0])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km",
|
||||
53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25",
|
||||
))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":42`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("JT808:013307811254", "JT808").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("JT808:013307811254", "JT808", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"JT808:013307811254", "", "2026-07-02", "JT808", MetricDailyTotalMileageKM, 10985.7, "km",
|
||||
10985.7, 10985.7, 1, "TOTAL_MILEAGE_DIFF", "2026-07-02 00:03:00", "2026-07-02 00:03:00",
|
||||
))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vehicleKey=JT808:013307811254&protocol=JT808", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"metric_value":10985.7`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("YUTONG_MQTT").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("YUTONG_MQTT", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?protocol=YUTONG_MQTT", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
|
||||
t.Fatalf("expected empty items array, got: %s", body)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerRejectsInvalidPagination(t *testing.T) {
|
||||
handler := NewMetricHandler(NewMetricRepository(&sql.DB{}))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?limit=2001", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
22
go/vehicle-gateway/internal/stats/schema.go
Normal file
22
go/vehicle-gateway/internal/stats/schema.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package stats
|
||||
|
||||
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vehicle_key VARCHAR(96) NOT NULL,
|
||||
vin VARCHAR(32) NOT NULL DEFAULT '',
|
||||
stat_date DATE NOT NULL,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,3) NOT NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key),
|
||||
KEY idx_vin (vin),
|
||||
KEY idx_stat_date (stat_date),
|
||||
KEY idx_protocol_metric (protocol, metric_key)
|
||||
)`
|
||||
@@ -49,5 +49,10 @@
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
@@ -8,7 +8,7 @@ import org.springframework.web.servlet.DispatcherServlet;
|
||||
|
||||
@AutoConfiguration
|
||||
@ConditionalOnClass(DispatcherServlet.class)
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.command-gateway", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.command-gateway", name = "enabled", havingValue = "true")
|
||||
@ComponentScan(basePackageClasses = TerminalCommandController.class)
|
||||
public class CommandGatewayAutoConfiguration {
|
||||
// 只扫描 REST 下行命令网关;不参与 32960 接收、RAW 冷存或历史查询链路。
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.lingniu.ingest.gateway;
|
||||
|
||||
import com.lingniu.ingest.session.CommandDispatcher;
|
||||
import com.lingniu.ingest.session.DeviceSession;
|
||||
import com.lingniu.ingest.session.SessionStore;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.function.UnaryOperator;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class CommandGatewayAutoConfigurationTest {
|
||||
|
||||
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(CommandGatewayAutoConfiguration.class))
|
||||
.withBean(SessionStore.class, EmptySessionStore::new)
|
||||
.withBean(CommandDispatcher.class, NoopCommandDispatcher::new);
|
||||
|
||||
@Test
|
||||
void backsOffByDefault() {
|
||||
contextRunner.run(context -> assertThat(context).doesNotHaveBean(TerminalCommandController.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void createsControllerWhenExplicitlyEnabled() {
|
||||
contextRunner
|
||||
.withPropertyValues("lingniu.ingest.command-gateway.enabled=true")
|
||||
.run(context -> assertThat(context).hasSingleBean(TerminalCommandController.class));
|
||||
}
|
||||
|
||||
private static final class NoopCommandDispatcher implements CommandDispatcher {
|
||||
@Override
|
||||
public CompletableFuture<Void> notify(String sessionId, Object command) {
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> CompletableFuture<T> request(String sessionId,
|
||||
Object command,
|
||||
Class<T> responseType,
|
||||
Duration timeout) {
|
||||
return CompletableFuture.failedFuture(new UnsupportedOperationException("not used"));
|
||||
}
|
||||
}
|
||||
|
||||
private static final class EmptySessionStore implements SessionStore {
|
||||
@Override public void put(DeviceSession session) {}
|
||||
@Override public Optional<DeviceSession> findBySessionId(String sessionId) { return Optional.empty(); }
|
||||
@Override public Optional<DeviceSession> findByVin(String vin) { return Optional.empty(); }
|
||||
@Override public Optional<DeviceSession> findByPhone(String phone) { return Optional.empty(); }
|
||||
@Override public Optional<DeviceSession> update(String sessionId, UnaryOperator<DeviceSession> updater) {
|
||||
return Optional.empty();
|
||||
}
|
||||
@Override public void remove(String sessionId) {}
|
||||
@Override public int size() { return 0; }
|
||||
}
|
||||
}
|
||||
@@ -25,13 +25,21 @@
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>observability</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>vehicle-identity</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>protocol-gb32960</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-archive</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -41,10 +49,6 @@
|
||||
<groupId>com.alibaba.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springdoc</groupId>
|
||||
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
@@ -58,17 +62,6 @@
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
<goal>build-info</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
|
||||
@SpringBootApplication
|
||||
public class Gb32960IngestApplication {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(Gb32960IngestApplication.class);
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.lingniu.ingest.gb32960app.actuator;
|
||||
|
||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960ConnectionDiagnostics;
|
||||
import org.springframework.boot.actuate.endpoint.annotation.Endpoint;
|
||||
import org.springframework.boot.actuate.endpoint.annotation.ReadOperation;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
@Endpoint(id = "gb32960sessions")
|
||||
public class Gb32960SessionsEndpoint {
|
||||
|
||||
private final Gb32960ConnectionDiagnostics diagnostics;
|
||||
|
||||
public Gb32960SessionsEndpoint(Gb32960ConnectionDiagnostics diagnostics) {
|
||||
this.diagnostics = diagnostics;
|
||||
}
|
||||
|
||||
@ReadOperation
|
||||
public Map<String, Object> sessions() {
|
||||
var sessions = diagnostics.activeSessions();
|
||||
return Map.of(
|
||||
"activeCount", sessions.size(),
|
||||
"sessions", sessions);
|
||||
}
|
||||
}
|
||||
@@ -17,16 +17,17 @@ spring:
|
||||
threads:
|
||||
virtual:
|
||||
enabled: true
|
||||
data:
|
||||
redis:
|
||||
host: ${REDIS_HOST:127.0.0.1}
|
||||
port: ${REDIS_PORT:6379}
|
||||
username: ${REDIS_USERNAME:}
|
||||
password: ${REDIS_PASSWORD:}
|
||||
database: ${REDIS_DATABASE:50}
|
||||
|
||||
server:
|
||||
port: ${HTTP_PORT:20100}
|
||||
|
||||
springdoc:
|
||||
swagger-ui:
|
||||
path: /swagger-ui.html
|
||||
api-docs:
|
||||
path: /v3/api-docs
|
||||
|
||||
lingniu:
|
||||
ingest:
|
||||
gb32960:
|
||||
@@ -42,10 +43,15 @@ lingniu:
|
||||
whitelist: []
|
||||
platforms:
|
||||
- username: ${GB32960_PLATFORM_USER_HYUNDAI:Hyundai}
|
||||
password: ${GB32960_PLATFORM_PWD_HYUNDAI:f2e3445d7cda409fb4f278f6fb890734}
|
||||
password: ${GB32960_PLATFORM_PWD_HYUNDAI:}
|
||||
allowed-ips:
|
||||
- ${GB32960_PLATFORM_IP_HYUNDAI:8.134.95.166}
|
||||
- ${GB32960_PLATFORM_IP_HYUNDAI:}
|
||||
description: 外部下级平台 - Hyundai
|
||||
- username: ${GB32960_PLATFORM_USER_YUEJIN:YueJin}
|
||||
password: ${GB32960_PLATFORM_PWD_YUEJIN:}
|
||||
allowed-ips:
|
||||
- ${GB32960_PLATFORM_IP_YUEJIN:}
|
||||
description: 外部下级平台 - YueJin
|
||||
tls:
|
||||
enabled: ${GB32960_TLS_ENABLED:false}
|
||||
cert-path: ${GB32960_TLS_CERT:}
|
||||
@@ -57,6 +63,7 @@ lingniu:
|
||||
match:
|
||||
platform-accounts:
|
||||
- Hyundai
|
||||
- YueJin
|
||||
vin-prefixes: []
|
||||
vins: []
|
||||
parse:
|
||||
@@ -75,16 +82,17 @@ lingniu:
|
||||
rate-limit:
|
||||
per-vin-qps: 50
|
||||
session:
|
||||
store: ${SESSION_STORE:memory}
|
||||
ttl: ${SESSION_TTL:30m}
|
||||
identity:
|
||||
store: ${VEHICLE_IDENTITY_STORE:file}
|
||||
file:
|
||||
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
|
||||
mysql:
|
||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
|
||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
kafka:
|
||||
enabled: true
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
compression-type: zstd
|
||||
linger-ms: 20
|
||||
@@ -102,20 +110,20 @@ lingniu:
|
||||
dlq: ${KAFKA_TOPIC_GB32960_DLQ:vehicle.dlq.gb32960.v1}
|
||||
consumer:
|
||||
enabled: false
|
||||
event-file-store:
|
||||
enabled: false
|
||||
event-history:
|
||||
enabled: false
|
||||
vehicle-state:
|
||||
enabled: false
|
||||
vehicle-stat:
|
||||
enabled: false
|
||||
archive:
|
||||
enabled: ${SINK_ARCHIVE_ENABLED:true}
|
||||
type: local
|
||||
path: ${SINK_ARCHIVE_PATH:./archive/}
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus,env
|
||||
include: health,info,metrics,prometheus,gb32960sessions
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
|
||||
health:
|
||||
redis:
|
||||
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
package com.lingniu.ingest.gb32960app;
|
||||
|
||||
import com.lingniu.ingest.core.config.IngestCoreAutoConfiguration;
|
||||
import com.lingniu.ingest.identity.MySqlVehicleIdentityService;
|
||||
import com.lingniu.ingest.identity.config.VehicleIdentityAutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
|
||||
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
|
||||
import com.lingniu.ingest.session.SessionStore;
|
||||
import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
||||
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
@@ -24,34 +30,38 @@ class Gb32960IngestAppCompositionTest {
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
IngestCoreAutoConfiguration.class,
|
||||
SessionCoreAutoConfiguration.class,
|
||||
SinkMqAutoConfiguration.class,
|
||||
VehicleIdentityAutoConfiguration.class,
|
||||
KafkaSinkAutoConfiguration.class,
|
||||
SinkArchiveAutoConfiguration.class,
|
||||
Gb32960AutoConfiguration.class))
|
||||
.withAllowBeanDefinitionOverriding(true)
|
||||
.withBean("kafkaProducer", KafkaProducer.class, Gb32960IngestAppCompositionTest::kafkaProducer)
|
||||
.withBean(SessionStore.class, () -> mock(SessionStore.class))
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.gb32960.enabled=true",
|
||||
"lingniu.ingest.gb32960.server.enabled=true",
|
||||
"lingniu.ingest.gb32960.port=0",
|
||||
"lingniu.ingest.session.store=memory",
|
||||
"lingniu.ingest.sink.mq.enabled=true",
|
||||
"lingniu.ingest.sink.mq.type=kafka",
|
||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
||||
"lingniu.ingest.event-file-store.enabled=false",
|
||||
"lingniu.ingest.event-history.enabled=false",
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=false");
|
||||
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||
"lingniu.ingest.sink.archive.enabled=true",
|
||||
"lingniu.ingest.sink.archive.path=target/test-archive-gb32960",
|
||||
"lingniu.ingest.event-history.enabled=false");
|
||||
|
||||
@Test
|
||||
void createsGb32960ListenerWithoutHistoryStorageBoundaries() {
|
||||
contextRunner.run(context -> {
|
||||
assertThat(context).hasSingleBean(Gb32960MessageDecoder.class);
|
||||
assertThat(context).hasSingleBean(Gb32960NettyServer.class);
|
||||
assertThat(context).hasSingleBean(MySqlVehicleIdentityService.class);
|
||||
assertThat(context).hasSingleBean(KafkaEventSink.class);
|
||||
assertThat(context).hasSingleBean(KafkaEnvelopeDeadLetterSink.class);
|
||||
assertThat(context).hasSingleBean(ArchiveStore.class);
|
||||
assertThat(context).hasSingleBean(RawArchiveEventSink.class);
|
||||
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.sink.archive.ArchiveStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Properties;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -11,8 +13,10 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
class Gb32960IngestAppDefaultsTest {
|
||||
|
||||
@Test
|
||||
void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() {
|
||||
void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() throws IOException {
|
||||
Properties properties = applicationProperties();
|
||||
String legacyKafkaPrefix = "lingniu.ingest.sink." + "mq.";
|
||||
String kafkaTypeProperty = "lingniu.ingest.sink.kafka." + "type";
|
||||
|
||||
assertThat(properties)
|
||||
.containsEntry("spring.application.name", "gb32960-ingest-app")
|
||||
@@ -20,15 +24,54 @@ class Gb32960IngestAppDefaultsTest {
|
||||
"spring.config.import[0]",
|
||||
"optional:nacos:${spring.application.name}.${NACOS_CONFIG_FILE_EXTENSION:yml}?group=${NACOS_GROUP:DEFAULT_GROUP}&refreshEnabled=${NACOS_REFRESH_ENABLED:true}")
|
||||
.containsEntry("spring.cloud.nacos.config.server-addr", "${NACOS_SERVER_ADDR:127.0.0.1:8848}")
|
||||
.containsEntry("spring.data.redis.host", "${REDIS_HOST:127.0.0.1}")
|
||||
.containsEntry("spring.data.redis.port", "${REDIS_PORT:6379}")
|
||||
.containsEntry("spring.data.redis.database", "${REDIS_DATABASE:50}")
|
||||
.containsEntry("lingniu.ingest.gb32960.enabled", true)
|
||||
.containsEntry("lingniu.ingest.gb32960.server.enabled", true)
|
||||
.containsEntry("lingniu.ingest.gb32960.port", "${GB32960_PORT:32960}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
|
||||
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
|
||||
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false);
|
||||
.containsEntry("lingniu.ingest.gb32960.auth.platforms[0].username",
|
||||
"${GB32960_PLATFORM_USER_HYUNDAI:Hyundai}")
|
||||
.containsEntry("lingniu.ingest.gb32960.auth.platforms[0].password",
|
||||
"${GB32960_PLATFORM_PWD_HYUNDAI:}")
|
||||
.containsEntry("lingniu.ingest.gb32960.auth.platforms[1].username",
|
||||
"${GB32960_PLATFORM_USER_YUEJIN:YueJin}")
|
||||
.containsEntry("lingniu.ingest.gb32960.auth.platforms[1].password",
|
||||
"${GB32960_PLATFORM_PWD_YUEJIN:}")
|
||||
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[0]",
|
||||
"Hyundai")
|
||||
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[1]",
|
||||
"YueJin")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
|
||||
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
||||
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
|
||||
.containsEntry("management.endpoints.web.exposure.include",
|
||||
"health,info,metrics,prometheus,gb32960sessions");
|
||||
|
||||
assertThat(properties.stringPropertyNames())
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
|
||||
.noneMatch(name -> name.startsWith(legacyKafkaPrefix))
|
||||
.noneMatch(name -> name.equals(kafkaTypeProperty))
|
||||
.noneMatch(name -> name.equals("lingniu.ingest.session.store"))
|
||||
.noneMatch(name -> name.equals("lingniu.ingest.identity.store"));
|
||||
assertThat(applicationYaml())
|
||||
.doesNotContain("event-history:")
|
||||
.doesNotContain("event-file-store:")
|
||||
.doesNotContain("vehicle-state:")
|
||||
.doesNotContain("vehicle-stat:")
|
||||
.doesNotContain("springdoc:");
|
||||
assertThat(applicationYaml())
|
||||
.contains("password: ${GB32960_PLATFORM_PWD_HYUNDAI:}")
|
||||
.contains("password: ${GB32960_PLATFORM_PWD_YUEJIN:}")
|
||||
.doesNotContain("SESSION_STORE")
|
||||
.doesNotContain("VEHICLE_IDENTITY_STORE")
|
||||
.doesNotContain("KAFKA_ENABLED");
|
||||
}
|
||||
|
||||
private static Properties applicationProperties() {
|
||||
@@ -40,4 +83,9 @@ class Gb32960IngestAppDefaultsTest {
|
||||
assertThat(properties).isNotNull();
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static String applicationYaml() throws IOException {
|
||||
return new String(new ClassPathResource("application.yml").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.lingniu.ingest.gb32960app.actuator;
|
||||
|
||||
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960ConnectionDiagnostics;
|
||||
import com.lingniu.ingest.protocol.gb32960.model.CommandType;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class Gb32960SessionsEndpointTest {
|
||||
|
||||
@Test
|
||||
void returnsActiveGb32960Sessions() {
|
||||
Gb32960ConnectionDiagnostics diagnostics = new Gb32960ConnectionDiagnostics();
|
||||
diagnostics.opened("session-1", "127.0.0.1:65469");
|
||||
diagnostics.received("session-1", CommandType.PLATFORM_LOGIN, "");
|
||||
diagnostics.platformLoginAccepted("session-1", "Hyundai");
|
||||
|
||||
Gb32960SessionsEndpoint endpoint = new Gb32960SessionsEndpoint(diagnostics);
|
||||
|
||||
Map<String, Object> result = endpoint.sessions();
|
||||
|
||||
assertThat(result).containsEntry("activeCount", 1);
|
||||
assertThat(result.get("sessions")).isInstanceOf(List.class);
|
||||
assertThat((List<?>) result.get("sessions")).singleElement()
|
||||
.hasFieldOrPropertyWithValue("sessionId", "session-1")
|
||||
.hasFieldOrPropertyWithValue("peer", "127.0.0.1:65469")
|
||||
.hasFieldOrPropertyWithValue("lastCommand", "PLATFORM_LOGIN")
|
||||
.hasFieldOrPropertyWithValue("platformAccount", "Hyundai");
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,11 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-archive</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -45,10 +49,6 @@
|
||||
<groupId>com.alibaba.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springdoc</groupId>
|
||||
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
@@ -62,17 +62,6 @@
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
<goal>build-info</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
|
||||
@SpringBootApplication
|
||||
public class Jt808IngestApplication {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(Jt808IngestApplication.class);
|
||||
|
||||
@@ -17,21 +17,22 @@ spring:
|
||||
threads:
|
||||
virtual:
|
||||
enabled: true
|
||||
data:
|
||||
redis:
|
||||
host: ${REDIS_HOST:127.0.0.1}
|
||||
port: ${REDIS_PORT:6379}
|
||||
username: ${REDIS_USERNAME:}
|
||||
password: ${REDIS_PASSWORD:}
|
||||
database: ${REDIS_DATABASE:50}
|
||||
|
||||
server:
|
||||
port: ${HTTP_PORT:20400}
|
||||
|
||||
springdoc:
|
||||
swagger-ui:
|
||||
path: /swagger-ui.html
|
||||
api-docs:
|
||||
path: /v3/api-docs
|
||||
|
||||
lingniu:
|
||||
ingest:
|
||||
jt808:
|
||||
enabled: true
|
||||
port: ${JT808_PORT:10808}
|
||||
port: ${JT808_PORT:808}
|
||||
worker-threads: ${JT808_WORKER_THREADS:0}
|
||||
pipeline:
|
||||
disruptor:
|
||||
@@ -45,16 +46,17 @@ lingniu:
|
||||
rate-limit:
|
||||
per-vin-qps: 50
|
||||
session:
|
||||
store: ${SESSION_STORE:memory}
|
||||
ttl: ${SESSION_TTL:30m}
|
||||
identity:
|
||||
store: ${VEHICLE_IDENTITY_STORE:file}
|
||||
file:
|
||||
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
|
||||
mysql:
|
||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
|
||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
kafka:
|
||||
enabled: true
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
compression-type: zstd
|
||||
linger-ms: 20
|
||||
@@ -72,20 +74,16 @@ lingniu:
|
||||
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
|
||||
consumer:
|
||||
enabled: false
|
||||
event-file-store:
|
||||
enabled: false
|
||||
event-history:
|
||||
enabled: false
|
||||
vehicle-state:
|
||||
enabled: false
|
||||
vehicle-stat:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus,env
|
||||
include: health,info,metrics,prometheus
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
|
||||
health:
|
||||
redis:
|
||||
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}
|
||||
|
||||
@@ -1,14 +1,19 @@
|
||||
package com.lingniu.ingest.jt808app;
|
||||
|
||||
import com.lingniu.ingest.core.config.IngestCoreAutoConfiguration;
|
||||
import com.lingniu.ingest.identity.MySqlVehicleIdentityService;
|
||||
import com.lingniu.ingest.identity.config.VehicleIdentityAutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.jt808.codec.Jt808MessageDecoder;
|
||||
import com.lingniu.ingest.protocol.jt808.config.Jt808AutoConfiguration;
|
||||
import com.lingniu.ingest.protocol.jt808.inbound.Jt808NettyServer;
|
||||
import com.lingniu.ingest.session.SessionStore;
|
||||
import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.archive.ArchiveStore;
|
||||
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
|
||||
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
@@ -26,33 +31,36 @@ class Jt808IngestAppCompositionTest {
|
||||
IngestCoreAutoConfiguration.class,
|
||||
SessionCoreAutoConfiguration.class,
|
||||
VehicleIdentityAutoConfiguration.class,
|
||||
SinkMqAutoConfiguration.class,
|
||||
KafkaSinkAutoConfiguration.class,
|
||||
SinkArchiveAutoConfiguration.class,
|
||||
Jt808AutoConfiguration.class))
|
||||
.withAllowBeanDefinitionOverriding(true)
|
||||
.withBean("kafkaProducer", KafkaProducer.class, Jt808IngestAppCompositionTest::kafkaProducer)
|
||||
.withBean(SessionStore.class, () -> mock(SessionStore.class))
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.jt808.enabled=true",
|
||||
"lingniu.ingest.jt808.port=0",
|
||||
"lingniu.ingest.session.store=memory",
|
||||
"lingniu.ingest.sink.mq.enabled=true",
|
||||
"lingniu.ingest.sink.mq.type=kafka",
|
||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
||||
"lingniu.ingest.event-file-store.enabled=false",
|
||||
"lingniu.ingest.event-history.enabled=false",
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=false");
|
||||
"lingniu.ingest.identity.mysql.initialize-schema=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||
"lingniu.ingest.sink.archive.enabled=true",
|
||||
"lingniu.ingest.sink.archive.path=target/test-archive-jt808",
|
||||
"lingniu.ingest.event-history.enabled=false");
|
||||
|
||||
@Test
|
||||
void createsJt808ListenerWithoutHistoryStorageBoundaries() {
|
||||
contextRunner.run(context -> {
|
||||
assertThat(context).hasSingleBean(Jt808MessageDecoder.class);
|
||||
assertThat(context).hasSingleBean(Jt808NettyServer.class);
|
||||
assertThat(context).hasSingleBean(MySqlVehicleIdentityService.class);
|
||||
assertThat(context).hasSingleBean(KafkaEventSink.class);
|
||||
assertThat(context).hasSingleBean(KafkaEnvelopeDeadLetterSink.class);
|
||||
assertThat(context).hasSingleBean(ArchiveStore.class);
|
||||
assertThat(context).hasSingleBean(RawArchiveEventSink.class);
|
||||
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.sink.archive.ArchiveStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Properties;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -11,7 +13,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
class Jt808IngestAppDefaultsTest {
|
||||
|
||||
@Test
|
||||
void applicationDefaultsKeepJt808IngestAsTcpKafkaProducerOnly() {
|
||||
void applicationDefaultsKeepJt808IngestAsTcpKafkaProducerOnly() throws IOException {
|
||||
Properties properties = applicationProperties();
|
||||
|
||||
assertThat(properties)
|
||||
@@ -20,18 +22,43 @@ class Jt808IngestAppDefaultsTest {
|
||||
"spring.config.import[0]",
|
||||
"optional:nacos:${spring.application.name}.${NACOS_CONFIG_FILE_EXTENSION:yml}?group=${NACOS_GROUP:DEFAULT_GROUP}&refreshEnabled=${NACOS_REFRESH_ENABLED:true}")
|
||||
.containsEntry("spring.cloud.nacos.config.server-addr", "${NACOS_SERVER_ADDR:127.0.0.1:8848}")
|
||||
.containsEntry("spring.data.redis.host", "${REDIS_HOST:127.0.0.1}")
|
||||
.containsEntry("spring.data.redis.port", "${REDIS_PORT:6379}")
|
||||
.containsEntry("spring.data.redis.database", "${REDIS_DATABASE:50}")
|
||||
.containsEntry("server.port", "${HTTP_PORT:20400}")
|
||||
.containsEntry("lingniu.ingest.jt808.enabled", true)
|
||||
.containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:10808}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
|
||||
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false);
|
||||
.containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:808}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
|
||||
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
||||
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url",
|
||||
"${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:root}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.password", "${VEHICLE_IDENTITY_MYSQL_PASSWORD:}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.initialize-schema",
|
||||
"${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}")
|
||||
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
|
||||
|
||||
assertThat(properties.stringPropertyNames())
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
|
||||
.noneMatch(name -> name.equals("lingniu.ingest.session.store"))
|
||||
.noneMatch(name -> name.equals("lingniu.ingest.identity.store"));
|
||||
assertThat(applicationYaml())
|
||||
.doesNotContain("event-history:")
|
||||
.doesNotContain("event-file-store:")
|
||||
.doesNotContain("vehicle-state:")
|
||||
.doesNotContain("vehicle-stat:")
|
||||
.doesNotContain("springdoc:")
|
||||
.doesNotContain("SESSION_STORE")
|
||||
.doesNotContain("VEHICLE_IDENTITY_STORE")
|
||||
.doesNotContain("KAFKA_ENABLED");
|
||||
}
|
||||
|
||||
private static Properties applicationProperties() {
|
||||
@@ -43,4 +70,9 @@ class Jt808IngestAppDefaultsTest {
|
||||
assertThat(properties).isNotNull();
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static String applicationYaml() throws IOException {
|
||||
return new String(new ClassPathResource("application.yml").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
<artifactId>vehicle-analytics-app</artifactId>
|
||||
<name>vehicle-analytics-app</name>
|
||||
<description>Vehicle state, daily statistics, alarms, and analytics runtime.</description>
|
||||
<description>Vehicle daily statistics and analytics runtime.</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -23,16 +23,16 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>vehicle-state-service</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>vehicle-stat-service</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
@@ -58,17 +58,6 @@
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
<goal>build-info</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
|
||||
@SpringBootApplication
|
||||
public class VehicleAnalyticsApplication {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(VehicleAnalyticsApplication.class);
|
||||
|
||||
@@ -17,9 +17,14 @@ spring:
|
||||
threads:
|
||||
virtual:
|
||||
enabled: true
|
||||
datasource:
|
||||
url: ${MYSQL_JDBC_URL:}
|
||||
username: ${MYSQL_USERNAME:}
|
||||
password: ${MYSQL_PASSWORD:}
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
|
||||
server:
|
||||
port: ${HTTP_PORT:20300}
|
||||
port: ${HTTP_PORT:20310}
|
||||
|
||||
springdoc:
|
||||
swagger-ui:
|
||||
@@ -29,53 +34,38 @@ springdoc:
|
||||
|
||||
lingniu:
|
||||
ingest:
|
||||
gb32960:
|
||||
enabled: false
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
kafka:
|
||||
enabled: true
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
topics:
|
||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
dlq: ${KAFKA_TOPIC_GB32960_DLQ:vehicle.dlq.gb32960.v1}
|
||||
consumer:
|
||||
dlq-topic: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
|
||||
enabled: ${KAFKA_CONSUMER_ENABLED:true}
|
||||
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics}
|
||||
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
|
||||
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:500}
|
||||
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:900000}
|
||||
bindings:
|
||||
vehicleStateEnvelopeConsumerProcessor:
|
||||
enabled: ${VEHICLE_STATE_ENABLED:false}
|
||||
group-id: ${KAFKA_GROUP_STATE:vehicle-state}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
vehicleStatEnvelopeConsumerProcessor:
|
||||
enabled: ${VEHICLE_STAT_ENABLED:true}
|
||||
enabled: true
|
||||
group-id: ${KAFKA_GROUP_STAT:vehicle-stat}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
archive:
|
||||
enabled: false
|
||||
event-file-store:
|
||||
enabled: false
|
||||
event-history:
|
||||
enabled: false
|
||||
vehicle-state:
|
||||
enabled: ${VEHICLE_STATE_ENABLED:false}
|
||||
- ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
|
||||
vehicle-stat:
|
||||
enabled: ${VEHICLE_STAT_ENABLED:true}
|
||||
file-path: ${VEHICLE_STAT_FILE_PATH:./target/vehicle-stat/}
|
||||
zone-id: ${VEHICLE_STAT_ZONE_ID:Asia/Shanghai}
|
||||
jt808:
|
||||
enabled: ${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus,env
|
||||
health:
|
||||
redis:
|
||||
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}
|
||||
include: health,info,metrics,prometheus
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
|
||||
metrics:
|
||||
tags:
|
||||
application: vehicle-analytics-app
|
||||
|
||||
@@ -1,64 +1,50 @@
|
||||
package com.lingniu.ingest.analyticsapp;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.mq.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
|
||||
import com.lingniu.ingest.vehiclestate.VehicleStateEnvelopeIngestor;
|
||||
import com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration;
|
||||
import com.lingniu.ingest.vehiclestat.DailyVehicleStatService;
|
||||
import com.lingniu.ingest.vehiclestat.FileVehicleStatRepository;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
|
||||
import com.lingniu.ingest.vehiclestat.JdbcVehicleStatMetricRepository;
|
||||
import com.lingniu.ingest.vehiclestat.VehicleStatController;
|
||||
import com.lingniu.ingest.vehiclestat.VehicleStatEnvelopeIngestor;
|
||||
import com.lingniu.ingest.vehiclestat.VehicleStatEventProcessor;
|
||||
import com.lingniu.ingest.vehiclestat.VehicleStatEventSink;
|
||||
import com.lingniu.ingest.vehiclestat.VehicleStatRepository;
|
||||
import com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration;
|
||||
import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.util.ClassUtils;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class VehicleAnalyticsAppCompositionTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@Test
|
||||
void createsStatsBeansWithoutProtocolListenerOrEventFileStore() {
|
||||
void createsStatsBeansWithoutProtocolListenerOrFileHistoryStore() {
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
SinkMqAutoConfiguration.class,
|
||||
VehicleStateAutoConfiguration.class,
|
||||
KafkaSinkAutoConfiguration.class,
|
||||
VehicleStatAutoConfiguration.class))
|
||||
.withAllowBeanDefinitionOverriding(true)
|
||||
.withBean("kafkaProducer", KafkaProducer.class, VehicleAnalyticsAppCompositionTest::kafkaProducer)
|
||||
.withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class))
|
||||
.withBean(ObjectMapper.class, ObjectMapper::new)
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.sink.mq.enabled=true",
|
||||
"lingniu.ingest.sink.mq.type=kafka",
|
||||
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=false",
|
||||
"lingniu.ingest.vehicle-state.enabled=false",
|
||||
"lingniu.ingest.sink.kafka.enabled=true",
|
||||
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
|
||||
"lingniu.ingest.sink.kafka.consumer.enabled=false",
|
||||
"lingniu.ingest.vehicle-stat.enabled=true",
|
||||
"lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"),
|
||||
"lingniu.ingest.event-file-store.enabled=false",
|
||||
"lingniu.ingest.event-history.enabled=false",
|
||||
"lingniu.ingest.gb32960.enabled=false")
|
||||
"lingniu.ingest.vehicle-stat.jt808.enabled=true")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(VehicleStatRepository.class);
|
||||
assertThat(context).hasSingleBean(FileVehicleStatRepository.class);
|
||||
assertThat(context).hasSingleBean(DailyVehicleStatService.class);
|
||||
assertThat(context).hasSingleBean(VehicleStatEventProcessor.class);
|
||||
assertThat(context).hasSingleBean(JdbcVehicleStatMetricRepository.class);
|
||||
assertThat(context).hasSingleBean(Jt808MileageStreamProcessor.class);
|
||||
assertThat(context).hasSingleBean(VehicleStatEnvelopeIngestor.class);
|
||||
assertThat(context).hasSingleBean(VehicleStatEventSink.class);
|
||||
assertThat(context).hasSingleBean(VehicleStatController.class);
|
||||
assertThat(context).hasSingleBean(KafkaEventSink.class);
|
||||
assertThat(context).hasSingleBean(KafkaEnvelopeDeadLetterSink.class);
|
||||
@@ -66,9 +52,12 @@ class VehicleAnalyticsAppCompositionTest {
|
||||
assertThat(context.getBean("vehicleStatEnvelopeConsumerProcessor"))
|
||||
.isInstanceOf(EnvelopeConsumerProcessor.class);
|
||||
|
||||
assertThat(context).doesNotHaveBean(VehicleStateEnvelopeIngestor.class);
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.vehiclestate.VehicleStateEnvelopeIngestor");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
|
||||
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
|
||||
assertTypeNotPresent(context,
|
||||
"com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Properties;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -11,7 +13,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
class VehicleAnalyticsAppDefaultsTest {
|
||||
|
||||
@Test
|
||||
void applicationDefaultsKeepAnalyticsAsStatConsumerRuntime() {
|
||||
void applicationDefaultsKeepAnalyticsAsStatConsumerRuntime() throws IOException {
|
||||
Properties properties = applicationProperties();
|
||||
|
||||
assertThat(properties)
|
||||
@@ -20,25 +22,42 @@ class VehicleAnalyticsAppDefaultsTest {
|
||||
"spring.config.import[0]",
|
||||
"optional:nacos:${spring.application.name}.${NACOS_CONFIG_FILE_EXTENSION:yml}?group=${NACOS_GROUP:DEFAULT_GROUP}&refreshEnabled=${NACOS_REFRESH_ENABLED:true}")
|
||||
.containsEntry("spring.cloud.nacos.config.server-addr", "${NACOS_SERVER_ADDR:127.0.0.1:8848}")
|
||||
.containsEntry("lingniu.ingest.gb32960.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.archive.enabled", false)
|
||||
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
|
||||
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
||||
.containsEntry("server.port", "${HTTP_PORT:20310}")
|
||||
.containsEntry("lingniu.ingest.vehicle-stat.enabled", "${VEHICLE_STAT_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.vehicle-state.enabled", "${VEHICLE_STATE_ENABLED:false}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
||||
.containsEntry(
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
|
||||
"${VEHICLE_STAT_ENABLED:true}")
|
||||
"lingniu.ingest.vehicle-stat.jt808.enabled",
|
||||
"${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
|
||||
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.kafka.consumer.dlq-topic",
|
||||
"${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
||||
.containsEntry(
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.enabled",
|
||||
"${VEHICLE_STATE_ENABLED:false}")
|
||||
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
|
||||
true)
|
||||
.containsEntry(
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
|
||||
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
|
||||
"${KAFKA_GROUP_STAT:vehicle-stat}")
|
||||
.containsEntry(
|
||||
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStateEnvelopeConsumerProcessor.group-id",
|
||||
"${KAFKA_GROUP_STATE:vehicle-state}");
|
||||
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
|
||||
|
||||
assertThat(properties.stringPropertyNames())
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.gb32960."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.sink.archive."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.sink.kafka.topics."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
|
||||
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
|
||||
.noneMatch(name -> name.contains("vehicleStateEnvelopeConsumerProcessor"))
|
||||
.noneMatch(name -> name.startsWith("spring.data.redis."))
|
||||
.noneMatch(name -> name.startsWith("management.health.redis."));
|
||||
assertThat(applicationYaml())
|
||||
.doesNotContain("gb32960:")
|
||||
.doesNotContain("event-history:")
|
||||
.doesNotContain("archive:")
|
||||
.doesNotContain("event-file-store:")
|
||||
.doesNotContain("vehicle-state:")
|
||||
.doesNotContain("vehicleStateEnvelopeConsumerProcessor")
|
||||
.doesNotContain("redis:")
|
||||
.doesNotContain("KAFKA_ENABLED");
|
||||
}
|
||||
|
||||
private static Properties applicationProperties() {
|
||||
@@ -50,4 +69,9 @@ class VehicleAnalyticsAppDefaultsTest {
|
||||
assertThat(properties).isNotNull();
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static String applicationYaml() throws IOException {
|
||||
return new String(new ClassPathResource("application.yml").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,15 +23,11 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-mq</artifactId>
|
||||
<artifactId>sink-kafka</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>sink-archive</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>event-file-store</artifactId>
|
||||
<artifactId>tdengine-history-store</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
@@ -66,17 +62,6 @@
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
<goal>build-info</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
|
||||
@SpringBootApplication
|
||||
public class VehicleHistoryApplication {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(VehicleHistoryApplication.class);
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.lingniu.ingest.historyapp;
|
||||
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerFactory;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerWorker;
|
||||
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
public class VehicleHistoryKafkaConsumerConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka.consumer", name = "enabled", havingValue = "true")
|
||||
public KafkaEnvelopeConsumerRunner vehicleHistoryKafkaEnvelopeConsumerRunner(
|
||||
@Qualifier("eventHistoryEnvelopeConsumerProcessor") EnvelopeConsumerProcessor processor,
|
||||
@Qualifier("eventHistoryRawEnvelopeConsumerProcessor") EnvelopeConsumerProcessor rawProcessor,
|
||||
KafkaSinkProperties props) {
|
||||
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(
|
||||
Map.of(
|
||||
"eventHistoryGb32960EnvelopeConsumerProcessor", processor,
|
||||
"eventHistoryJt808EnvelopeConsumerProcessor", processor,
|
||||
"eventHistoryYutongMqttEnvelopeConsumerProcessor", processor,
|
||||
"eventHistoryGb32960RawEnvelopeConsumerProcessor", rawProcessor,
|
||||
"eventHistoryJt808RawEnvelopeConsumerProcessor", rawProcessor,
|
||||
"eventHistoryYutongMqttRawEnvelopeConsumerProcessor", rawProcessor),
|
||||
props);
|
||||
if (workers.isEmpty()) {
|
||||
throw new IllegalStateException("no vehicle history kafka consumer workers created; check consumer bindings");
|
||||
}
|
||||
return new KafkaEnvelopeConsumerRunner(
|
||||
workers,
|
||||
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
|
||||
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
|
||||
props.getConsumer().isAutoStartup());
|
||||
}
|
||||
}
|
||||
@@ -29,53 +29,72 @@ springdoc:
|
||||
|
||||
lingniu:
|
||||
ingest:
|
||||
gb32960:
|
||||
enabled: true
|
||||
server:
|
||||
enabled: false
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
type: kafka
|
||||
kafka:
|
||||
enabled: true
|
||||
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
|
||||
topics:
|
||||
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
raw-archive: ${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}
|
||||
dlq: ${KAFKA_TOPIC_GB32960_DLQ:vehicle.dlq.gb32960.v1}
|
||||
consumer:
|
||||
dlq-topic: ${KAFKA_TOPIC_HISTORY_DLQ:vehicle.dlq.history.v1}
|
||||
enabled: ${KAFKA_CONSUMER_ENABLED:true}
|
||||
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history}
|
||||
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
|
||||
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:500}
|
||||
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}
|
||||
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:1800000}
|
||||
concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3}
|
||||
bindings:
|
||||
eventHistoryEnvelopeConsumerProcessor:
|
||||
eventHistoryGb32960EnvelopeConsumerProcessor:
|
||||
enabled: true
|
||||
group-id: ${KAFKA_GROUP_HISTORY:vehicle-history}
|
||||
group-id: ${KAFKA_GROUP_HISTORY_GB32960_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-event}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
|
||||
eventHistoryJt808EnvelopeConsumerProcessor:
|
||||
enabled: true
|
||||
group-id: ${KAFKA_GROUP_HISTORY_JT808_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-event}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
|
||||
eventHistoryYutongMqttEnvelopeConsumerProcessor:
|
||||
enabled: true
|
||||
group-id: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-event}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:vehicle.event.mqtt-yutong.v1}
|
||||
eventHistoryGb32960RawEnvelopeConsumerProcessor:
|
||||
enabled: true
|
||||
group-id: ${KAFKA_GROUP_HISTORY_GB32960_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-gb32960-raw}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}
|
||||
archive:
|
||||
enabled: ${SINK_ARCHIVE_ENABLED:true}
|
||||
type: local
|
||||
path: ${SINK_ARCHIVE_PATH:./archive/}
|
||||
event-file-store:
|
||||
enabled: ${EVENT_FILE_STORE_ENABLED:true}
|
||||
path: ${EVENT_FILE_STORE_PATH:./target/event-store/}
|
||||
zone-id: ${EVENT_FILE_STORE_ZONE_ID:Asia/Shanghai}
|
||||
batch-size: ${EVENT_FILE_STORE_BATCH_SIZE:500}
|
||||
flush-interval-millis: ${EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS:1000}
|
||||
eventHistoryJt808RawEnvelopeConsumerProcessor:
|
||||
enabled: true
|
||||
group-id: ${KAFKA_GROUP_HISTORY_JT808_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-jt808-raw}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}
|
||||
eventHistoryYutongMqttRawEnvelopeConsumerProcessor:
|
||||
enabled: true
|
||||
group-id: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-raw}
|
||||
topics:
|
||||
- ${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}
|
||||
tdengine-history:
|
||||
enabled: ${TDENGINE_HISTORY_ENABLED:false}
|
||||
database: ${TDENGINE_HISTORY_DATABASE:vehicle_history}
|
||||
jdbc-url: ${TDENGINE_JDBC_URL:jdbc:TAOS-RS://${TDENGINE_HOST:127.0.0.1}:${TDENGINE_PORT:6041}/${TDENGINE_HISTORY_DATABASE:vehicle_history}}
|
||||
username: ${TDENGINE_USERNAME:root}
|
||||
password: ${TDENGINE_PASSWORD:taosdata}
|
||||
driver-class-name: ${TDENGINE_DRIVER_CLASS_NAME:com.taosdata.jdbc.rs.RestfulDriver}
|
||||
maximum-pool-size: ${TDENGINE_MAX_POOL_SIZE:16}
|
||||
minimum-idle: ${TDENGINE_MIN_IDLE:0}
|
||||
connection-timeout-millis: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:5000}
|
||||
initialization-fail-timeout-millis: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:-1}
|
||||
event-history:
|
||||
enabled: true
|
||||
vehicle-state:
|
||||
enabled: false
|
||||
vehicle-stat:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus,env
|
||||
include: health,info,metrics,prometheus
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
|
||||
health:
|
||||
redis:
|
||||
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user