Compare commits
14 Commits
cde55e1ae6
...
796d79dd30
| Author | SHA1 | Date | |
|---|---|---|---|
|
796d79dd30 | ||
|
|
5f4f4fd124 | ||
|
|
e0068a750d | ||
|
|
665122b23b | ||
|
|
60ca42e5d5 | ||
|
|
80474eeccb | ||
|
|
17024651c8 | ||
|
|
d8c77d7882 | ||
|
|
b55b075a97 | ||
|
|
bac8864e79 | ||
|
|
5d844701ff | ||
|
|
967981c040 | ||
|
|
1fdbc29e27 | ||
|
|
d148a4e3af |
103
deploy/portainer/docker-compose-go.yml
Normal file
103
deploy/portainer/docker-compose-go.yml
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
version: "3.8"
|
||||||
|
|
||||||
|
# Go sidecar stack for the vehicle ingest redesign.
|
||||||
|
# It is intentionally separate from deploy/portainer/docker-compose.yml so the
|
||||||
|
# current Java production stack can keep running until Go evidence is captured.
|
||||||
|
|
||||||
|
x-common-env: &common-env
|
||||||
|
TZ: Asia/Shanghai
|
||||||
|
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
|
||||||
|
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||||
|
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||||
|
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.yutong-mqtt.v1}
|
||||||
|
KAFKA_TOPIC_UNIFIED: ${KAFKA_TOPIC_UNIFIED:-vehicle.event.unified.v1}
|
||||||
|
|
||||||
|
x-restart: &restart-policy
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- vehicle-ingest
|
||||||
|
logging:
|
||||||
|
driver: json-file
|
||||||
|
options:
|
||||||
|
max-size: "100m"
|
||||||
|
max-file: "5"
|
||||||
|
|
||||||
|
services:
|
||||||
|
go-vehicle-gateway:
|
||||||
|
<<: *restart-policy
|
||||||
|
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||||
|
container_name: go-vehicle-gateway
|
||||||
|
command: ["/app/gateway"]
|
||||||
|
mem_limit: ${GO_GATEWAY_MEM_LIMIT:-512m}
|
||||||
|
environment:
|
||||||
|
<<: *common-env
|
||||||
|
GB32960_TCP_ADDR: ":32960"
|
||||||
|
JT808_TCP_ADDR: ":808"
|
||||||
|
TCP_READ_BUFFER_BYTES: ${TCP_READ_BUFFER_BYTES:-65536}
|
||||||
|
TCP_IDLE_TIMEOUT_SECONDS: ${TCP_IDLE_TIMEOUT_SECONDS:-180}
|
||||||
|
TCP_MAX_CONNECTIONS: ${TCP_MAX_CONNECTIONS:-20000}
|
||||||
|
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
|
||||||
|
YUTONG_MQTT_ENDPOINT: ${YUTONG_MQTT_ENDPOINT:-yutong}
|
||||||
|
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
|
||||||
|
YUTONG_MQTT_TOPICS: ${YUTONG_MQTT_TOPICS:-/ytforward/shln/+}
|
||||||
|
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-2}
|
||||||
|
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt}
|
||||||
|
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
|
||||||
|
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
|
||||||
|
IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-}
|
||||||
|
VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding}
|
||||||
|
ports:
|
||||||
|
- "${GO_GB32960_TCP_PORT:-32960}:32960"
|
||||||
|
- "${GO_JT808_TCP_PORT:-808}:808"
|
||||||
|
|
||||||
|
go-history-writer:
|
||||||
|
<<: *restart-policy
|
||||||
|
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||||
|
container_name: go-history-writer
|
||||||
|
command: ["/app/history-writer"]
|
||||||
|
mem_limit: ${GO_HISTORY_WRITER_MEM_LIMIT:-768m}
|
||||||
|
environment:
|
||||||
|
<<: *common-env
|
||||||
|
KAFKA_GROUP: ${KAFKA_GROUP_GO_HISTORY:-go-history-writer}
|
||||||
|
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_HISTORY:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1}
|
||||||
|
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
|
||||||
|
TDENGINE_DSN: ${TDENGINE_DSN:?set TDENGINE_DSN, example root:password@ws(172.17.111.57:6041)/lingniu_vehicle_ts}
|
||||||
|
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
|
||||||
|
TDENGINE_ENSURE_SCHEMA: ${TDENGINE_ENSURE_SCHEMA:-true}
|
||||||
|
|
||||||
|
go-stat-writer:
|
||||||
|
<<: *restart-policy
|
||||||
|
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||||
|
container_name: go-stat-writer
|
||||||
|
command: ["/app/stat-writer"]
|
||||||
|
mem_limit: ${GO_STAT_WRITER_MEM_LIMIT:-512m}
|
||||||
|
environment:
|
||||||
|
<<: *common-env
|
||||||
|
KAFKA_GROUP: ${KAFKA_GROUP_GO_STAT:-go-stat-writer}
|
||||||
|
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_STAT:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1}
|
||||||
|
MYSQL_DSN: ${MYSQL_DSN:?set MYSQL_DSN}
|
||||||
|
MYSQL_ENSURE_SCHEMA: ${MYSQL_ENSURE_SCHEMA:-true}
|
||||||
|
LOCAL_TZ: ${LOCAL_TZ:-Asia/Shanghai}
|
||||||
|
|
||||||
|
go-realtime-api:
|
||||||
|
<<: *restart-policy
|
||||||
|
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||||
|
container_name: go-realtime-api
|
||||||
|
command: ["/app/realtime-api"]
|
||||||
|
mem_limit: ${GO_REALTIME_API_MEM_LIMIT:-512m}
|
||||||
|
environment:
|
||||||
|
<<: *common-env
|
||||||
|
HTTP_ADDR: ":20210"
|
||||||
|
KAFKA_GROUP: ${KAFKA_GROUP_GO_REALTIME:-go-realtime-api}
|
||||||
|
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_REALTIME:-vehicle.event.unified.v1}
|
||||||
|
REDIS_ADDR: ${REDIS_ADDR:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379}
|
||||||
|
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||||
|
REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
|
||||||
|
REDIS_DB: ${REDIS_DB:-50}
|
||||||
|
ONLINE_TTL_SECONDS: ${ONLINE_TTL_SECONDS:-600}
|
||||||
|
ports:
|
||||||
|
- "${GO_REALTIME_HTTP_PORT:-20210}:20210"
|
||||||
|
|
||||||
|
networks:
|
||||||
|
vehicle-ingest:
|
||||||
|
name: vehicle-ingest
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
# Go Vehicle Gateway 旁路验证记录
|
||||||
|
|
||||||
|
验证时间：2026-07-01 22:00-22:10 CST
|
||||||
|
|
||||||
|
## 部署版本
|
||||||
|
|
||||||
|
- Git commit：`60ca42e`
|
||||||
|
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-60ca42e-20260701220622`
|
||||||
|
- 部署方式：ECS `115.29.187.205` 上 Docker Compose 旁路部署
|
||||||
|
- Compose 目录：`/opt/lingniu-go`
|
||||||
|
|
||||||
|
## 旁路端口和 topic
|
||||||
|
|
||||||
|
为了不影响现有 Java 生产服务，本次没有占用生产端口和生产 topic。
|
||||||
|
|
||||||
|
| 项 | 值 |
|
||||||
|
|---|---|
|
||||||
|
| GB32960 旁路 TCP | `115.29.187.205:23296 -> container :32960` |
|
||||||
|
| JT808 旁路 TCP | `115.29.187.205:18080 -> container :808` |
|
||||||
|
| Realtime API | `115.29.187.205:20210` |
|
||||||
|
| GB32960 RAW topic | `vehicle.raw.go.gb32960.v1` |
|
||||||
|
| JT808 RAW topic | `vehicle.raw.go.jt808.v1` |
|
||||||
|
| Yutong MQTT RAW topic | `vehicle.raw.go.yutong-mqtt.v1` |
|
||||||
|
| Unified topic | `vehicle.event.go.unified.v1` |
|
||||||
|
|
||||||
|
## 容器状态
|
||||||
|
|
||||||
|
ECS 上四个 Go 容器均已启动：
|
||||||
|
|
||||||
|
```text
|
||||||
|
go-vehicle-gateway Up
|
||||||
|
go-history-writer Up
|
||||||
|
go-stat-writer Up
|
||||||
|
go-realtime-api Up
|
||||||
|
```
|
||||||
|
|
||||||
|
## Kafka 验证
|
||||||
|
|
||||||
|
向 `18080` 注入 JT808 样例帧后，`vehicle.raw.go.jt808.v1` 和
|
||||||
|
`vehicle.event.go.unified.v1` 均消费到统一 JSON envelope。
|
||||||
|
|
||||||
|
样例解析结果包含：
|
||||||
|
|
||||||
|
- `protocol=JT808`
|
||||||
|
- `message_id=0x0200`
|
||||||
|
- `phone=013307795425`
|
||||||
|
- `longitude=121.069881`
|
||||||
|
- `latitude=30.590151`
|
||||||
|
- `speed_kmh=23`
|
||||||
|
- `total_mileage_km=10241.2`
|
||||||
|
- 表 27 附加项 `0x01` 原始值 `0001900C`
|
||||||
|
|
||||||
|
## TDengine 验证
|
||||||
|
|
||||||
|
目标库：`lingniu_vehicle_ts`
|
||||||
|
|
||||||
|
```text
|
||||||
|
raw_frames count = 3
|
||||||
|
vehicle_locations count = 2
|
||||||
|
vehicle_mileage_points count = 2
|
||||||
|
```
|
||||||
|
|
||||||
|
最新 RAW 标识：
|
||||||
|
|
||||||
|
```text
|
||||||
|
GB32960 | LNBSCB3D4R1234567 | LNBSCB3D4R1234567 |
|
||||||
|
JT808 | JT808:013307795425 | | 013307795425
|
||||||
|
JT808 | JT808:013307795425 | | 013307795425
|
||||||
|
```
|
||||||
|
|
||||||
|
说明：JT808 样例 phone 当前没有在 `vehicle_identity_binding` 中解析到 VIN，因此
|
||||||
|
TDengine tag 使用 `vehicle_key=JT808:013307795425`。
|
||||||
|
|
||||||
|
## Redis 验证
|
||||||
|
|
||||||
|
向 `23296` 注入带 VIN 的 GB32960 合成帧后，Realtime API 可查询：
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/realtime/vehicles/LNBSCB3D4R1234567
|
||||||
|
```
|
||||||
|
|
||||||
|
返回字段包含：
|
||||||
|
|
||||||
|
- `vin=LNBSCB3D4R1234567`
|
||||||
|
- `protocols=["GB32960"]`
|
||||||
|
- `online=true`
|
||||||
|
- `speed_kmh=30`
|
||||||
|
- `total_mileage_km=10000`
|
||||||
|
- `soc_percent=85`
|
||||||
|
- `longitude=121`
|
||||||
|
- `latitude=30.56`
|
||||||
|
|
||||||
|
## MySQL 统计验证
|
||||||
|
|
||||||
|
向 `23296` 注入带 VIN 的 GB32960 合成帧后，`vehicle_daily_metric` 写入：
|
||||||
|
|
||||||
|
```text
|
||||||
|
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_mileage_km | 0.000 | 10000.000 | 10000.000 | 1
|
||||||
|
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_total_mileage_km | 10000.000 | 10000.000 | 10000.000 | 1
|
||||||
|
```
|
||||||
|
|
||||||
|
说明：本次 GB32960 合成帧的设备时间为测试值，因此统计日期为 `2020-07-01`。
|
||||||
|
|
||||||
|
## 已修复问题
|
||||||
|
|
||||||
|
- TDengine WebSocket 普通 `INSERT` 使用参数占位符时，字符串未按预期加引号，导致语法错误。已在 `60ca42e` 中改为显式 SQL literal 转义。
|
||||||
|
- MySQL Go DSN 中 `charset=utf8mb4%2Cutf8` 会被 driver 传给 MySQL，导致语法错误；ECS 旁路 env 已改为 `charset=utf8mb4`。
|
||||||
|
- TDengine ECS 当前可用 root 密码与早先记录不一致，本次旁路 env 已按实际可认证值修正。
|
||||||
|
|
||||||
|
## 未完成
|
||||||
|
|
||||||
|
- 未切换真实生产端口 `32960/808`。
|
||||||
|
- 未将真实外部 32960/808 流量转发到 Go 旁路端口。
|
||||||
|
- 未开启宇通 MQTT 正式订阅。
|
||||||
|
- JT808 样例未解析到 VIN，需维护 `vehicle_identity_binding` 后再验证 808 每日统计。
|
||||||
|
|
||||||
|
## 2026-07-01 22:22 真实归档帧复验
|
||||||
|
|
||||||
|
部署版本已更新：
|
||||||
|
|
||||||
|
- Git commit：`5f4f4fd`
|
||||||
|
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5f4f4fd-20260701222128`
|
||||||
|
|
||||||
|
本轮补充验证内容：
|
||||||
|
|
||||||
|
- JT808 支持 2019 版 versioned header，真实帧 `0x0200` 可解析出 `phone=14894135060`、`longitude=117.178187`、`latitude=30.570155`、`speed_kmh=33.4`、`total_mileage_km=7868.9`。
|
||||||
|
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 可解析 `0x01` 整车、`0x02` 驱动电机、`0x03` 燃料电池、`0x04` 发动机、`0x05` 位置、`0x06` 极值、`0x07` 报警、`0x08` 电压、`0x09` 温度；后续 `0x30` 广东燃料电池扩展暂以 unknown raw 保留。
|
||||||
|
- GB32960 真实帧字段包含 `fuel_cell_hydrogen_consumption_kg_per_100km=1.8`、`longitude=120.800326`、`latitude=31.634907`、`total_mileage_km=53490.9`。
|
||||||
|
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回上述经纬度和氢耗字段。
|
||||||
|
- TDengine `vehicle_locations` 和 `vehicle_mileage_points` 已写入真实 GB32960 位置和里程点：
|
||||||
|
|
||||||
|
```text
|
||||||
|
2020-07-01 08:09:22 | GB32960 | LB9A32A21R0LS1707 | lon=120.800326 | lat=31.634907 | mileage=53490.9
|
||||||
|
```
|
||||||
|
|
||||||
|
仍需后续补齐：
|
||||||
|
|
||||||
|
- GB32960 广东燃料电池扩展 `0x30` 及后续 vendor block 的结构化解析。
|
||||||
|
- JT808 真实 phone `14894135060` 仍未解析到 VIN，需维护身份绑定后再验证 VIN 维度实时和统计。
|
||||||
129
docs/operations/go-vehicle-gateway-verification.md
Normal file
129
docs/operations/go-vehicle-gateway-verification.md
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
# Go Vehicle Gateway 验证手册
|
||||||
|
|
||||||
|
本文档用于验证 Go 重构运行面：
|
||||||
|
|
||||||
|
- `go-vehicle-gateway`：GB32960 TCP、JT808 TCP、宇通 MQTT 接入。
|
||||||
|
- `go-history-writer`：Kafka RAW 写 TDengine。
|
||||||
|
- `go-stat-writer`：Kafka RAW 写 MySQL 每日指标。
|
||||||
|
- `go-realtime-api`：Kafka unified event 写 Redis，并提供实时查询 API。
|
||||||
|
|
||||||
|
## 本地构建验证
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./...
|
||||||
|
go build ./cmd/gateway
|
||||||
|
go build ./cmd/history-writer
|
||||||
|
go build ./cmd/stat-writer
|
||||||
|
go build ./cmd/realtime-api
|
||||||
|
```
|
||||||
|
|
||||||
|
## 镜像构建
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
docker build -t vehicle-gateway-go:local .
|
||||||
|
```
|
||||||
|
|
||||||
|
同一个镜像包含四个二进制：
|
||||||
|
|
||||||
|
- `/app/gateway`
|
||||||
|
- `/app/history-writer`
|
||||||
|
- `/app/stat-writer`
|
||||||
|
- `/app/realtime-api`
|
||||||
|
|
||||||
|
## Portainer 部署
|
||||||
|
|
||||||
|
Go 旁路 stack 文件：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
deploy/portainer/docker-compose-go.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
必填变量：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
LINGNIU_GO_IMAGE_VERSION=<image-version>
|
||||||
|
KAFKA_BROKERS=172.17.111.56:9092
|
||||||
|
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
|
||||||
|
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
|
||||||
|
IDENTITY_MYSQL_DSN=${MYSQL_DSN}
|
||||||
|
VEHICLE_IDENTITY_TABLE=vehicle_identity_binding
|
||||||
|
REDIS_ADDR=r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379
|
||||||
|
REDIS_PASSWORD=<password>
|
||||||
|
REDIS_DB=50
|
||||||
|
```
|
||||||
|
|
||||||
|
宇通 MQTT 开启时再配置：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
YUTONG_MQTT_ENABLED=true
|
||||||
|
YUTONG_MQTT_URI=<mqtt-uri>
|
||||||
|
YUTONG_MQTT_TOPICS=/ytforward/shln/+
|
||||||
|
YUTONG_MQTT_CLIENT_ID=lingniu-go-yutong-mqtt
|
||||||
|
YUTONG_MQTT_USERNAME=<username>
|
||||||
|
YUTONG_MQTT_PASSWORD=<password>
|
||||||
|
```
|
||||||
|
|
||||||
|
## ECS 进程检查
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' \
|
||||||
|
| egrep 'go-vehicle-gateway|go-history-writer|go-stat-writer|go-realtime-api|NAMES'
|
||||||
|
|
||||||
|
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Kafka 验证
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||||
|
--topic vehicle.raw.jt808.v1 --max-messages 1 --timeout-ms 10000
|
||||||
|
|
||||||
|
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||||
|
--topic vehicle.raw.gb32960.v1 --max-messages 1 --timeout-ms 10000
|
||||||
|
|
||||||
|
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||||
|
--topic vehicle.event.unified.v1 --max-messages 1 --timeout-ms 10000
|
||||||
|
```
|
||||||
|
|
||||||
|
## TDengine 验证
|
||||||
|
|
||||||
|
```sql
|
||||||
|
USE lingniu_vehicle_ts;
|
||||||
|
SHOW STABLES;
|
||||||
|
SELECT COUNT(*) FROM raw_frames;
|
||||||
|
SELECT COUNT(*) FROM vehicle_locations;
|
||||||
|
SELECT COUNT(*) FROM vehicle_mileage_points;
|
||||||
|
```
|
||||||
|
|
||||||
|
## MySQL 验证
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT protocol, metric_key, COUNT(*)
|
||||||
|
FROM vehicle_daily_metric
|
||||||
|
GROUP BY protocol, metric_key;
|
||||||
|
|
||||||
|
SELECT *
|
||||||
|
FROM vehicle_daily_metric
|
||||||
|
WHERE metric_key IN ('daily_mileage_km', 'daily_total_mileage_km')
|
||||||
|
ORDER BY updated_at DESC
|
||||||
|
LIMIT 20;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Redis 实时查询
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>'
|
||||||
|
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/online'
|
||||||
|
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/protocols/JT808'
|
||||||
|
```
|
||||||
|
|
||||||
|
## 验收证据
|
||||||
|
|
||||||
|
正式切换前至少记录：
|
||||||
|
|
||||||
|
- 一个真实 GB32960 VIN 的 RAW、location、mileage point、Redis snapshot。
|
||||||
|
- 一个真实 JT808 VIN/phone 的 RAW、location、mileage point、daily metric、Redis snapshot。
|
||||||
|
- 一个真实宇通 MQTT VIN 或 device_id 的 RAW、Redis snapshot。
|
||||||
|
- `go-history-writer` 和 `go-stat-writer` 重启后 Kafka offset 能继续推进。
|
||||||
720
docs/superpowers/plans/2026-07-01-go-ingest-redesign-phase1.md
Normal file
720
docs/superpowers/plans/2026-07-01-go-ingest-redesign-phase1.md
Normal file
@@ -0,0 +1,720 @@
|
|||||||
|
# Go Vehicle Ingest Redesign Phase 1 Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Build the first production-capable Go runtime for GB32960, JT808, and Yutong MQTT ingestion with unified Kafka, TDengine, MySQL statistics, and Redis realtime state boundaries.
|
||||||
|
|
||||||
|
**Architecture:** Create a new single Go module at `go/vehicle-gateway` and migrate only the useful pieces from the existing `go/ingest-edge` and `go/vehicle-state` prototypes. The gateway produces unified envelopes to Kafka; independent consumers write TDengine history, MySQL daily metrics, and Redis realtime state.
|
||||||
|
|
||||||
|
**Tech Stack:** Go 1.26+, Kafka (`segmentio/kafka-go`), Redis (`redis/go-redis`), MySQL (`go-sql-driver/mysql`), TDengine official Go connector, MQTT (`eclipse/paho.mqtt.golang`), standard library TCP.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## File Structure
|
||||||
|
|
||||||
|
- Create: `go/vehicle-gateway/go.mod`
|
||||||
|
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||||
|
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||||
|
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||||
|
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/protocol/jt808/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/protocol/gb32960/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/protocol/yutongmqtt/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/gateway/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/identity/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/eventbus/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/history/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/stats/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/realtime/*`
|
||||||
|
- Create: `go/vehicle-gateway/internal/observability/*`
|
||||||
|
- Modify: `README.md`
|
||||||
|
- Modify: `docs/target-architecture.md`
|
||||||
|
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||||
|
|
||||||
|
The existing `go/ingest-edge` and `go/vehicle-state` directories are migration sources only. After phase 1 is verified, remove or mark them superseded.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Create Single Go Module
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/go.mod`
|
||||||
|
- Create: `go/vehicle-gateway/internal/observability/logger.go`
|
||||||
|
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Create module manifest**
|
||||||
|
|
||||||
|
Add `go/vehicle-gateway/go.mod`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||||
|
|
||||||
|
go 1.26
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||||
|
github.com/go-sql-driver/mysql v1.9.3
|
||||||
|
github.com/redis/go-redis/v9 v9.17.2
|
||||||
|
github.com/segmentio/kafka-go v0.4.49
|
||||||
|
github.com/taosdata/driver-go/v3 v3.8.1
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Add logger helper**
|
||||||
|
|
||||||
|
Add `go/vehicle-gateway/internal/observability/logger.go`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package observability
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewLogger(service string) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||||
|
return slog.New(handler).With("service", service)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Add temporary gateway entrypoint**
|
||||||
|
|
||||||
|
Add `go/vehicle-gateway/cmd/gateway/main.go`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
logger := observability.NewLogger("vehicle-gateway")
|
||||||
|
logger.Info("vehicle gateway scaffold started")
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify scaffold builds**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go mod tidy
|
||||||
|
go test ./...
|
||||||
|
go build ./cmd/gateway
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: all commands exit `0`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: Define Unified Envelope
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write envelope tests**
|
||||||
|
|
||||||
|
Add `go/vehicle-gateway/internal/envelope/envelope_test.go`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package envelope
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||||
|
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||||
|
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||||
|
t.Fatalf("VehicleKey() = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||||
|
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||||
|
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||||
|
t.Fatalf("VehicleKey() = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||||
|
e := FrameEnvelope{
|
||||||
|
Protocol: ProtocolJT808,
|
||||||
|
MessageID: "0x0200",
|
||||||
|
Phone: "013307795425",
|
||||||
|
Sequence: 1,
|
||||||
|
EventTimeMS: 1782745114000,
|
||||||
|
ReceivedAtMS: 1782745114999,
|
||||||
|
RawHex: "7e02000000ff7e",
|
||||||
|
}
|
||||||
|
a := e.StableEventID()
|
||||||
|
b := e.StableEventID()
|
||||||
|
if a == "" || a != b {
|
||||||
|
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run tests and confirm failure**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./internal/envelope
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: fail because `FrameEnvelope` is not defined.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Implement envelope**
|
||||||
|
|
||||||
|
Add `go/vehicle-gateway/internal/envelope/envelope.go`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package envelope
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Protocol string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ProtocolGB32960 Protocol = "GB32960"
|
||||||
|
ProtocolJT808 Protocol = "JT808"
|
||||||
|
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ParseStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ParseOK ParseStatus = "OK"
|
||||||
|
ParsePartial ParseStatus = "PARTIAL"
|
||||||
|
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||||
|
)
|
||||||
|
|
||||||
|
type FrameEnvelope struct {
|
||||||
|
EventID string `json:"event_id"`
|
||||||
|
TraceID string `json:"trace_id"`
|
||||||
|
Protocol Protocol `json:"protocol"`
|
||||||
|
MessageID string `json:"message_id"`
|
||||||
|
Sequence uint16 `json:"sequence"`
|
||||||
|
VIN string `json:"vin,omitempty"`
|
||||||
|
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||||
|
Phone string `json:"phone,omitempty"`
|
||||||
|
DeviceID string `json:"device_id,omitempty"`
|
||||||
|
Plate string `json:"plate,omitempty"`
|
||||||
|
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||||
|
EventTimeMS int64 `json:"event_time_ms"`
|
||||||
|
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||||
|
RawHex string `json:"raw_hex,omitempty"`
|
||||||
|
RawText string `json:"raw_text,omitempty"`
|
||||||
|
Parsed map[string]any `json:"parsed,omitempty"`
|
||||||
|
Fields map[string]any `json:"fields,omitempty"`
|
||||||
|
ParseStatus ParseStatus `json:"parse_status"`
|
||||||
|
ParseError string `json:"parse_error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e FrameEnvelope) VehicleKey() string {
|
||||||
|
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||||
|
return string(e.Protocol) + ":" + key
|
||||||
|
}
|
||||||
|
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||||
|
return string(e.Protocol) + ":" + key
|
||||||
|
}
|
||||||
|
return string(e.Protocol) + ":unknown"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e FrameEnvelope) StableEventID() string {
|
||||||
|
if strings.TrimSpace(e.EventID) != "" {
|
||||||
|
return e.EventID
|
||||||
|
}
|
||||||
|
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||||
|
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||||
|
sum := sha256.Sum256([]byte(input))
|
||||||
|
return hex.EncodeToString(sum[:16])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||||
|
if e.EventID == "" {
|
||||||
|
e.EventID = e.StableEventID()
|
||||||
|
}
|
||||||
|
if e.ParseStatus == "" {
|
||||||
|
e.ParseStatus = ParseOK
|
||||||
|
}
|
||||||
|
return json.Marshal(e)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify tests pass**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./internal/envelope
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: pass.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: Implement JT808 Frame and 0200 Parser
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser_test.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add sample frame tests**
|
||||||
|
|
||||||
|
Use the production sample provided in the thread:
|
||||||
|
|
||||||
|
```text
|
||||||
|
7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected assertions:
|
||||||
|
|
||||||
|
- message id is `0x0200`
|
||||||
|
- phone keeps normalized BCD string `013307795425`
|
||||||
|
- sequence is `1`
|
||||||
|
- `fields.total_mileage_km` is parsed from additional item `0x01`
|
||||||
|
- latitude, longitude, speed, direction, alarm, status, and device time are present
|
||||||
|
|
||||||
|
- [ ] **Step 2: Implement parser**
|
||||||
|
|
||||||
|
Implementation rules:
|
||||||
|
|
||||||
|
- strip `0x7e` start/end delimiters
|
||||||
|
- unescape `0x7d 0x02 -> 0x7e`
|
||||||
|
- unescape `0x7d 0x01 -> 0x7d`
|
||||||
|
- verify XOR checksum
|
||||||
|
- parse 2011/2013 common header
|
||||||
|
- parse BCD phone from 6 bytes and keep both raw and normalized values in `parsed.header`
|
||||||
|
- parse 0200 fixed body
|
||||||
|
- parse additional item list as `id,length,value_hex`
|
||||||
|
- parse additional `0x01` as `total_mileage_km = uint32 / 10`
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./internal/protocol/jt808
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: pass.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 4: Implement GB32960 Frame and Data Unit Parser
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser_test.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add parser tests**
|
||||||
|
|
||||||
|
Tests must cover:
|
||||||
|
|
||||||
|
- `##` frame boundary
|
||||||
|
- command id
|
||||||
|
- response flag
|
||||||
|
- 17-byte VIN
|
||||||
|
- encryption flag
|
||||||
|
- payload length
|
||||||
|
- BCC verification
|
||||||
|
- realtime data command `0x02`
|
||||||
|
- reissue data command `0x03`
|
||||||
|
- data unit `0x01` vehicle status fields
|
||||||
|
- data unit `0x05` position fields
|
||||||
|
|
||||||
|
- [ ] **Step 2: Implement parser**
|
||||||
|
|
||||||
|
Implementation rules:
|
||||||
|
|
||||||
|
- parse header without allocating large temporary buffers
|
||||||
|
- keep RAW as hex in envelope
|
||||||
|
- write all recognized data units to `Parsed`
|
||||||
|
- write core fields to `Fields`
|
||||||
|
- unsupported data unit stays in `Parsed["unknown_units"]`
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./internal/protocol/gb32960
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: pass.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 5: Implement Kafka Sink
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink_test.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add sink interface**
|
||||||
|
|
||||||
|
Create a sink interface:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package eventbus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Sink interface {
|
||||||
|
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||||
|
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Implement Kafka topic routing**
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- `GB32960 -> vehicle.raw.gb32960.v1`
|
||||||
|
- `JT808 -> vehicle.raw.jt808.v1`
|
||||||
|
- `YUTONG_MQTT -> vehicle.raw.yutong-mqtt.v1`
|
||||||
|
- unified topic is `vehicle.event.unified.v1`
|
||||||
|
- message key is `env.VehicleKey()`
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify routing with unit tests**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./internal/eventbus
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: pass.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 6: Implement TDengine History Writer
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/internal/history/schema.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/history/writer.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/history/writer_test.go`
|
||||||
|
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add schema bootstrap SQL**
|
||||||
|
|
||||||
|
Implement schema strings for:
|
||||||
|
|
||||||
|
- database `lingniu_vehicle_ts`
|
||||||
|
- stable `raw_frames`
|
||||||
|
- stable `vehicle_locations`
|
||||||
|
- stable `vehicle_mileage_points`
|
||||||
|
|
||||||
|
- [ ] **Step 2: Implement writer**
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- `AppendRawFrame` always writes one raw row.
|
||||||
|
- `AppendLocation` writes only when longitude and latitude exist.
|
||||||
|
- `AppendMileagePoint` writes only when `total_mileage_km` exists.
|
||||||
|
- child table name is deterministic hash of `protocol + vehicle_key`.
|
||||||
|
- escape tag values.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Use TDengine official driver**
|
||||||
|
|
||||||
|
Import WebSocket driver in command:
|
||||||
|
|
||||||
|
```go
|
||||||
|
import _ "github.com/taosdata/driver-go/v3/taosWS"
|
||||||
|
```
|
||||||
|
|
||||||
|
Default driver name:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TDENGINE_DRIVER=taosWS
|
||||||
|
```
|
||||||
|
|
||||||
|
Short-term compatibility:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TDENGINE_DRIVER=taosSql
|
||||||
|
```
|
||||||
|
|
||||||
|
Only use `taosSql` when ECS TDengine WebSocket is not available.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./internal/history
|
||||||
|
go build ./cmd/history-writer
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: pass.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 7: Implement MySQL Daily Metric Writer
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/internal/stats/schema.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/stats/daily_metric.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/stats/daily_metric_test.go`
|
||||||
|
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add schema bootstrap**
|
||||||
|
|
||||||
|
Implement `vehicle_daily_metric` schema from the design spec.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Add metric derivation tests**
|
||||||
|
|
||||||
|
Test cases:
|
||||||
|
|
||||||
|
- no `total_mileage_km` produces no metric
|
||||||
|
- one sample produces:
|
||||||
|
- `daily_mileage_km = 0`
|
||||||
|
- `daily_total_mileage_km = sample`
|
||||||
|
- later larger sample updates:
|
||||||
|
- `latest_total_mileage_km`
|
||||||
|
- `daily_mileage_km`
|
||||||
|
- `daily_total_mileage_km`
|
||||||
|
- out-of-order smaller sample updates:
|
||||||
|
- `first_total_mileage_km`
|
||||||
|
- `daily_mileage_km`
|
||||||
|
|
||||||
|
- [ ] **Step 3: Implement MySQL upsert**
|
||||||
|
|
||||||
|
Use one table and one idempotent upsert:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
INSERT INTO vehicle_daily_metric
|
||||||
|
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||||
|
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
first_total_mileage_km = LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)),
|
||||||
|
latest_total_mileage_km = GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)),
|
||||||
|
metric_value = CASE
|
||||||
|
WHEN metric_key = 'daily_mileage_km'
|
||||||
|
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||||
|
- LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||||
|
WHEN metric_key = 'daily_total_mileage_km'
|
||||||
|
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||||
|
ELSE VALUES(metric_value)
|
||||||
|
END,
|
||||||
|
sample_count = sample_count + 1,
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./internal/stats
|
||||||
|
go build ./cmd/stat-writer
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: pass.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 8: Implement Redis Realtime API
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/internal/realtime/repository.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/realtime/repository_test.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/realtime/http.go`
|
||||||
|
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add repository contract**
|
||||||
|
|
||||||
|
Methods:
|
||||||
|
|
||||||
|
- `Update(ctx, envelope.FrameEnvelope) error`
|
||||||
|
- `GetMerged(ctx, vin string) (Snapshot, error)`
|
||||||
|
- `GetProtocol(ctx, vin string, protocol envelope.Protocol) (Snapshot, error)`
|
||||||
|
- `IsOnline(ctx, vin string) (OnlineStatus, error)`
|
||||||
|
|
||||||
|
- [ ] **Step 2: Implement merge logic**
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- update `vehicle:latest:{vin}:{protocol}`
|
||||||
|
- update `vehicle:latest:{vin}` with newest fields
|
||||||
|
- update `vehicle:online:{vin}`
|
||||||
|
- update sorted set `vehicle:last_seen`
|
||||||
|
|
||||||
|
- [ ] **Step 3: Implement HTTP API**
|
||||||
|
|
||||||
|
Routes:
|
||||||
|
|
||||||
|
- `GET /api/realtime/vehicles/{vin}`
|
||||||
|
- `GET /api/realtime/vehicles/{vin}/online`
|
||||||
|
- `GET /api/realtime/vehicles/{vin}/protocols/{protocol}`
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
go test ./internal/realtime
|
||||||
|
go build ./cmd/realtime-api
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: pass.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 9: Wire Gateway Runtime
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/internal/gateway/tcp_server.go`
|
||||||
|
- Create: `go/vehicle-gateway/internal/gateway/mqtt_client.go`
|
||||||
|
- Modify: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Implement TCP server**
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- one goroutine per accepted connection
|
||||||
|
- bounded max connections
|
||||||
|
- read timeout and idle timeout
|
||||||
|
- protocol-specific frame extractor
|
||||||
|
- structured peer endpoint
|
||||||
|
- graceful shutdown on SIGTERM
|
||||||
|
|
||||||
|
- [ ] **Step 2: Implement MQTT client**
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- connect with official production config from environment
|
||||||
|
- subscribe configured topic list
|
||||||
|
- convert each message into envelope
|
||||||
|
- publish raw and unified events
|
||||||
|
- reconnect with backoff
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify local JSON mode**
|
||||||
|
|
||||||
|
Run without Kafka:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
GB32960_TCP_ADDR=:132960 JT808_TCP_ADDR=:18080 go run ./cmd/gateway
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: service starts and logs configured listeners.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 10: Docker and ECS Deployment
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `go/vehicle-gateway/Dockerfile`
|
||||||
|
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||||
|
- Modify: `docs/operations/current-ecs-deployment.md`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add multi-stage Dockerfile**
|
||||||
|
|
||||||
|
Build all commands:
|
||||||
|
|
||||||
|
- `gateway`
|
||||||
|
- `history-writer`
|
||||||
|
- `stat-writer`
|
||||||
|
- `realtime-api`
|
||||||
|
|
||||||
|
- [ ] **Step 2: Add Portainer compose**
|
||||||
|
|
||||||
|
Services:
|
||||||
|
|
||||||
|
- `go-vehicle-gateway`
|
||||||
|
- `go-history-writer`
|
||||||
|
- `go-stat-writer`
|
||||||
|
- `go-realtime-api`
|
||||||
|
|
||||||
|
Each service must include:
|
||||||
|
|
||||||
|
- restart policy
|
||||||
|
- memory limit
|
||||||
|
- Kafka env
|
||||||
|
- MySQL/TDengine/Redis env as needed
|
||||||
|
- logging options
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify Linux build**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd go/vehicle-gateway
|
||||||
|
GOOS=linux GOARCH=amd64 go build ./cmd/gateway
|
||||||
|
GOOS=linux GOARCH=amd64 go build ./cmd/history-writer
|
||||||
|
GOOS=linux GOARCH=amd64 go build ./cmd/stat-writer
|
||||||
|
GOOS=linux GOARCH=amd64 go build ./cmd/realtime-api
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: all commands exit `0`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 11: Production Verification
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `docs/operations/go-vehicle-gateway-verification.md`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Record test commands**
|
||||||
|
|
||||||
|
Document commands to verify:
|
||||||
|
|
||||||
|
- gateway process health
|
||||||
|
- Kafka topic consumption
|
||||||
|
- TDengine row counts
|
||||||
|
- MySQL daily metric rows
|
||||||
|
- Redis realtime lookup
|
||||||
|
|
||||||
|
- [ ] **Step 2: Validate real traffic**
|
||||||
|
|
||||||
|
Evidence required:
|
||||||
|
|
||||||
|
- one real 32960 VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||||
|
- one real JT808 phone/VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||||
|
- one real Yutong MQTT VIN with RAW and Redis snapshot
|
||||||
|
|
||||||
|
- [ ] **Step 3: Keep old Java services until evidence is captured**
|
||||||
|
|
||||||
|
Only disable Java equivalents after the evidence file contains successful command outputs and timestamps.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Self-Review Checklist
|
||||||
|
|
||||||
|
- The plan creates one new Go module instead of extending scattered prototypes.
|
||||||
|
- The plan covers GB32960, JT808, and Yutong MQTT ingress.
|
||||||
|
- The plan covers Kafka, TDengine, MySQL, and Redis.
|
||||||
|
- The plan includes 32960 and 808 daily mileage and daily total mileage.
|
||||||
|
- The plan includes local and ECS verification.
|
||||||
|
- The plan does not restore Xinda Push.
|
||||||
|
- The plan keeps Java services untouched until Go evidence exists.
|
||||||
486
docs/superpowers/specs/2026-07-01-go-ingest-redesign-design.md
Normal file
486
docs/superpowers/specs/2026-07-01-go-ingest-redesign-design.md
Normal file
@@ -0,0 +1,486 @@
|
|||||||
|
# Go 车辆接入重构设计规格
|
||||||
|
|
||||||
|
## 目标
|
||||||
|
|
||||||
|
使用 Go 重新设计车辆数据接入运行面，让系统回到第一性原则：
|
||||||
|
|
||||||
|
- 接入层只负责可靠收包、协议解析、应答、身份解析和投递。
|
||||||
|
- Kafka 是唯一在线消息通道。
|
||||||
|
- TDengine 保存高吞吐时序明细和完整 RAW 解析 JSON。
|
||||||
|
- MySQL 保存低频配置、身份映射和每日统计窄表。
|
||||||
|
- Redis 保存可重建的准实时状态。
|
||||||
|
- 32960、JT808、宇通 MQTT 使用同一套数据层接口，避免每个协议各自落库。
|
||||||
|
|
||||||
|
第一阶段只做生产链路的骨架和核心数据闭环：接收、解析、Kafka、TDengine、MySQL 统计、Redis 实时查询、ECS 验证。历史查询 API 和业务侧复杂分析放在后续阶段。
|
||||||
|
|
||||||
|
## 参考原则
|
||||||
|
|
||||||
|
本设计只吸收开源项目的边界思想，不复制外部实现代码。
|
||||||
|
|
||||||
|
- GB32960 参考 `DarkInno/gb32960-go-sdk`：连接管理、Handler、Forwarder、鉴权接口分离。
|
||||||
|
- JT808 参考 `cuteLittleDevil/go-jt808`：高并发 Go 原生 TCP、协议核心小而可扩展、事件/适配器机制。
|
||||||
|
- JT808 参考 `fakeyanss/jt808-server-go`：`FramePayload -> PacketData -> JT808Msg -> Reply` 的分层处理、Gateway 模式、保活和版本兼容。
|
||||||
|
- TDengine 参考官方数据模型：按数据采集点建立 supertable，静态维度做 tags，明细数据按子表写入；Go 连接优先使用 WebSocket 驱动。
|
||||||
|
|
||||||
|
## 非目标
|
||||||
|
|
||||||
|
- 第一阶段不继续扩展 Java 服务。
|
||||||
|
- 第一阶段不做统一大查询 API。
|
||||||
|
- 第一阶段不恢复信达 Push。
|
||||||
|
- 第一阶段不把所有协议字段展开成一张超宽 TDengine 表。
|
||||||
|
- Redis 不作为历史事实源。
|
||||||
|
- MySQL 不保存高频原始明细。
|
||||||
|
|
||||||
|
## 目标目录
|
||||||
|
|
||||||
|
新的 Go 运行面放在 `go/vehicle-gateway`，不继续沿用实验性的多 module 分散结构。现有 `go/ingest-edge` 和 `go/vehicle-state` 可以作为迁移素材，最终归并或删除。
|
||||||
|
|
||||||
|
```text
|
||||||
|
go/vehicle-gateway/
|
||||||
|
cmd/
|
||||||
|
gateway/ # 协议接入进程：32960 TCP、JT808 TCP、宇通 MQTT
|
||||||
|
history-writer/ # Kafka -> TDengine RAW/location/mileage points
|
||||||
|
stat-writer/ # Kafka -> MySQL 每日统计窄表
|
||||||
|
realtime-api/ # Redis 准实时查询 API
|
||||||
|
internal/
|
||||||
|
config/ # 环境变量、配置文件、校验
|
||||||
|
gateway/ # TCP/MQTT 生命周期、连接限制、优雅停机
|
||||||
|
protocol/
|
||||||
|
gb32960/ # 32960 编解码
|
||||||
|
jt808/ # 808 编解码
|
||||||
|
yutongmqtt/ # 宇通 MQTT payload 解析
|
||||||
|
identity/ # VIN、phone、device_id、plate 绑定
|
||||||
|
envelope/ # 统一事件模型
|
||||||
|
eventbus/ # Kafka producer/consumer
|
||||||
|
history/ # TDengine adapter
|
||||||
|
stats/ # 每日里程聚合
|
||||||
|
realtime/ # Redis adapter 与快照合并
|
||||||
|
observability/ # 日志、metrics、健康检查
|
||||||
|
```
|
||||||
|
|
||||||
|
## 协议接入设计
|
||||||
|
|
||||||
|
### GB32960
|
||||||
|
|
||||||
|
职责：
|
||||||
|
|
||||||
|
- 监听 TCP `32960`。
|
||||||
|
- 支持车辆登入、实时信息上报、补发信息上报、车辆登出、平台登入/登出、心跳、校时。
|
||||||
|
- 正确返回协议应答。
|
||||||
|
- 完整解析 2016 数据单元：
|
||||||
|
- 整车数据
|
||||||
|
- 驱动电机数据
|
||||||
|
- 燃料电池数据
|
||||||
|
- 发动机数据
|
||||||
|
- 车辆位置数据
|
||||||
|
- 极值数据
|
||||||
|
- 报警数据
|
||||||
|
- 可充电储能装置电压数据
|
||||||
|
- 可充电储能装置温度数据
|
||||||
|
- RAW 中保存完整 parsed JSON。
|
||||||
|
- 关键字段标准化为统一 field：
|
||||||
|
- `speed_kmh`
|
||||||
|
- `total_mileage_km`
|
||||||
|
- `soc_percent`
|
||||||
|
- `longitude`
|
||||||
|
- `latitude`
|
||||||
|
- `vehicle_status`
|
||||||
|
- `charge_status`
|
||||||
|
|
||||||
|
### JT808
|
||||||
|
|
||||||
|
职责：
|
||||||
|
|
||||||
|
- 监听 TCP `808`。
|
||||||
|
- 支持 2011/2013 基础兼容，预留 2019 版本标记。
|
||||||
|
- 分层处理：
|
||||||
|
- Frame：`0x7e` 包边界、转义、校验。
|
||||||
|
- Packet：消息头、手机号 BCD、流水号、分包。
|
||||||
|
- Message：注册、鉴权、心跳、注销、位置、批量位置、未知上行。
|
||||||
|
- Reply：注册应答、平台通用应答。
|
||||||
|
- 终端手机号按协议头 BCD 解析，内部同时保留原始 BCD、规范化 phone、去前导零 phone。
|
||||||
|
- 注册帧和鉴权帧写入 808 registration 表。
|
||||||
|
- 如果设备直接上报位置帧，也要走 identity 解析，尝试用 phone、device_id、plate 找到 VIN。
|
||||||
|
- 0200 附加信息必须解析表 27：
|
||||||
|
- `0x01` GPS 总里程，DWORD，单位 0.1 km。
|
||||||
|
- 其他已知附加项结构化放入 `parsed_json.additional`。
|
||||||
|
- 标准化字段：
|
||||||
|
- `speed_kmh`
|
||||||
|
- `total_mileage_km`
|
||||||
|
- `longitude`
|
||||||
|
- `latitude`
|
||||||
|
- `altitude_m`
|
||||||
|
- `direction_deg`
|
||||||
|
- `alarm_flag`
|
||||||
|
- `status_flag`
|
||||||
|
|
||||||
|
### 宇通 MQTT
|
||||||
|
|
||||||
|
职责：
|
||||||
|
|
||||||
|
- 使用正式 MQTT 配置订阅生产 topic。
|
||||||
|
- 接收 payload 后解析为统一 envelope。
|
||||||
|
- 按 payload 中的车辆标识解析 VIN、车牌、设备号。
|
||||||
|
- 保存完整 payload 和 parsed JSON。
|
||||||
|
- 标准化字段向 32960/808 对齐：
|
||||||
|
- `speed_kmh`
|
||||||
|
- `total_mileage_km`
|
||||||
|
- `longitude`
|
||||||
|
- `latitude`
|
||||||
|
- `soc_percent`
|
||||||
|
- `vehicle_status`
|
||||||
|
|
||||||
|
## 统一 Envelope
|
||||||
|
|
||||||
|
所有协议接入后输出同一结构。
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"event_id": "string",
|
||||||
|
"trace_id": "string",
|
||||||
|
"protocol": "GB32960|JT808|YUTONG_MQTT",
|
||||||
|
"message_id": "string",
|
||||||
|
"vin": "string",
|
||||||
|
"vehicle_key": "string",
|
||||||
|
"phone": "string",
|
||||||
|
"device_id": "string",
|
||||||
|
"plate": "string",
|
||||||
|
"source_endpoint": "ip:port or mqtt://broker/topic",
|
||||||
|
"event_time_ms": 0,
|
||||||
|
"received_at_ms": 0,
|
||||||
|
"raw_hex": "string",
|
||||||
|
"raw_text": "string",
|
||||||
|
"parsed": {},
|
||||||
|
"fields": {
|
||||||
|
"speed_kmh": 0,
|
||||||
|
"total_mileage_km": 0,
|
||||||
|
"longitude": 0,
|
||||||
|
"latitude": 0
|
||||||
|
},
|
||||||
|
"parse_status": "OK|PARTIAL|BAD_FRAME",
|
||||||
|
"parse_error": ""
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
规则：
|
||||||
|
|
||||||
|
- `event_id` 使用协议、设备标识、消息流水、事件时间、RAW hash 生成，保证幂等。
|
||||||
|
- Kafka partition key 优先使用 VIN；没有 VIN 时使用 `protocol:phone/device_id`。
|
||||||
|
- `parsed` 保存协议完整结构化字段。
|
||||||
|
- `fields` 只保存跨协议核心字段，用于统计、位置和实时快照。
|
||||||
|
|
||||||
|
## Kafka Topic
|
||||||
|
|
||||||
|
生产只支持 Kafka。
|
||||||
|
|
||||||
|
| Topic | 内容 | Key |
|
||||||
|
|---|---|---|
|
||||||
|
| `vehicle.raw.gb32960.v1` | 32960 完整 RAW envelope | VIN 或 vehicle_key |
|
||||||
|
| `vehicle.raw.jt808.v1` | 808 完整 RAW envelope | VIN 或 phone |
|
||||||
|
| `vehicle.raw.yutong-mqtt.v1` | 宇通 MQTT 完整 RAW envelope | VIN 或 vehicle_key |
|
||||||
|
| `vehicle.event.unified.v1` | 统一轻量事件，用于业务消费 | VIN 或 vehicle_key |
|
||||||
|
|
||||||
|
接入层必须先写 RAW topic，再写 unified event。后续消费者只依赖 Kafka，不直接依赖接入进程内存。
|
||||||
|
|
||||||
|
## TDengine 数据库设计
|
||||||
|
|
||||||
|
如果现有 TDengine 库不符合目标，可以新建库并迁移。建议库名：
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE DATABASE IF NOT EXISTS lingniu_vehicle_ts KEEP 7300 DURATION 10 BUFFER 256;
|
||||||
|
```
|
||||||
|
|
||||||
|
### raw_frames
|
||||||
|
|
||||||
|
保存完整 RAW 和完整 parsed JSON。
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||||
|
ts TIMESTAMP,
|
||||||
|
frame_id NCHAR(64),
|
||||||
|
event_id NCHAR(64),
|
||||||
|
message_id INT,
|
||||||
|
event_time TIMESTAMP,
|
||||||
|
received_at TIMESTAMP,
|
||||||
|
raw_size_bytes INT,
|
||||||
|
raw_hex BINARY(16374),
|
||||||
|
raw_text BINARY(16374),
|
||||||
|
parsed_json BINARY(16374),
|
||||||
|
fields_json BINARY(4096),
|
||||||
|
parse_status NCHAR(16),
|
||||||
|
parse_error BINARY(1024),
|
||||||
|
source_endpoint NCHAR(128)
|
||||||
|
) TAGS (
|
||||||
|
protocol NCHAR(32),
|
||||||
|
vehicle_key NCHAR(64),
|
||||||
|
vin NCHAR(32),
|
||||||
|
phone NCHAR(32),
|
||||||
|
device_id NCHAR(64)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
说明：
|
||||||
|
|
||||||
|
- 子表按 `protocol + vehicle_key hash` 创建。
|
||||||
|
- RAW 查询优先按 tags 和时间范围过滤。
|
||||||
|
- `parsed_json` 是完整协议字段；查询 API 可选择是否返回，避免默认大字段拖慢。
|
||||||
|
|
||||||
|
### vehicle_locations
|
||||||
|
|
||||||
|
只保存位置查询核心字段。
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||||
|
ts TIMESTAMP,
|
||||||
|
event_id NCHAR(64),
|
||||||
|
frame_id NCHAR(64),
|
||||||
|
received_at TIMESTAMP,
|
||||||
|
longitude DOUBLE,
|
||||||
|
latitude DOUBLE,
|
||||||
|
altitude_m DOUBLE,
|
||||||
|
speed_kmh DOUBLE,
|
||||||
|
direction_deg INT,
|
||||||
|
alarm_flag BIGINT,
|
||||||
|
status_flag BIGINT,
|
||||||
|
total_mileage_km DOUBLE
|
||||||
|
) TAGS (
|
||||||
|
protocol NCHAR(32),
|
||||||
|
vehicle_key NCHAR(64),
|
||||||
|
vin NCHAR(32),
|
||||||
|
phone NCHAR(32),
|
||||||
|
device_id NCHAR(64)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
### vehicle_mileage_points
|
||||||
|
|
||||||
|
保存总里程采样点，用于回放重算和查询。
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||||
|
ts TIMESTAMP,
|
||||||
|
event_id NCHAR(64),
|
||||||
|
frame_id NCHAR(64),
|
||||||
|
received_at TIMESTAMP,
|
||||||
|
total_mileage_km DOUBLE,
|
||||||
|
speed_kmh DOUBLE,
|
||||||
|
longitude DOUBLE,
|
||||||
|
latitude DOUBLE
|
||||||
|
) TAGS (
|
||||||
|
protocol NCHAR(32),
|
||||||
|
vehicle_key NCHAR(64),
|
||||||
|
vin NCHAR(32),
|
||||||
|
phone NCHAR(32),
|
||||||
|
device_id NCHAR(64)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
## MySQL 数据库设计
|
||||||
|
|
||||||
|
MySQL 只保存配置、身份映射、808 注册鉴权和每日统计窄表。
|
||||||
|
|
||||||
|
### vehicle_identity_binding
|
||||||
|
|
||||||
|
用于把外部标识定位到 VIN。
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE vehicle_identity_binding (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
vin VARCHAR(32) NOT NULL,
|
||||||
|
plate VARCHAR(32) NULL,
|
||||||
|
phone VARCHAR(32) NULL,
|
||||||
|
device_id VARCHAR(64) NULL,
|
||||||
|
remark VARCHAR(255) NULL,
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
UNIQUE KEY uk_vin (vin),
|
||||||
|
KEY idx_plate (plate),
|
||||||
|
KEY idx_phone (phone),
|
||||||
|
KEY idx_device_id (device_id)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
### jt808_registration
|
||||||
|
|
||||||
|
只记录 808 独有的注册、鉴权和在线身份信息。
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE jt808_registration (
|
||||||
|
phone VARCHAR(32) PRIMARY KEY,
|
||||||
|
vin VARCHAR(32) NULL,
|
||||||
|
plate VARCHAR(32) NULL,
|
||||||
|
device_id VARCHAR(64) NULL,
|
||||||
|
manufacturer_id VARCHAR(32) NULL,
|
||||||
|
terminal_model VARCHAR(64) NULL,
|
||||||
|
terminal_id VARCHAR(64) NULL,
|
||||||
|
auth_code VARCHAR(128) NULL,
|
||||||
|
source_endpoint VARCHAR(128) NULL,
|
||||||
|
first_registered_at DATETIME NULL,
|
||||||
|
latest_registered_at DATETIME NULL,
|
||||||
|
latest_authed_at DATETIME NULL,
|
||||||
|
latest_seen_at DATETIME NULL,
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
KEY idx_vin (vin),
|
||||||
|
KEY idx_plate (plate),
|
||||||
|
KEY idx_device_id (device_id)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
### vehicle_daily_metric
|
||||||
|
|
||||||
|
32960 和 808 的每日里程、每日总里程都进入同一张窄表。
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE vehicle_daily_metric (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
vin VARCHAR(32) NOT NULL,
|
||||||
|
stat_date DATE NOT NULL,
|
||||||
|
protocol VARCHAR(32) NOT NULL,
|
||||||
|
metric_key VARCHAR(64) NOT NULL,
|
||||||
|
metric_value DECIMAL(18,3) NOT NULL,
|
||||||
|
metric_unit VARCHAR(16) NOT NULL,
|
||||||
|
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||||
|
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||||
|
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
calculation_method VARCHAR(64) NOT NULL,
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
|
||||||
|
KEY idx_stat_date (stat_date),
|
||||||
|
KEY idx_protocol_metric (protocol, metric_key)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
指标：
|
||||||
|
|
||||||
|
- `daily_mileage_km = max(total_mileage_km) - min(total_mileage_km)`
|
||||||
|
- `daily_total_mileage_km = max(total_mileage_km)`
|
||||||
|
|
||||||
|
统计消费者可幂等重放。每个样本 upsert 时只维护当天 `min/max total_mileage_km`，不依赖进程内状态。
|
||||||
|
|
||||||
|
## Redis 实时数据设计
|
||||||
|
|
||||||
|
Redis 保存可从 Kafka 重建的数据。
|
||||||
|
|
||||||
|
| Key | 内容 |
|
||||||
|
|---|---|
|
||||||
|
| `vehicle:online:{vin}` | 在线状态、最后活跃时间、协议列表 |
|
||||||
|
| `vehicle:latest:{vin}` | 合并后的 VIN 最新快照 |
|
||||||
|
| `vehicle:latest:{vin}:{protocol}` | 单协议最新快照 |
|
||||||
|
| `vehicle:last_seen` | VIN 到最后接收时间的 sorted set |
|
||||||
|
|
||||||
|
合并规则：
|
||||||
|
|
||||||
|
- 位置以最新 event_time 为准。
|
||||||
|
- 总里程取最新上报值，不做递增假设。
|
||||||
|
- 32960 多帧字段按 field timestamp 合并。
|
||||||
|
- 808 和 MQTT 不覆盖其他协议独有字段。
|
||||||
|
- Redis TTL 用于在线判断，不作为数据删除依据。
|
||||||
|
|
||||||
|
## 数据流
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
source["车辆 / 平台"] --> gateway["Go gateway"]
|
||||||
|
gateway --> parser["protocol parser"]
|
||||||
|
parser --> identity["identity resolver"]
|
||||||
|
identity --> kafkaRaw["Kafka RAW topics"]
|
||||||
|
identity --> kafkaEvent["Kafka unified event"]
|
||||||
|
kafkaRaw --> history["history-writer"]
|
||||||
|
kafkaRaw --> stats["stat-writer"]
|
||||||
|
kafkaEvent --> realtime["realtime consumer"]
|
||||||
|
history --> td["TDengine raw_frames / locations / mileage_points"]
|
||||||
|
stats --> mysql["MySQL vehicle_daily_metric"]
|
||||||
|
realtime --> redis["Redis latest state"]
|
||||||
|
```
|
||||||
|
|
||||||
|
## 错误处理
|
||||||
|
|
||||||
|
- 协议坏帧仍写 RAW topic，`parse_status=BAD_FRAME`。
|
||||||
|
- 可部分解析的帧写 `parse_status=PARTIAL`，保留 parse error。
|
||||||
|
- Kafka 写失败时接入层不确认业务成功；协议是否应答按协议要求处理，但必须打错误日志和 metrics。
|
||||||
|
- TDengine 写失败不提交 Kafka offset。
|
||||||
|
- MySQL 统计写失败不提交 Kafka offset。
|
||||||
|
- Redis 写失败不影响历史和统计，但要通过 metrics 暴露。
|
||||||
|
- 808 相同 phone 新连接时，旧连接应被替换并记录 latest_seen。
|
||||||
|
|
||||||
|
## 部署设计
|
||||||
|
|
||||||
|
生产 ECS 运行 Go 二进制或容器：
|
||||||
|
|
||||||
|
- `vehicle-gateway`
|
||||||
|
- `vehicle-history-writer`
|
||||||
|
- `vehicle-stat-writer`
|
||||||
|
- `vehicle-realtime-api`
|
||||||
|
|
||||||
|
配置来源：
|
||||||
|
|
||||||
|
- 环境变量优先。
|
||||||
|
- Nacos 可作为后续配置中心，但 Go 第一阶段必须能只靠环境变量启动，便于 Portainer 部署和故障恢复。
|
||||||
|
|
||||||
|
TDengine 连接：
|
||||||
|
|
||||||
|
- 优先 WebSocket `taosWS`。
|
||||||
|
- 若当前 ECS 只开放原生端口，可短期兼容 `taosSql`，但代码配置必须显式标记为兼容模式。
|
||||||
|
|
||||||
|
## 验证计划
|
||||||
|
|
||||||
|
### 本地验证
|
||||||
|
|
||||||
|
- 用样例 808 报文验证：
|
||||||
|
- phone BCD 解析。
|
||||||
|
- 0200 经纬度、速度、方向、时间。
|
||||||
|
- 表 27 `0x01` GPS 总里程。
|
||||||
|
- 用真实 32960 报文验证：
|
||||||
|
- 登录应答正确。
|
||||||
|
- 实时帧和补发帧都能写 RAW。
|
||||||
|
- 位置和总里程字段进入统一 fields。
|
||||||
|
- 用宇通 MQTT payload 验证：
|
||||||
|
- payload 完整保存。
|
||||||
|
- 核心字段归一化。
|
||||||
|
- 不连 Kafka 时可输出 JSON log。
|
||||||
|
- 连接 Kafka 时 RAW topic 可消费到 envelope。
|
||||||
|
|
||||||
|
### ECS 验证
|
||||||
|
|
||||||
|
- 部署前先旁路接收或短窗口切流。
|
||||||
|
- 验证端口：
|
||||||
|
- `32960`
|
||||||
|
- `808`
|
||||||
|
- MQTT 订阅连接。
|
||||||
|
- 验证 Kafka topic 有持续写入。
|
||||||
|
- 验证 TDengine：
|
||||||
|
- `raw_frames` 有三种协议数据。
|
||||||
|
- `vehicle_locations` 能按 VIN、协议、时间分页查询。
|
||||||
|
- `vehicle_mileage_points` 有 32960 和 808 总里程采样。
|
||||||
|
- 验证 MySQL：
|
||||||
|
- `vehicle_daily_metric` 有 32960 和 808 的 `daily_mileage_km`、`daily_total_mileage_km`。
|
||||||
|
- 验证 Redis：
|
||||||
|
- 查询 VIN 在线。
|
||||||
|
- 查询 VIN 合并实时快照。
|
||||||
|
- 查询 VIN 分协议实时快照。
|
||||||
|
|
||||||
|
## 验收标准
|
||||||
|
|
||||||
|
- Go gateway 能在 ECS 上接收真实 32960、808、宇通 MQTT 数据。
|
||||||
|
- 三种协议 RAW 都进入 Kafka。
|
||||||
|
- 三种协议 RAW 和 parsed JSON 都进入 TDengine。
|
||||||
|
- 32960 和 808 的位置进入 `vehicle_locations`。
|
||||||
|
- 32960 和 808 的总里程采样进入 `vehicle_mileage_points`。
|
||||||
|
- 32960 和 808 的每日里程、每日总里程进入 MySQL 窄表。
|
||||||
|
- Redis 可以查询 VIN 在线和实时数据。
|
||||||
|
- 任一消费者宕机后可通过 Kafka offset 继续恢复。
|
||||||
|
- 查询和统计验证使用 ECS 上的真实数据完成。
|
||||||
|
|
||||||
|
## 实施顺序
|
||||||
|
|
||||||
|
1. 创建 `go/vehicle-gateway` 单 module，迁移现有 Go 原型中可用的解析、Kafka、TDengine、MySQL、Redis 代码。
|
||||||
|
2. 先写协议解析单元测试，覆盖 808 样例报文和 32960 核心字段。
|
||||||
|
3. 完成统一 envelope 和 Kafka sink。
|
||||||
|
4. 完成 TDengine schema bootstrap 与 history writer。
|
||||||
|
5. 完成 MySQL schema bootstrap 与 stat writer。
|
||||||
|
6. 完成 Redis realtime writer/API。
|
||||||
|
7. 本地启动 gateway，用 ECS 转发流量验证。
|
||||||
|
8. 构建 Linux 镜像，部署 ECS。
|
||||||
|
9. 按验收标准逐项验证并记录证据。
|
||||||
29
go/vehicle-gateway/Dockerfile
Normal file
29
go/vehicle-gateway/Dockerfile
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
FROM golang:1.26-bookworm AS build
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/gateway ./cmd/gateway \
|
||||||
|
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/history-writer ./cmd/history-writer \
|
||||||
|
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/stat-writer ./cmd/stat-writer \
|
||||||
|
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/realtime-api ./cmd/realtime-api
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends ca-certificates tzdata \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY --from=build /out/gateway /app/gateway
|
||||||
|
COPY --from=build /out/history-writer /app/history-writer
|
||||||
|
COPY --from=build /out/stat-writer /app/stat-writer
|
||||||
|
COPY --from=build /out/realtime-api /app/realtime-api
|
||||||
|
|
||||||
|
ENV TZ=Asia/Shanghai
|
||||||
|
|
||||||
|
EXPOSE 808 32960 20210
|
||||||
|
|
||||||
|
CMD ["/app/gateway"]
|
||||||
195
go/vehicle-gateway/cmd/gateway/main.go
Normal file
195
go/vehicle-gateway/cmd/gateway/main.go
Normal file
@@ -0,0 +1,195 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "github.com/go-sql-driver/mysql"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/gateway"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
logger := observability.NewLogger("vehicle-gateway")
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
sink, err := buildSink(logger)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("build sink failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
defer sink.Close()
|
||||||
|
resolver, closeResolver, err := buildIdentityResolver(ctx, logger)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("build identity resolver failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
defer closeResolver()
|
||||||
|
|
||||||
|
protocols := []gateway.TCPProtocol{
|
||||||
|
{
|
||||||
|
Protocol: envelope.ProtocolGB32960,
|
||||||
|
Addr: env("GB32960_TCP_ADDR", ":32960"),
|
||||||
|
Extract: gb32960.ExtractFrames,
|
||||||
|
Parse: gb32960.ParseFrame,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
Addr: env("JT808_TCP_ADDR", ":808"),
|
||||||
|
Extract: jt808.ExtractFrames,
|
||||||
|
Parse: jt808.ParseFrame,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errs := make(chan error, len(protocols))
|
||||||
|
for _, protocol := range protocols {
|
||||||
|
server, err := gateway.NewTCPServer(gateway.TCPServerConfig{
|
||||||
|
Protocol: protocol,
|
||||||
|
Sink: sink,
|
||||||
|
Resolver: resolver,
|
||||||
|
Logger: logger,
|
||||||
|
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
|
||||||
|
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
|
||||||
|
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if err := server.ListenAndServe(ctx); err != nil {
|
||||||
|
errs <- err
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
if envBool("YUTONG_MQTT_ENABLED", false) {
|
||||||
|
client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{
|
||||||
|
EndpointName: env("YUTONG_MQTT_ENDPOINT", "yutong"),
|
||||||
|
Broker: env("YUTONG_MQTT_URI", ""),
|
||||||
|
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
|
||||||
|
Username: env("YUTONG_MQTT_USERNAME", ""),
|
||||||
|
Password: env("YUTONG_MQTT_PASSWORD", ""),
|
||||||
|
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
|
||||||
|
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
|
||||||
|
Sink: sink,
|
||||||
|
Resolver: resolver,
|
||||||
|
Logger: logger,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("build yutong mqtt client failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
if err := client.Start(ctx); err != nil {
|
||||||
|
logger.Error("start yutong mqtt client failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
logger.Info("yutong mqtt client started")
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("vehicle gateway started")
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case err := <-errs:
|
||||||
|
logger.Error("vehicle gateway listener failed", "error", err)
|
||||||
|
stop()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.Resolver, func(), error) {
|
||||||
|
dsn := env("IDENTITY_MYSQL_DSN", strings.TrimSpace(os.Getenv("MYSQL_DSN")))
|
||||||
|
if strings.TrimSpace(dsn) == "" {
|
||||||
|
logger.Warn("identity mysql dsn is empty; using noop identity resolver")
|
||||||
|
return identity.NoopResolver{}, func() {}, nil
|
||||||
|
}
|
||||||
|
db, err := sql.Open("mysql", dsn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if err := db.PingContext(ctx); err != nil {
|
||||||
|
_ = db.Close()
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding")
|
||||||
|
logger.Info("identity mysql resolver enabled", "table", table)
|
||||||
|
return identity.NewMySQLResolver(db, table), func() { _ = db.Close() }, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildSink(logger *slog.Logger) (eventbus.Sink, error) {
|
||||||
|
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
|
||||||
|
if len(brokers) == 0 {
|
||||||
|
logger.Warn("KAFKA_BROKERS is empty; using log sink")
|
||||||
|
return eventbus.NewLogSink(logger), nil
|
||||||
|
}
|
||||||
|
return eventbus.NewKafkaSink(eventbus.KafkaConfig{
|
||||||
|
Brokers: brokers,
|
||||||
|
RawTopics: map[envelope.Protocol]string{
|
||||||
|
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1"),
|
||||||
|
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1"),
|
||||||
|
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1"),
|
||||||
|
},
|
||||||
|
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(key string, fallback string) string {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func envInt(key string, fallback int) int {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
var out int
|
||||||
|
for _, r := range value {
|
||||||
|
if r < '0' || r > '9' {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
out = out*10 + int(r-'0')
|
||||||
|
}
|
||||||
|
if out <= 0 {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func envBool(key string, fallback bool) bool {
|
||||||
|
value := strings.ToLower(strings.TrimSpace(os.Getenv(key)))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return value == "1" || value == "true" || value == "yes" || value == "on"
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitCSV(value string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, item := range strings.Split(value, ",") {
|
||||||
|
item = strings.TrimSpace(item)
|
||||||
|
if item != "" {
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
123
go/vehicle-gateway/cmd/history-writer/main.go
Normal file
123
go/vehicle-gateway/cmd/history-writer/main.go
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/segmentio/kafka-go"
|
||||||
|
_ "github.com/taosdata/driver-go/v3/taosWS"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
logger := observability.NewLogger("vehicle-history-writer")
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
cfg := loadConfig()
|
||||||
|
db, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("tdengine open failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
if err := db.PingContext(ctx); err != nil {
|
||||||
|
logger.Error("tdengine ping failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
writer := history.NewWriter(db)
|
||||||
|
if cfg.EnsureSchema {
|
||||||
|
if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
|
||||||
|
logger.Error("tdengine schema bootstrap failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||||
|
Brokers: cfg.KafkaBrokers,
|
||||||
|
GroupID: cfg.KafkaGroup,
|
||||||
|
GroupTopics: cfg.KafkaTopics,
|
||||||
|
MinBytes: 1,
|
||||||
|
MaxBytes: 10e6,
|
||||||
|
})
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
logger.Info("history writer started",
|
||||||
|
"driver", cfg.TDengineDriver,
|
||||||
|
"group", cfg.KafkaGroup,
|
||||||
|
"topics", strings.Join(cfg.KafkaTopics, ","))
|
||||||
|
|
||||||
|
for {
|
||||||
|
message, err := reader.FetchMessage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logger.Error("kafka fetch failed", "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var env envelope.FrameEnvelope
|
||||||
|
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||||
|
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||||
|
_ = reader.CommitMessages(ctx, message)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := writer.AppendAll(ctx, env); err != nil {
|
||||||
|
logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||||
|
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type config struct {
|
||||||
|
KafkaBrokers []string
|
||||||
|
KafkaTopics []string
|
||||||
|
KafkaGroup string
|
||||||
|
TDengineDriver string
|
||||||
|
TDengineDSN string
|
||||||
|
TDengineDatabase string
|
||||||
|
EnsureSchema bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadConfig() config {
|
||||||
|
return config{
|
||||||
|
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||||
|
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1")),
|
||||||
|
KafkaGroup: env("KAFKA_GROUP", "go-history-writer"),
|
||||||
|
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
|
||||||
|
TDengineDSN: env("TDENGINE_DSN", ""),
|
||||||
|
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
|
||||||
|
EnsureSchema: env("TDENGINE_ENSURE_SCHEMA", "true") != "false",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(key string, fallback string) string {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitCSV(value string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, item := range strings.Split(value, ",") {
|
||||||
|
item = strings.TrimSpace(item)
|
||||||
|
if item != "" {
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
135
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
135
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"github.com/segmentio/kafka-go"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
logger := observability.NewLogger("vehicle-realtime-api")
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
client := redis.NewClient(&redis.Options{
|
||||||
|
Addr: env("REDIS_ADDR", "127.0.0.1:6379"),
|
||||||
|
Username: env("REDIS_USERNAME", ""),
|
||||||
|
Password: env("REDIS_PASSWORD", ""),
|
||||||
|
DB: envInt("REDIS_DB", 0),
|
||||||
|
})
|
||||||
|
defer client.Close()
|
||||||
|
if err := client.Ping(ctx).Err(); err != nil {
|
||||||
|
logger.Error("redis ping failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
repository := realtime.NewRepository(client, realtime.Config{
|
||||||
|
OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", 600)) * time.Second,
|
||||||
|
})
|
||||||
|
if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
|
||||||
|
go consumeKafka(ctx, logger, repository, brokers)
|
||||||
|
} else {
|
||||||
|
logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
|
||||||
|
}
|
||||||
|
|
||||||
|
server := &http.Server{
|
||||||
|
Addr: env("HTTP_ADDR", ":20210"),
|
||||||
|
Handler: realtime.NewHandler(repository),
|
||||||
|
ReadHeaderTimeout: 5 * time.Second,
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = server.Shutdown(shutdownCtx)
|
||||||
|
}()
|
||||||
|
|
||||||
|
logger.Info("realtime api started", "addr", server.Addr)
|
||||||
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
|
logger.Error("http server failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func consumeKafka(ctx context.Context, logger interface {
|
||||||
|
Info(string, ...any)
|
||||||
|
Error(string, ...any)
|
||||||
|
Warn(string, ...any)
|
||||||
|
}, repository *realtime.Repository, brokers []string) {
|
||||||
|
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||||
|
Brokers: brokers,
|
||||||
|
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
|
||||||
|
GroupTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.event.unified.v1")),
|
||||||
|
MinBytes: 1,
|
||||||
|
MaxBytes: 10e6,
|
||||||
|
})
|
||||||
|
defer reader.Close()
|
||||||
|
logger.Info("realtime kafka consumer started", "topics", env("KAFKA_TOPICS", "vehicle.event.unified.v1"))
|
||||||
|
for {
|
||||||
|
message, err := reader.FetchMessage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logger.Error("kafka fetch failed", "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var env envelope.FrameEnvelope
|
||||||
|
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||||
|
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||||
|
_ = reader.CommitMessages(ctx, message)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := repository.Update(ctx, env); err != nil {
|
||||||
|
logger.Error("redis realtime update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||||
|
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(key string, fallback string) string {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func envInt(key string, fallback int) int {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
parsed, err := strconv.Atoi(value)
|
||||||
|
if err != nil {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitCSV(value string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, item := range strings.Split(value, ",") {
|
||||||
|
item = strings.TrimSpace(item)
|
||||||
|
if item != "" {
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
122
go/vehicle-gateway/cmd/stat-writer/main.go
Normal file
122
go/vehicle-gateway/cmd/stat-writer/main.go
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "github.com/go-sql-driver/mysql"
|
||||||
|
"github.com/segmentio/kafka-go"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
logger := observability.NewLogger("vehicle-stat-writer")
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
cfg := loadConfig()
|
||||||
|
db, err := sql.Open("mysql", cfg.MySQLDSN)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("mysql open failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
if err := db.PingContext(ctx); err != nil {
|
||||||
|
logger.Error("mysql ping failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
writer := stats.NewWriter(db, cfg.Location)
|
||||||
|
if cfg.EnsureSchema {
|
||||||
|
if err := writer.EnsureSchema(ctx); err != nil {
|
||||||
|
logger.Error("mysql schema bootstrap failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||||
|
Brokers: cfg.KafkaBrokers,
|
||||||
|
GroupID: cfg.KafkaGroup,
|
||||||
|
GroupTopics: cfg.KafkaTopics,
|
||||||
|
MinBytes: 1,
|
||||||
|
MaxBytes: 10e6,
|
||||||
|
})
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","))
|
||||||
|
for {
|
||||||
|
message, err := reader.FetchMessage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logger.Error("kafka fetch failed", "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var env envelope.FrameEnvelope
|
||||||
|
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||||
|
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||||
|
_ = reader.CommitMessages(ctx, message)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := writer.Append(ctx, env); err != nil {
|
||||||
|
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||||
|
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type config struct {
|
||||||
|
KafkaBrokers []string
|
||||||
|
KafkaTopics []string
|
||||||
|
KafkaGroup string
|
||||||
|
MySQLDSN string
|
||||||
|
EnsureSchema bool
|
||||||
|
Location *time.Location
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadConfig() config {
|
||||||
|
loc, err := time.LoadLocation(env("LOCAL_TZ", "Asia/Shanghai"))
|
||||||
|
if err != nil {
|
||||||
|
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||||
|
}
|
||||||
|
return config{
|
||||||
|
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||||
|
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1")),
|
||||||
|
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
|
||||||
|
MySQLDSN: env("MYSQL_DSN", ""),
|
||||||
|
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",
|
||||||
|
Location: loc,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(key string, fallback string) string {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitCSV(value string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, item := range strings.Split(value, ",") {
|
||||||
|
item = strings.TrimSpace(item)
|
||||||
|
if item != "" {
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
29
go/vehicle-gateway/go.mod
Normal file
29
go/vehicle-gateway/go.mod
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||||
|
|
||||||
|
go 1.26
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||||
|
github.com/alicebob/miniredis/v2 v2.35.0
|
||||||
|
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||||
|
github.com/go-sql-driver/mysql v1.9.3
|
||||||
|
github.com/redis/go-redis/v9 v9.17.2
|
||||||
|
github.com/segmentio/kafka-go v0.4.49
|
||||||
|
github.com/taosdata/driver-go/v3 v3.8.1
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
filippo.io/edwards25519 v1.1.0 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
|
github.com/gorilla/websocket v1.5.3 // indirect
|
||||||
|
github.com/json-iterator/go v1.1.12 // indirect
|
||||||
|
github.com/klauspost/compress v1.15.9 // indirect
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||||
|
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||||
|
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||||
|
github.com/yuin/gopher-lua v1.1.1 // indirect
|
||||||
|
golang.org/x/net v0.44.0 // indirect
|
||||||
|
golang.org/x/sync v0.17.0 // indirect
|
||||||
|
)
|
||||||
73
go/vehicle-gateway/go.sum
Normal file
73
go/vehicle-gateway/go.sum
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||||
|
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||||
|
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
|
||||||
|
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
|
||||||
|
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||||
|
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||||
|
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||||
|
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||||
|
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||||
|
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||||
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||||
|
github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE=
|
||||||
|
github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU=
|
||||||
|
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||||
|
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||||
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||||
|
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||||
|
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||||
|
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||||
|
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||||
|
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
|
||||||
|
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
|
||||||
|
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
|
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||||
|
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||||
|
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||||
|
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||||
|
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||||
|
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
||||||
|
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||||
|
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||||
|
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||||
|
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||||
|
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||||
|
github.com/taosdata/driver-go/v3 v3.8.1 h1:kkd4ABsGiU+oXDbsw/sic985LKAvnpF9Gb/TEunTnLE=
|
||||||
|
github.com/taosdata/driver-go/v3 v3.8.1/go.mod h1:S6OGOinfR0xxxaMGsvBi9cLkYxEIW1p6qqr8QJATTlg=
|
||||||
|
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||||
|
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||||
|
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||||
|
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||||
|
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||||
|
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||||
|
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||||
|
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||||
|
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
|
||||||
|
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
|
||||||
|
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||||
|
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||||
|
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||||
|
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
95
go/vehicle-gateway/internal/envelope/envelope.go
Normal file
95
go/vehicle-gateway/internal/envelope/envelope.go
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
package envelope
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Protocol string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ProtocolGB32960 Protocol = "GB32960"
|
||||||
|
ProtocolJT808 Protocol = "JT808"
|
||||||
|
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ParseStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ParseOK ParseStatus = "OK"
|
||||||
|
ParsePartial ParseStatus = "PARTIAL"
|
||||||
|
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
FieldSpeedKMH = "speed_kmh"
|
||||||
|
FieldTotalMileageKM = "total_mileage_km"
|
||||||
|
FieldLongitude = "longitude"
|
||||||
|
FieldLatitude = "latitude"
|
||||||
|
FieldSOCPercent = "soc_percent"
|
||||||
|
)
|
||||||
|
|
||||||
|
type FrameEnvelope struct {
|
||||||
|
EventID string `json:"event_id"`
|
||||||
|
TraceID string `json:"trace_id"`
|
||||||
|
Protocol Protocol `json:"protocol"`
|
||||||
|
MessageID string `json:"message_id"`
|
||||||
|
Sequence uint16 `json:"sequence"`
|
||||||
|
VIN string `json:"vin,omitempty"`
|
||||||
|
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||||
|
Phone string `json:"phone,omitempty"`
|
||||||
|
DeviceID string `json:"device_id,omitempty"`
|
||||||
|
Plate string `json:"plate,omitempty"`
|
||||||
|
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||||
|
EventTimeMS int64 `json:"event_time_ms"`
|
||||||
|
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||||
|
RawHex string `json:"raw_hex,omitempty"`
|
||||||
|
RawText string `json:"raw_text,omitempty"`
|
||||||
|
Parsed map[string]any `json:"parsed,omitempty"`
|
||||||
|
Fields map[string]any `json:"fields,omitempty"`
|
||||||
|
ParseStatus ParseStatus `json:"parse_status"`
|
||||||
|
ParseError string `json:"parse_error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e FrameEnvelope) VehicleKey() string {
|
||||||
|
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||||
|
return string(e.Protocol) + ":" + key
|
||||||
|
}
|
||||||
|
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||||
|
return string(e.Protocol) + ":" + key
|
||||||
|
}
|
||||||
|
return string(e.Protocol) + ":unknown"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e FrameEnvelope) StableEventID() string {
|
||||||
|
if strings.TrimSpace(e.EventID) != "" {
|
||||||
|
return e.EventID
|
||||||
|
}
|
||||||
|
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||||
|
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||||
|
sum := sha256.Sum256([]byte(input))
|
||||||
|
return hex.EncodeToString(sum[:16])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e FrameEnvelope) KafkaKey() []byte {
|
||||||
|
return []byte(e.VehicleKey())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||||
|
if e.EventID == "" {
|
||||||
|
e.EventID = e.StableEventID()
|
||||||
|
}
|
||||||
|
if e.ParseStatus == "" {
|
||||||
|
e.ParseStatus = ParseOK
|
||||||
|
}
|
||||||
|
return json.Marshal(e)
|
||||||
|
}
|
||||||
55
go/vehicle-gateway/internal/envelope/envelope_test.go
Normal file
55
go/vehicle-gateway/internal/envelope/envelope_test.go
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
package envelope
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||||
|
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||||
|
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||||
|
t.Fatalf("VehicleKey() = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||||
|
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||||
|
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||||
|
t.Fatalf("VehicleKey() = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||||
|
e := FrameEnvelope{
|
||||||
|
Protocol: ProtocolJT808,
|
||||||
|
MessageID: "0x0200",
|
||||||
|
Phone: "013307795425",
|
||||||
|
Sequence: 1,
|
||||||
|
EventTimeMS: 1782745114000,
|
||||||
|
ReceivedAtMS: 1782745114999,
|
||||||
|
RawHex: "7e02000000ff7e",
|
||||||
|
}
|
||||||
|
a := e.StableEventID()
|
||||||
|
b := e.StableEventID()
|
||||||
|
if a == "" || a != b {
|
||||||
|
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameEnvelopeMarshalDefaults(t *testing.T) {
|
||||||
|
e := FrameEnvelope{Protocol: ProtocolGB32960, VIN: "LNBVIN00000000002", MessageID: "0x02"}
|
||||||
|
payload, err := e.MarshalJSONBytes()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MarshalJSONBytes() error = %v", err)
|
||||||
|
}
|
||||||
|
var decoded FrameEnvelope
|
||||||
|
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
|
}
|
||||||
|
if decoded.EventID == "" {
|
||||||
|
t.Fatal("event id should be filled")
|
||||||
|
}
|
||||||
|
if decoded.ParseStatus != ParseOK {
|
||||||
|
t.Fatalf("parse status = %q", decoded.ParseStatus)
|
||||||
|
}
|
||||||
|
}
|
||||||
94
go/vehicle-gateway/internal/eventbus/kafka_sink.go
Normal file
94
go/vehicle-gateway/internal/eventbus/kafka_sink.go
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
package eventbus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/segmentio/kafka-go"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Sink interface {
|
||||||
|
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||||
|
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
type KafkaSink struct {
|
||||||
|
writer kafkaWriter
|
||||||
|
rawTopics map[envelope.Protocol]string
|
||||||
|
unifiedTopic string
|
||||||
|
}
|
||||||
|
|
||||||
|
type kafkaWriter interface {
|
||||||
|
WriteMessages(context.Context, ...kafka.Message) error
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
type KafkaConfig struct {
|
||||||
|
Brokers []string
|
||||||
|
RawTopics map[envelope.Protocol]string
|
||||||
|
UnifiedTopic string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) {
|
||||||
|
if len(cfg.Brokers) == 0 {
|
||||||
|
return nil, errors.New("kafka brokers are required")
|
||||||
|
}
|
||||||
|
return newKafkaSinkWithWriter(&kafka.Writer{
|
||||||
|
Addr: kafka.TCP(cfg.Brokers...),
|
||||||
|
Balancer: &kafka.Hash{},
|
||||||
|
AllowAutoTopicCreation: false,
|
||||||
|
}, cfg), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
|
||||||
|
rawTopics := map[envelope.Protocol]string{
|
||||||
|
envelope.ProtocolGB32960: "vehicle.raw.gb32960.v1",
|
||||||
|
envelope.ProtocolJT808: "vehicle.raw.jt808.v1",
|
||||||
|
envelope.ProtocolYutongMQTT: "vehicle.raw.yutong-mqtt.v1",
|
||||||
|
}
|
||||||
|
for protocol, topic := range cfg.RawTopics {
|
||||||
|
if topic != "" {
|
||||||
|
rawTopics[protocol] = topic
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unifiedTopic := cfg.UnifiedTopic
|
||||||
|
if unifiedTopic == "" {
|
||||||
|
unifiedTopic = "vehicle.event.unified.v1"
|
||||||
|
}
|
||||||
|
return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
topic, ok := s.rawTopics[env.Protocol]
|
||||||
|
if !ok || topic == "" {
|
||||||
|
return fmt.Errorf("raw topic not configured for protocol %s", env.Protocol)
|
||||||
|
}
|
||||||
|
return s.publish(ctx, topic, env)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
return s.publish(ctx, s.unifiedTopic, env)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *KafkaSink) Close() error {
|
||||||
|
if s == nil || s.writer == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return s.writer.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
|
||||||
|
payload, err := env.MarshalJSONBytes()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return s.writer.WriteMessages(ctx, kafka.Message{
|
||||||
|
Topic: topic,
|
||||||
|
Key: env.KafkaKey(),
|
||||||
|
Value: payload,
|
||||||
|
})
|
||||||
|
}
|
||||||
77
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
Normal file
77
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
package eventbus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/segmentio/kafka-go"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestKafkaSinkRoutesRawTopics(t *testing.T) {
|
||||||
|
writer := &recordingWriter{}
|
||||||
|
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
|
||||||
|
|
||||||
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "013307795425", MessageID: "0x0200"}
|
||||||
|
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||||
|
t.Fatalf("PublishRaw() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(writer.messages) != 1 {
|
||||||
|
t.Fatalf("messages = %d", len(writer.messages))
|
||||||
|
}
|
||||||
|
msg := writer.messages[0]
|
||||||
|
if msg.Topic != "vehicle.raw.jt808.v1" {
|
||||||
|
t.Fatalf("topic = %q", msg.Topic)
|
||||||
|
}
|
||||||
|
if string(msg.Key) != "JT808:013307795425" {
|
||||||
|
t.Fatalf("key = %q", string(msg.Key))
|
||||||
|
}
|
||||||
|
var decoded envelope.FrameEnvelope
|
||||||
|
if err := json.Unmarshal(msg.Value, &decoded); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
|
}
|
||||||
|
if decoded.EventID == "" || decoded.ParseStatus != envelope.ParseOK {
|
||||||
|
t.Fatalf("unexpected payload defaults: %#v", decoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKafkaSinkRoutesUnifiedTopic(t *testing.T) {
|
||||||
|
writer := &recordingWriter{}
|
||||||
|
sink := newKafkaSinkWithWriter(writer, KafkaConfig{UnifiedTopic: "custom.unified"})
|
||||||
|
|
||||||
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000001", MessageID: "0x02"}
|
||||||
|
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||||
|
t.Fatalf("PublishUnified() error = %v", err)
|
||||||
|
}
|
||||||
|
if writer.messages[0].Topic != "custom.unified" {
|
||||||
|
t.Fatalf("topic = %q", writer.messages[0].Topic)
|
||||||
|
}
|
||||||
|
if string(writer.messages[0].Key) != "LNBVIN00000000001" {
|
||||||
|
t.Fatalf("key = %q", string(writer.messages[0].Key))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKafkaSinkRejectsUnknownProtocol(t *testing.T) {
|
||||||
|
writer := &recordingWriter{}
|
||||||
|
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
|
||||||
|
|
||||||
|
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected unknown protocol error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingWriter struct {
|
||||||
|
messages []kafka.Message
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *recordingWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
|
||||||
|
w.messages = append(w.messages, messages...)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *recordingWriter) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
33
go/vehicle-gateway/internal/eventbus/log_sink.go
Normal file
33
go/vehicle-gateway/internal/eventbus/log_sink.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package eventbus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type LogSink struct {
|
||||||
|
logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewLogSink(logger *slog.Logger) *LogSink {
|
||||||
|
if logger == nil {
|
||||||
|
logger = slog.Default()
|
||||||
|
}
|
||||||
|
return &LogSink{logger: logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *LogSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
s.logger.Info("raw envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID, "status", env.ParseStatus)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *LogSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
s.logger.Info("unified envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *LogSink) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
146
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
146
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
@@ -0,0 +1,146 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MQTTClientConfig struct {
|
||||||
|
EndpointName string
|
||||||
|
Broker string
|
||||||
|
ClientID string
|
||||||
|
Username string
|
||||||
|
Password string
|
||||||
|
Topics []string
|
||||||
|
QoS byte
|
||||||
|
Sink eventbus.Sink
|
||||||
|
Resolver identity.Resolver
|
||||||
|
Logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
type MQTTClient struct {
|
||||||
|
cfg MQTTClientConfig
|
||||||
|
client mqtt.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
|
||||||
|
if strings.TrimSpace(cfg.Broker) == "" {
|
||||||
|
return nil, errors.New("mqtt broker is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.ClientID) == "" {
|
||||||
|
return nil, errors.New("mqtt client id is required")
|
||||||
|
}
|
||||||
|
if len(cfg.Topics) == 0 {
|
||||||
|
return nil, errors.New("mqtt topics are required")
|
||||||
|
}
|
||||||
|
if cfg.Sink == nil {
|
||||||
|
return nil, errors.New("sink is required")
|
||||||
|
}
|
||||||
|
if cfg.Logger == nil {
|
||||||
|
cfg.Logger = slog.Default()
|
||||||
|
}
|
||||||
|
if cfg.Resolver == nil {
|
||||||
|
cfg.Resolver = identity.NoopResolver{}
|
||||||
|
}
|
||||||
|
if cfg.EndpointName == "" {
|
||||||
|
cfg.EndpointName = "yutong"
|
||||||
|
}
|
||||||
|
return &MQTTClient{cfg: cfg}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *MQTTClient) Start(ctx context.Context) error {
|
||||||
|
opts := mqtt.NewClientOptions().
|
||||||
|
AddBroker(c.cfg.Broker).
|
||||||
|
SetClientID(c.cfg.ClientID).
|
||||||
|
SetUsername(c.cfg.Username).
|
||||||
|
SetPassword(c.cfg.Password).
|
||||||
|
SetCleanSession(false).
|
||||||
|
SetAutoReconnect(true).
|
||||||
|
SetConnectRetry(true).
|
||||||
|
SetConnectRetryInterval(5 * time.Second).
|
||||||
|
SetKeepAlive(20 * time.Second).
|
||||||
|
SetConnectTimeout(10 * time.Second)
|
||||||
|
|
||||||
|
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
|
||||||
|
c.handleMessage(ctx, message.Topic(), message.Payload())
|
||||||
|
})
|
||||||
|
opts.OnConnect = func(client mqtt.Client) {
|
||||||
|
for _, topic := range c.cfg.Topics {
|
||||||
|
token := client.Subscribe(topic, c.cfg.QoS, nil)
|
||||||
|
if token.Wait() && token.Error() != nil {
|
||||||
|
c.cfg.Logger.Error("mqtt subscribe failed", "broker", c.cfg.Broker, "topic", topic, "error", token.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.cfg.Logger.Info("mqtt subscribed", "broker", c.cfg.Broker, "topic", topic, "qos", c.cfg.QoS)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opts.OnConnectionLost = func(_ mqtt.Client, err error) {
|
||||||
|
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.client = mqtt.NewClient(opts)
|
||||||
|
token := c.client.Connect()
|
||||||
|
if token.Wait() && token.Error() != nil {
|
||||||
|
return token.Error()
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
if c.client != nil && c.client.IsConnected() {
|
||||||
|
c.client.Disconnect(250)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
|
||||||
|
receivedAtMS := time.Now().UnixMilli()
|
||||||
|
env, err := yutongmqtt.ParseMessage(c.cfg.EndpointName, topic, payload, receivedAtMS)
|
||||||
|
if err != nil {
|
||||||
|
env = envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolYutongMQTT,
|
||||||
|
MessageID: "MQTT",
|
||||||
|
VehicleKeyHint: "unknown",
|
||||||
|
SourceEndpoint: "mqtt://" + c.cfg.EndpointName + topic,
|
||||||
|
EventTimeMS: receivedAtMS,
|
||||||
|
ReceivedAtMS: receivedAtMS,
|
||||||
|
RawText: string(payload),
|
||||||
|
RawHex: strings.ToUpper(hex.EncodeToString(payload)),
|
||||||
|
ParseStatus: envelope.ParseBadFrame,
|
||||||
|
ParseError: err.Error(),
|
||||||
|
}
|
||||||
|
env.EventID = env.StableEventID()
|
||||||
|
} else {
|
||||||
|
resolved, resolveErr := c.cfg.Resolver.Resolve(ctx, env)
|
||||||
|
if resolveErr != nil {
|
||||||
|
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
|
||||||
|
if env.Parsed == nil {
|
||||||
|
env.Parsed = map[string]any{}
|
||||||
|
}
|
||||||
|
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
|
||||||
|
env.ParseStatus = envelope.ParsePartial
|
||||||
|
} else {
|
||||||
|
env = resolved
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := c.cfg.Sink.PublishRaw(ctx, env); err != nil {
|
||||||
|
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if env.ParseStatus == envelope.ParseBadFrame {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := c.cfg.Sink.PublishUnified(ctx, env); err != nil {
|
||||||
|
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
61
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
61
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
|
||||||
|
sink := &recordingSink{}
|
||||||
|
client, err := NewMQTTClient(MQTTClientConfig{
|
||||||
|
EndpointName: "endpoint-a",
|
||||||
|
Broker: "tcp://127.0.0.1:1883",
|
||||||
|
ClientID: "test-client",
|
||||||
|
Topics: []string{"/ytforward/shln/+"},
|
||||||
|
Sink: sink,
|
||||||
|
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
|
||||||
|
"device":"LTEST000000000001",
|
||||||
|
"time":"20260413100000",
|
||||||
|
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
|
||||||
|
}`))
|
||||||
|
|
||||||
|
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||||
|
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
|
}
|
||||||
|
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
|
||||||
|
t.Fatalf("unexpected raw envelope: %#v", sink.raw[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
|
||||||
|
sink := &recordingSink{}
|
||||||
|
client, err := NewMQTTClient(MQTTClientConfig{
|
||||||
|
EndpointName: "endpoint-a",
|
||||||
|
Broker: "tcp://127.0.0.1:1883",
|
||||||
|
ClientID: "test-client",
|
||||||
|
Topics: []string{"/ytforward/shln/+"},
|
||||||
|
Sink: sink,
|
||||||
|
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client.handleMessage(context.Background(), "/ytforward/shln/bad", []byte("{bad-json"))
|
||||||
|
|
||||||
|
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||||
|
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
|
}
|
||||||
|
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||||
|
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||||
|
}
|
||||||
|
}
|
||||||
220
go/vehicle-gateway/internal/gateway/tcp_server.go
Normal file
220
go/vehicle-gateway/internal/gateway/tcp_server.go
Normal file
@@ -0,0 +1,220 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
|
||||||
|
|
||||||
|
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
|
||||||
|
|
||||||
|
type TCPProtocol struct {
|
||||||
|
Protocol envelope.Protocol
|
||||||
|
Addr string
|
||||||
|
Extract FrameExtractor
|
||||||
|
Parse FrameParser
|
||||||
|
}
|
||||||
|
|
||||||
|
type TCPServer struct {
|
||||||
|
protocol TCPProtocol
|
||||||
|
sink eventbus.Sink
|
||||||
|
resolver identity.Resolver
|
||||||
|
logger *slog.Logger
|
||||||
|
readBufferSize int
|
||||||
|
idleTimeout time.Duration
|
||||||
|
maxConnections int
|
||||||
|
}
|
||||||
|
|
||||||
|
type TCPServerConfig struct {
|
||||||
|
Protocol TCPProtocol
|
||||||
|
Sink eventbus.Sink
|
||||||
|
Resolver identity.Resolver
|
||||||
|
Logger *slog.Logger
|
||||||
|
ReadBufferSize int
|
||||||
|
IdleTimeout time.Duration
|
||||||
|
MaxConnections int
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
|
||||||
|
if cfg.Protocol.Protocol == "" {
|
||||||
|
return nil, errors.New("protocol is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.Protocol.Addr) == "" {
|
||||||
|
return nil, errors.New("listen addr is required")
|
||||||
|
}
|
||||||
|
if cfg.Protocol.Extract == nil {
|
||||||
|
return nil, errors.New("frame extractor is required")
|
||||||
|
}
|
||||||
|
if cfg.Protocol.Parse == nil {
|
||||||
|
return nil, errors.New("frame parser is required")
|
||||||
|
}
|
||||||
|
if cfg.Sink == nil {
|
||||||
|
return nil, errors.New("sink is required")
|
||||||
|
}
|
||||||
|
if cfg.Logger == nil {
|
||||||
|
cfg.Logger = slog.Default()
|
||||||
|
}
|
||||||
|
if cfg.Resolver == nil {
|
||||||
|
cfg.Resolver = identity.NoopResolver{}
|
||||||
|
}
|
||||||
|
if cfg.ReadBufferSize <= 0 {
|
||||||
|
cfg.ReadBufferSize = 32 * 1024
|
||||||
|
}
|
||||||
|
if cfg.IdleTimeout <= 0 {
|
||||||
|
cfg.IdleTimeout = 2 * time.Minute
|
||||||
|
}
|
||||||
|
if cfg.MaxConnections <= 0 {
|
||||||
|
cfg.MaxConnections = 10_000
|
||||||
|
}
|
||||||
|
return &TCPServer{
|
||||||
|
protocol: cfg.Protocol,
|
||||||
|
sink: cfg.Sink,
|
||||||
|
resolver: cfg.Resolver,
|
||||||
|
logger: cfg.Logger,
|
||||||
|
readBufferSize: cfg.ReadBufferSize,
|
||||||
|
idleTimeout: cfg.IdleTimeout,
|
||||||
|
maxConnections: cfg.MaxConnections,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *TCPServer) ListenAndServe(ctx context.Context) error {
|
||||||
|
var lc net.ListenConfig
|
||||||
|
listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer listener.Close()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
_ = listener.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
|
||||||
|
sem := make(chan struct{}, s.maxConnections)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
defer wg.Wait()
|
||||||
|
|
||||||
|
for {
|
||||||
|
conn, err := listener.Accept()
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case sem <- struct{}{}:
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
defer func() { <-sem }()
|
||||||
|
s.handleConnection(ctx, conn)
|
||||||
|
}()
|
||||||
|
default:
|
||||||
|
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
|
||||||
|
_ = conn.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
||||||
|
defer conn.Close()
|
||||||
|
source := conn.RemoteAddr().String()
|
||||||
|
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
|
||||||
|
log.Info("tcp connection opened")
|
||||||
|
defer log.Info("tcp connection closed")
|
||||||
|
|
||||||
|
readBuffer := make([]byte, s.readBufferSize)
|
||||||
|
var pending []byte
|
||||||
|
for {
|
||||||
|
_ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout))
|
||||||
|
n, err := conn.Read(readBuffer)
|
||||||
|
if n > 0 {
|
||||||
|
pending = append(pending, readBuffer[:n]...)
|
||||||
|
frames, remainder, extractErr := s.protocol.Extract(pending)
|
||||||
|
if extractErr != nil {
|
||||||
|
log.Warn("frame extraction failed", "error", extractErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pending = remainder
|
||||||
|
for _, frame := range frames {
|
||||||
|
s.handleFrame(ctx, frame, source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var netErr net.Error
|
||||||
|
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||||
|
log.Warn("tcp connection idle timeout")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
log.Warn("tcp read failed", "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *TCPServer) handleFrame(ctx context.Context, raw []byte, source string) {
|
||||||
|
receivedAtMS := time.Now().UnixMilli()
|
||||||
|
env, err := s.protocol.Parse(raw, receivedAtMS, source)
|
||||||
|
if err != nil {
|
||||||
|
env = envelope.FrameEnvelope{
|
||||||
|
Protocol: s.protocol.Protocol,
|
||||||
|
SourceEndpoint: source,
|
||||||
|
ReceivedAtMS: receivedAtMS,
|
||||||
|
EventTimeMS: receivedAtMS,
|
||||||
|
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||||
|
ParseStatus: envelope.ParseBadFrame,
|
||||||
|
ParseError: err.Error(),
|
||||||
|
}
|
||||||
|
env.EventID = env.StableEventID()
|
||||||
|
} else {
|
||||||
|
resolved, resolveErr := s.resolver.Resolve(ctx, env)
|
||||||
|
if resolveErr != nil {
|
||||||
|
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
|
||||||
|
if env.Parsed == nil {
|
||||||
|
env.Parsed = map[string]any{}
|
||||||
|
}
|
||||||
|
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
|
||||||
|
env.ParseStatus = envelope.ParsePartial
|
||||||
|
} else {
|
||||||
|
env = resolved
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.sink.PublishRaw(ctx, env); err != nil {
|
||||||
|
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if env.ParseStatus == envelope.ParseBadFrame {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := s.sink.PublishUnified(ctx, env); err != nil {
|
||||||
|
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p TCPProtocol) String() string {
|
||||||
|
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
|
||||||
|
}
|
||||||
145
go/vehicle-gateway/internal/gateway/tcp_server_test.go
Normal file
145
go/vehicle-gateway/internal/gateway/tcp_server_test.go
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/hex"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
|
||||||
|
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sink := &recordingSink{}
|
||||||
|
server := newTestServer(t, TCPProtocol{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
Addr: ":0",
|
||||||
|
Extract: jt808.ExtractFrames,
|
||||||
|
Parse: jt808.ParseFrame,
|
||||||
|
}, sink)
|
||||||
|
|
||||||
|
client, done := runPipe(t, server)
|
||||||
|
if _, err := client.Write(frame); err != nil {
|
||||||
|
t.Fatalf("client.Write() error = %v", err)
|
||||||
|
}
|
||||||
|
_ = client.Close()
|
||||||
|
<-done
|
||||||
|
|
||||||
|
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||||
|
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
|
}
|
||||||
|
if sink.raw[0].Phone != "013307795425" {
|
||||||
|
t.Fatalf("phone = %q", sink.raw[0].Phone)
|
||||||
|
}
|
||||||
|
if sink.raw[0].Fields[envelope.FieldTotalMileageKM] != 10241.2 {
|
||||||
|
t.Fatalf("total mileage = %#v", sink.raw[0].Fields[envelope.FieldTotalMileageKM])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
|
||||||
|
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||||
|
good[len(good)-1] ^= 0xff
|
||||||
|
sink := &recordingSink{}
|
||||||
|
server := newTestServer(t, TCPProtocol{
|
||||||
|
Protocol: envelope.ProtocolGB32960,
|
||||||
|
Addr: ":0",
|
||||||
|
Extract: gb32960.ExtractFrames,
|
||||||
|
Parse: gb32960.ParseFrame,
|
||||||
|
}, sink)
|
||||||
|
|
||||||
|
client, done := runPipe(t, server)
|
||||||
|
if _, err := client.Write(good); err != nil {
|
||||||
|
t.Fatalf("client.Write() error = %v", err)
|
||||||
|
}
|
||||||
|
_ = client.Close()
|
||||||
|
<-done
|
||||||
|
|
||||||
|
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||||
|
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
|
}
|
||||||
|
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||||
|
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||||
|
}
|
||||||
|
if sink.raw[0].ParseError == "" {
|
||||||
|
t.Fatal("parse error should be recorded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer {
|
||||||
|
t.Helper()
|
||||||
|
server, err := NewTCPServer(TCPServerConfig{
|
||||||
|
Protocol: protocol,
|
||||||
|
Sink: sink,
|
||||||
|
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||||
|
ReadBufferSize: 1024,
|
||||||
|
IdleTimeout: time.Second,
|
||||||
|
MaxConnections: 1,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewTCPServer() error = %v", err)
|
||||||
|
}
|
||||||
|
return server
|
||||||
|
}
|
||||||
|
|
||||||
|
func runPipe(t *testing.T, server *TCPServer) (net.Conn, <-chan struct{}) {
|
||||||
|
t.Helper()
|
||||||
|
client, srv := net.Pipe()
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
server.handleConnection(context.Background(), srv)
|
||||||
|
}()
|
||||||
|
return client, done
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingSink struct {
|
||||||
|
raw []envelope.FrameEnvelope
|
||||||
|
unified []envelope.FrameEnvelope
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *recordingSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
s.raw = append(s.raw, env)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *recordingSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
s.unified = append(s.unified, env)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *recordingSink) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type testWriter struct {
|
||||||
|
t *testing.T
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w testWriter) Write(p []byte) (int, error) {
|
||||||
|
w.t.Log(string(p))
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildGBFrame(command byte, response byte, vin string, body []byte) []byte {
|
||||||
|
frame := []byte{'#', '#', command, response}
|
||||||
|
vinBytes := []byte(vin)
|
||||||
|
if len(vinBytes) < 17 {
|
||||||
|
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
|
||||||
|
}
|
||||||
|
frame = append(frame, vinBytes[:17]...)
|
||||||
|
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
|
||||||
|
frame = append(frame, body...)
|
||||||
|
var bcc byte
|
||||||
|
for _, value := range frame[2:] {
|
||||||
|
bcc ^= value
|
||||||
|
}
|
||||||
|
return append(frame, bcc)
|
||||||
|
}
|
||||||
71
go/vehicle-gateway/internal/history/schema.go
Normal file
71
go/vehicle-gateway/internal/history/schema.go
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
package history
|
||||||
|
|
||||||
|
const DefaultDatabase = "lingniu_vehicle_ts"
|
||||||
|
|
||||||
|
func SchemaStatements(database string) []string {
|
||||||
|
if database == "" {
|
||||||
|
database = DefaultDatabase
|
||||||
|
}
|
||||||
|
return []string{
|
||||||
|
"CREATE DATABASE IF NOT EXISTS " + database + " KEEP 7300 DURATION 10 BUFFER 256",
|
||||||
|
"USE " + database,
|
||||||
|
`CREATE STABLE IF NOT EXISTS raw_frames (
|
||||||
|
ts TIMESTAMP,
|
||||||
|
frame_id NCHAR(64),
|
||||||
|
event_id NCHAR(64),
|
||||||
|
message_id INT,
|
||||||
|
event_time TIMESTAMP,
|
||||||
|
received_at TIMESTAMP,
|
||||||
|
raw_size_bytes INT,
|
||||||
|
raw_hex BINARY(16374),
|
||||||
|
raw_text BINARY(16374),
|
||||||
|
parsed_json BINARY(16374),
|
||||||
|
fields_json BINARY(4096),
|
||||||
|
parse_status NCHAR(16),
|
||||||
|
parse_error BINARY(1024),
|
||||||
|
source_endpoint NCHAR(128)
|
||||||
|
) TAGS (
|
||||||
|
protocol NCHAR(32),
|
||||||
|
vehicle_key NCHAR(64),
|
||||||
|
vin NCHAR(32),
|
||||||
|
phone NCHAR(32),
|
||||||
|
device_id NCHAR(64)
|
||||||
|
)`,
|
||||||
|
`CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||||
|
ts TIMESTAMP,
|
||||||
|
event_id NCHAR(64),
|
||||||
|
frame_id NCHAR(64),
|
||||||
|
received_at TIMESTAMP,
|
||||||
|
longitude DOUBLE,
|
||||||
|
latitude DOUBLE,
|
||||||
|
altitude_m DOUBLE,
|
||||||
|
speed_kmh DOUBLE,
|
||||||
|
direction_deg INT,
|
||||||
|
alarm_flag BIGINT,
|
||||||
|
status_flag BIGINT,
|
||||||
|
total_mileage_km DOUBLE
|
||||||
|
) TAGS (
|
||||||
|
protocol NCHAR(32),
|
||||||
|
vehicle_key NCHAR(64),
|
||||||
|
vin NCHAR(32),
|
||||||
|
phone NCHAR(32),
|
||||||
|
device_id NCHAR(64)
|
||||||
|
)`,
|
||||||
|
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||||
|
ts TIMESTAMP,
|
||||||
|
event_id NCHAR(64),
|
||||||
|
frame_id NCHAR(64),
|
||||||
|
received_at TIMESTAMP,
|
||||||
|
total_mileage_km DOUBLE,
|
||||||
|
speed_kmh DOUBLE,
|
||||||
|
longitude DOUBLE,
|
||||||
|
latitude DOUBLE
|
||||||
|
) TAGS (
|
||||||
|
protocol NCHAR(32),
|
||||||
|
vehicle_key NCHAR(64),
|
||||||
|
vin NCHAR(32),
|
||||||
|
phone NCHAR(32),
|
||||||
|
device_id NCHAR(64)
|
||||||
|
)`,
|
||||||
|
}
|
||||||
|
}
|
||||||
336
go/vehicle-gateway/internal/history/writer.go
Normal file
336
go/vehicle-gateway/internal/history/writer.go
Normal file
@@ -0,0 +1,336 @@
|
|||||||
|
package history
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha1"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Execer interface {
|
||||||
|
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type Writer struct {
|
||||||
|
exec Execer
|
||||||
|
cache tableCache
|
||||||
|
}
|
||||||
|
|
||||||
|
type tableCache struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
seen map[string]struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWriter(exec Execer) *Writer {
|
||||||
|
if exec == nil {
|
||||||
|
panic("history execer must not be nil")
|
||||||
|
}
|
||||||
|
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
||||||
|
for _, statement := range SchemaStatements(database) {
|
||||||
|
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
if err := w.AppendRawFrame(ctx, env); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := w.AppendLocation(ctx, env); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return w.AppendMileagePoint(ctx, env)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
table := tableName("raw", env)
|
||||||
|
if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
|
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
||||||
|
raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
|
||||||
|
VALUES (%s)`, table, joinLiterals(rawValues(env))))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
longitude, okLon := floatField(env, envelope.FieldLongitude)
|
||||||
|
latitude, okLat := floatField(env, envelope.FieldLatitude)
|
||||||
|
if !okLon || !okLat {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
table := tableName("loc", env)
|
||||||
|
if err := w.ensureChild(ctx, table, "vehicle_locations", env); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
|
(ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
||||||
|
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
||||||
|
VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
table := tableName("mil", env)
|
||||||
|
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
|
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
|
||||||
|
VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage))))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
|
||||||
|
key := stable + "." + table
|
||||||
|
w.cache.mu.Lock()
|
||||||
|
_, ok := w.cache.seen[key]
|
||||||
|
w.cache.mu.Unlock()
|
||||||
|
if ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
|
||||||
|
table, stable,
|
||||||
|
quote(string(env.Protocol)),
|
||||||
|
quote(env.VehicleKey()),
|
||||||
|
quote(env.VIN),
|
||||||
|
quote(env.Phone),
|
||||||
|
quote(env.DeviceID))
|
||||||
|
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
w.cache.mu.Lock()
|
||||||
|
w.cache.seen[key] = struct{}{}
|
||||||
|
w.cache.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func rawValues(env envelope.FrameEnvelope) []any {
|
||||||
|
received := millis(env.ReceivedAtMS)
|
||||||
|
eventTime := millis(env.EventTimeMS)
|
||||||
|
return []any{
|
||||||
|
received,
|
||||||
|
frameID(env),
|
||||||
|
env.StableEventID(),
|
||||||
|
messageIDInt(env.MessageID),
|
||||||
|
eventTime,
|
||||||
|
received,
|
||||||
|
rawSize(env.RawHex),
|
||||||
|
env.RawHex,
|
||||||
|
env.RawText,
|
||||||
|
jsonString(env.Parsed),
|
||||||
|
jsonString(env.Fields),
|
||||||
|
string(env.ParseStatus),
|
||||||
|
env.ParseError,
|
||||||
|
env.SourceEndpoint,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
|
||||||
|
received := millis(env.ReceivedAtMS)
|
||||||
|
return []any{
|
||||||
|
eventTimeOrReceived(env),
|
||||||
|
env.StableEventID(),
|
||||||
|
frameID(env),
|
||||||
|
received,
|
||||||
|
longitude,
|
||||||
|
latitude,
|
||||||
|
floatFieldOrNil(env, "altitude_m"),
|
||||||
|
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||||
|
intFieldOrNil(env, "direction_deg"),
|
||||||
|
intFieldOrNil(env, "alarm_flag"),
|
||||||
|
intFieldOrNil(env, "status_flag"),
|
||||||
|
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
|
||||||
|
received := millis(env.ReceivedAtMS)
|
||||||
|
return []any{
|
||||||
|
eventTimeOrReceived(env),
|
||||||
|
env.StableEventID(),
|
||||||
|
frameID(env),
|
||||||
|
received,
|
||||||
|
totalMileage,
|
||||||
|
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||||
|
floatFieldOrNil(env, envelope.FieldLongitude),
|
||||||
|
floatFieldOrNil(env, envelope.FieldLatitude),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func frameID(env envelope.FrameEnvelope) string {
|
||||||
|
return "go_" + env.StableEventID()
|
||||||
|
}
|
||||||
|
|
||||||
|
func tableName(prefix string, env envelope.FrameEnvelope) string {
|
||||||
|
return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
|
||||||
|
}
|
||||||
|
|
||||||
|
func hash16(value string) string {
|
||||||
|
sum := sha1.Sum([]byte(value))
|
||||||
|
return hex.EncodeToString(sum[:8])
|
||||||
|
}
|
||||||
|
|
||||||
|
func messageIDInt(value string) int64 {
|
||||||
|
value = strings.TrimSpace(strings.ToLower(value))
|
||||||
|
value = strings.TrimPrefix(value, "0x")
|
||||||
|
parsed, err := strconv.ParseInt(value, 16, 32)
|
||||||
|
if err == nil {
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
parsed, _ = strconv.ParseInt(value, 10, 32)
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
func rawSize(rawHex string) int {
|
||||||
|
rawHex = strings.TrimSpace(rawHex)
|
||||||
|
if len(rawHex)%2 != 0 {
|
||||||
|
return len(rawHex) / 2
|
||||||
|
}
|
||||||
|
return len(rawHex) / 2
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonString(value any) string {
|
||||||
|
data, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return "{}"
|
||||||
|
}
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||||
|
if env.Fields == nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
value, ok := env.Fields[key]
|
||||||
|
if !ok || value == nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case float64:
|
||||||
|
return typed, true
|
||||||
|
case float32:
|
||||||
|
return float64(typed), true
|
||||||
|
case int:
|
||||||
|
return float64(typed), true
|
||||||
|
case int64:
|
||||||
|
return float64(typed), true
|
||||||
|
case uint16:
|
||||||
|
return float64(typed), true
|
||||||
|
case uint32:
|
||||||
|
return float64(typed), true
|
||||||
|
case string:
|
||||||
|
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||||
|
return parsed, err == nil
|
||||||
|
default:
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||||
|
value, ok := floatField(env, key)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||||
|
if env.Fields == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
value, ok := env.Fields[key]
|
||||||
|
if !ok || value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case int:
|
||||||
|
return typed
|
||||||
|
case int64:
|
||||||
|
return typed
|
||||||
|
case uint16:
|
||||||
|
return int64(typed)
|
||||||
|
case uint32:
|
||||||
|
return int64(typed)
|
||||||
|
case float64:
|
||||||
|
return int64(typed)
|
||||||
|
case string:
|
||||||
|
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||||
|
if err == nil {
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
|
||||||
|
if env.EventTimeMS > 0 {
|
||||||
|
return millis(env.EventTimeMS)
|
||||||
|
}
|
||||||
|
return millis(env.ReceivedAtMS)
|
||||||
|
}
|
||||||
|
|
||||||
|
func millis(value int64) time.Time {
|
||||||
|
if value <= 0 {
|
||||||
|
return time.UnixMilli(time.Now().UnixMilli()).UTC()
|
||||||
|
}
|
||||||
|
return time.UnixMilli(value).UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
func quote(value string) string {
|
||||||
|
return strings.ReplaceAll(value, "'", "''")
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinLiterals(values []any) string {
|
||||||
|
out := make([]string, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
out = append(out, literal(value))
|
||||||
|
}
|
||||||
|
return strings.Join(out, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func literal(value any) string {
|
||||||
|
if value == nil {
|
||||||
|
return "NULL"
|
||||||
|
}
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case time.Time:
|
||||||
|
return "'" + typed.UTC().Format("2006-01-02 15:04:05.000") + "'"
|
||||||
|
case string:
|
||||||
|
return "'" + quote(typed) + "'"
|
||||||
|
case envelope.ParseStatus:
|
||||||
|
return "'" + quote(string(typed)) + "'"
|
||||||
|
case int:
|
||||||
|
return strconv.FormatInt(int64(typed), 10)
|
||||||
|
case int64:
|
||||||
|
return strconv.FormatInt(typed, 10)
|
||||||
|
case uint16:
|
||||||
|
return strconv.FormatUint(uint64(typed), 10)
|
||||||
|
case uint32:
|
||||||
|
return strconv.FormatUint(uint64(typed), 10)
|
||||||
|
case float64:
|
||||||
|
return strconv.FormatFloat(typed, 'f', -1, 64)
|
||||||
|
case float32:
|
||||||
|
return strconv.FormatFloat(float64(typed), 'f', -1, 64)
|
||||||
|
default:
|
||||||
|
return "'" + quote(fmt.Sprint(typed)) + "'"
|
||||||
|
}
|
||||||
|
}
|
||||||
134
go/vehicle-gateway/internal/history/writer_test.go
Normal file
134
go/vehicle-gateway/internal/history/writer_test.go
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
package history
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
|
||||||
|
statements := strings.Join(SchemaStatements("test_ts"), "\n")
|
||||||
|
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} {
|
||||||
|
if !strings.Contains(statements, want) {
|
||||||
|
t.Fatalf("schema missing %q:\n%s", want, statements)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
|
||||||
|
exec := &recordingExec{}
|
||||||
|
writer := NewWriter(exec)
|
||||||
|
env := sampleEnvelope()
|
||||||
|
|
||||||
|
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||||
|
t.Fatalf("AppendAll() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := countSQL(exec.calls, "USING raw_frames"); got != 1 {
|
||||||
|
t.Fatalf("raw child create count = %d", got)
|
||||||
|
}
|
||||||
|
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
|
||||||
|
t.Fatalf("location child create count = %d", got)
|
||||||
|
}
|
||||||
|
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 {
|
||||||
|
t.Fatalf("mileage child create count = %d", got)
|
||||||
|
}
|
||||||
|
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||||
|
t.Fatalf("raw insert count = %d", got)
|
||||||
|
}
|
||||||
|
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
|
||||||
|
t.Fatalf("location insert count = %d", got)
|
||||||
|
}
|
||||||
|
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 {
|
||||||
|
t.Fatalf("mileage insert count = %d", got)
|
||||||
|
}
|
||||||
|
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
|
||||||
|
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {
|
||||||
|
t.Fatalf("raw insert should quote string literals: %s", rawInsert)
|
||||||
|
}
|
||||||
|
if strings.Contains(rawInsert, "VALUES (?,") {
|
||||||
|
t.Fatalf("raw insert should not use placeholders for tdengine websocket plain insert: %s", rawInsert)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
|
||||||
|
exec := &recordingExec{}
|
||||||
|
writer := NewWriter(exec)
|
||||||
|
env := sampleEnvelope()
|
||||||
|
env.Fields = map[string]any{}
|
||||||
|
|
||||||
|
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||||
|
t.Fatalf("AppendAll() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||||
|
t.Fatalf("raw insert count = %d", got)
|
||||||
|
}
|
||||||
|
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
|
||||||
|
t.Fatalf("location insert count = %d", got)
|
||||||
|
}
|
||||||
|
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
|
||||||
|
t.Fatalf("mileage insert count = %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sampleEnvelope() envelope.FrameEnvelope {
|
||||||
|
return envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
MessageID: "0x0200",
|
||||||
|
Sequence: 1,
|
||||||
|
VIN: "LNBVIN00000000001",
|
||||||
|
Phone: "013307795425",
|
||||||
|
SourceEndpoint: "127.0.0.1:18080",
|
||||||
|
EventTimeMS: 1782745114000,
|
||||||
|
ReceivedAtMS: 1782745114999,
|
||||||
|
RawHex: "7E0200",
|
||||||
|
Parsed: map[string]any{"message": "location"},
|
||||||
|
Fields: map[string]any{
|
||||||
|
envelope.FieldLongitude: 121.069881,
|
||||||
|
envelope.FieldLatitude: 30.590151,
|
||||||
|
envelope.FieldSpeedKMH: 23.0,
|
||||||
|
envelope.FieldTotalMileageKM: 10241.2,
|
||||||
|
"direction_deg": uint16(79),
|
||||||
|
"alarm_flag": uint32(0),
|
||||||
|
"status_flag": uint32(72),
|
||||||
|
},
|
||||||
|
ParseStatus: envelope.ParseOK,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func countSQL(calls []execCall, pattern string) int {
|
||||||
|
var count int
|
||||||
|
for _, call := range calls {
|
||||||
|
if strings.Contains(call.query, pattern) {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func findSQL(calls []execCall, pattern string) string {
|
||||||
|
for _, call := range calls {
|
||||||
|
if strings.Contains(call.query, pattern) {
|
||||||
|
return call.query
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type execCall struct {
|
||||||
|
query string
|
||||||
|
args []any
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingExec struct {
|
||||||
|
calls []execCall
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||||
|
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
112
go/vehicle-gateway/internal/identity/resolver.go
Normal file
112
go/vehicle-gateway/internal/identity/resolver.go
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
package identity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Resolver interface {
|
||||||
|
Resolve(context.Context, envelope.FrameEnvelope) (envelope.FrameEnvelope, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type NoopResolver struct{}
|
||||||
|
|
||||||
|
func (NoopResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||||
|
return env, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type MySQLResolver struct {
|
||||||
|
db *sql.DB
|
||||||
|
table string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver {
|
||||||
|
if db == nil {
|
||||||
|
panic("identity db must not be nil")
|
||||||
|
}
|
||||||
|
table = strings.TrimSpace(table)
|
||||||
|
if table == "" || !safeIdentifier(table) {
|
||||||
|
table = "vehicle_identity_binding"
|
||||||
|
}
|
||||||
|
return &MySQLResolver{db: db, table: table}
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeIdentifier(value string) bool {
|
||||||
|
if value == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, r := range value {
|
||||||
|
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||||
|
if strings.TrimSpace(env.VIN) != "" {
|
||||||
|
return env, nil
|
||||||
|
}
|
||||||
|
candidates := CandidateKeys(env)
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
vin, err := r.lookup(ctx, candidate.Column, candidate.Value)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return env, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(vin) != "" {
|
||||||
|
env.VIN = strings.TrimSpace(vin)
|
||||||
|
if env.Parsed == nil {
|
||||||
|
env.Parsed = map[string]any{}
|
||||||
|
}
|
||||||
|
env.Parsed["identity"] = map[string]any{
|
||||||
|
"resolved": true,
|
||||||
|
"source": candidate.Column,
|
||||||
|
"value": candidate.Value,
|
||||||
|
}
|
||||||
|
env.EventID = env.StableEventID()
|
||||||
|
return env, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return env, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) {
|
||||||
|
query := "SELECT vin FROM " + r.table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' ORDER BY updated_at DESC LIMIT 1"
|
||||||
|
var vin string
|
||||||
|
err := r.db.QueryRowContext(ctx, query, value).Scan(&vin)
|
||||||
|
return vin, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type CandidateKey struct {
|
||||||
|
Column string
|
||||||
|
Value string
|
||||||
|
}
|
||||||
|
|
||||||
|
func CandidateKeys(env envelope.FrameEnvelope) []CandidateKey {
|
||||||
|
var out []CandidateKey
|
||||||
|
add := func(column string, value string) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" || strings.EqualFold(value, "unknown") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, existing := range out {
|
||||||
|
if existing.Column == column && existing.Value == value {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, CandidateKey{Column: column, Value: value})
|
||||||
|
}
|
||||||
|
add("phone", env.Phone)
|
||||||
|
add("device_id", env.DeviceID)
|
||||||
|
add("plate", env.Plate)
|
||||||
|
add("vin", env.VehicleKeyHint)
|
||||||
|
return out
|
||||||
|
}
|
||||||
97
go/vehicle-gateway/internal/identity/resolver_test.go
Normal file
97
go/vehicle-gateway/internal/identity/resolver_test.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
package identity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DATA-DOG/go-sqlmock"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCandidateKeysPrioritizesPhoneDevicePlate(t *testing.T) {
|
||||||
|
keys := CandidateKeys(envelope.FrameEnvelope{
|
||||||
|
Phone: "013307795425",
|
||||||
|
DeviceID: "D1",
|
||||||
|
Plate: "豫A12345",
|
||||||
|
VehicleKeyHint: "D1",
|
||||||
|
})
|
||||||
|
got := []string{}
|
||||||
|
for _, key := range keys {
|
||||||
|
got = append(got, key.Column+"="+key.Value)
|
||||||
|
}
|
||||||
|
want := []string{"phone=013307795425", "device_id=D1", "plate=豫A12345", "vin=D1"}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("candidate count = %d got=%#v", len(got), got)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
|
||||||
|
db, mock := newMockDB(t)
|
||||||
|
defer db.Close()
|
||||||
|
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||||
|
WithArgs("013307795425").
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001"))
|
||||||
|
|
||||||
|
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||||
|
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
Phone: "013307795425",
|
||||||
|
Parsed: map[string]any{},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Resolve() error = %v", err)
|
||||||
|
}
|
||||||
|
if env.VIN != "LNBVIN00000000001" {
|
||||||
|
t.Fatalf("vin = %q", env.VIN)
|
||||||
|
}
|
||||||
|
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||||
|
if !ok || identity["source"] != "phone" {
|
||||||
|
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("sql expectations: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMySQLResolverFallsBackToDeviceID(t *testing.T) {
|
||||||
|
db, mock := newMockDB(t)
|
||||||
|
defer db.Close()
|
||||||
|
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||||
|
WithArgs("013307795425").
|
||||||
|
WillReturnError(sql.ErrNoRows)
|
||||||
|
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
|
||||||
|
WithArgs("D1").
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000002"))
|
||||||
|
|
||||||
|
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||||
|
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
Phone: "013307795425",
|
||||||
|
DeviceID: "D1",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Resolve() error = %v", err)
|
||||||
|
}
|
||||||
|
if env.VIN != "LNBVIN00000000002" {
|
||||||
|
t.Fatalf("vin = %q", env.VIN)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("sql expectations: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) {
|
||||||
|
t.Helper()
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("sqlmock.New() error = %v", err)
|
||||||
|
}
|
||||||
|
return db, mock
|
||||||
|
}
|
||||||
13
go/vehicle-gateway/internal/observability/logger.go
Normal file
13
go/vehicle-gateway/internal/observability/logger.go
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
package observability
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewLogger returns a JSON logger with a stable service field so ECS logs from
|
||||||
|
// different Go processes can be filtered without relying on container names.
|
||||||
|
func NewLogger(service string) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||||
|
return slog.New(handler).With("service", service)
|
||||||
|
}
|
||||||
521
go/vehicle-gateway/internal/protocol/gb32960/parser.go
Normal file
521
go/vehicle-gateway/internal/protocol/gb32960/parser.go
Normal file
@@ -0,0 +1,521 @@
|
|||||||
|
package gb32960
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrFrameTooShort = errors.New("gb32960 frame too short")
|
||||||
|
ErrBadStartSymbol = errors.New("gb32960 bad start symbol")
|
||||||
|
ErrBodyLength = errors.New("gb32960 body length mismatch")
|
||||||
|
ErrBCC = errors.New("gb32960 bcc mismatch")
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
minFrameLen = 25
|
||||||
|
headerLen = 24
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExtractFrames splits GB/T 32960 TCP streams using the protocol length field.
|
||||||
|
// Returned frames include the start symbols and trailing BCC byte.
|
||||||
|
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
|
||||||
|
offset := 0
|
||||||
|
for {
|
||||||
|
start := indexStart(stream[offset:])
|
||||||
|
if start < 0 {
|
||||||
|
return frames, nil, nil
|
||||||
|
}
|
||||||
|
offset += start
|
||||||
|
remaining := stream[offset:]
|
||||||
|
if len(remaining) < headerLen {
|
||||||
|
return frames, append([]byte(nil), remaining...), nil
|
||||||
|
}
|
||||||
|
bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
|
||||||
|
frameLen := headerLen + bodyLen + 1
|
||||||
|
if len(remaining) < frameLen {
|
||||||
|
return frames, append([]byte(nil), remaining...), nil
|
||||||
|
}
|
||||||
|
frames = append(frames, append([]byte(nil), remaining[:frameLen]...))
|
||||||
|
offset += frameLen
|
||||||
|
if offset >= len(stream) {
|
||||||
|
return frames, nil, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||||
|
if len(raw) < minFrameLen {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||||
|
}
|
||||||
|
version, ok := version(raw[0], raw[1])
|
||||||
|
if !ok {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
|
||||||
|
}
|
||||||
|
bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
|
||||||
|
if len(raw) != headerLen+bodyLen+1 {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
|
||||||
|
}
|
||||||
|
if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrBCC, got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
command := raw[2]
|
||||||
|
responseFlag := raw[3]
|
||||||
|
vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
|
||||||
|
body := raw[headerLen : headerLen+bodyLen]
|
||||||
|
parsed := map[string]any{
|
||||||
|
"header": map[string]any{
|
||||||
|
"version": version,
|
||||||
|
"command": fmt.Sprintf("0x%02X", command),
|
||||||
|
"response_flag": fmt.Sprintf("0x%02X", responseFlag),
|
||||||
|
"vin": vin,
|
||||||
|
"encrypt": raw[21],
|
||||||
|
"body_length": bodyLen,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fields := map[string]any{}
|
||||||
|
eventTimeMS := receivedAtMS
|
||||||
|
|
||||||
|
if command == 0x02 || command == 0x03 {
|
||||||
|
eventTime, units := parseDataBody(version, body, fields)
|
||||||
|
if !eventTime.IsZero() {
|
||||||
|
eventTimeMS = eventTime.UnixMilli()
|
||||||
|
fields["device_time"] = eventTime.Format(time.RFC3339)
|
||||||
|
parsed["device_time"] = eventTime.Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
parsed["data_units"] = units
|
||||||
|
}
|
||||||
|
|
||||||
|
env := envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolGB32960,
|
||||||
|
MessageID: fmt.Sprintf("0x%02X", command),
|
||||||
|
VIN: vin,
|
||||||
|
SourceEndpoint: sourceEndpoint,
|
||||||
|
EventTimeMS: eventTimeMS,
|
||||||
|
ReceivedAtMS: receivedAtMS,
|
||||||
|
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||||
|
Parsed: parsed,
|
||||||
|
Fields: fields,
|
||||||
|
ParseStatus: envelope.ParseOK,
|
||||||
|
}
|
||||||
|
env.EventID = env.StableEventID()
|
||||||
|
return env, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
|
||||||
|
if len(body) < 6 {
|
||||||
|
return time.Time{}, nil
|
||||||
|
}
|
||||||
|
eventTime := parseBCDTime(body[:6])
|
||||||
|
cursor := 6
|
||||||
|
var units []map[string]any
|
||||||
|
for cursor < len(body) {
|
||||||
|
unitType := body[cursor]
|
||||||
|
cursor++
|
||||||
|
switch unitType {
|
||||||
|
case 0x01:
|
||||||
|
size := 20
|
||||||
|
if version == "V2025" {
|
||||||
|
size = 18
|
||||||
|
}
|
||||||
|
if len(body[cursor:]) < size {
|
||||||
|
units = append(units, map[string]any{"type": "0x01", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parseVehicleData(body[cursor : cursor+size])
|
||||||
|
units = append(units, map[string]any{"type": "0x01", "name": "vehicle", "value": unit})
|
||||||
|
fields["vehicle_status"] = unit["vehicle_status"]
|
||||||
|
fields["charge_status"] = unit["charge_status"]
|
||||||
|
fields["running_mode"] = unit["running_mode"]
|
||||||
|
fields[envelope.FieldSpeedKMH] = unit["speed_kmh"]
|
||||||
|
fields[envelope.FieldTotalMileageKM] = unit["total_mileage_km"]
|
||||||
|
fields[envelope.FieldSOCPercent] = unit["soc_percent"]
|
||||||
|
cursor += size
|
||||||
|
case 0x05:
|
||||||
|
if len(body[cursor:]) < 9 {
|
||||||
|
units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parsePositionData(body[cursor : cursor+9])
|
||||||
|
units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
|
||||||
|
fields["position_status"] = unit["position_status"]
|
||||||
|
fields[envelope.FieldLongitude] = unit["longitude"]
|
||||||
|
fields[envelope.FieldLatitude] = unit["latitude"]
|
||||||
|
cursor += 9
|
||||||
|
case 0x02:
|
||||||
|
if len(body[cursor:]) < 1 {
|
||||||
|
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
size := 1 + int(body[cursor])*12
|
||||||
|
if len(body[cursor:]) < size {
|
||||||
|
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parseDriveMotorData(body[cursor : cursor+size])
|
||||||
|
units = append(units, map[string]any{"type": "0x02", "name": "drive_motor", "value": unit})
|
||||||
|
cursor += size
|
||||||
|
case 0x03:
|
||||||
|
if len(body[cursor:]) < 8 {
|
||||||
|
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
probeCount := int(binary.BigEndian.Uint16(body[cursor+6 : cursor+8]))
|
||||||
|
size := 8 + probeCount + 10
|
||||||
|
if len(body[cursor:]) < size {
|
||||||
|
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parseFuelCellData(body[cursor : cursor+size])
|
||||||
|
units = append(units, map[string]any{"type": "0x03", "name": "fuel_cell", "value": unit})
|
||||||
|
fields["fuel_cell_hydrogen_consumption_kg_per_100km"] = unit["hydrogen_consumption_kg_per_100km"]
|
||||||
|
fields["fuel_cell_voltage_v"] = unit["fuel_cell_voltage_v"]
|
||||||
|
fields["fuel_cell_current_a"] = unit["fuel_cell_current_a"]
|
||||||
|
cursor += size
|
||||||
|
case 0x04:
|
||||||
|
if len(body[cursor:]) < 5 {
|
||||||
|
units = append(units, map[string]any{"type": "0x04", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parseEngineData(body[cursor : cursor+5])
|
||||||
|
units = append(units, map[string]any{"type": "0x04", "name": "engine", "value": unit})
|
||||||
|
cursor += 5
|
||||||
|
case 0x06:
|
||||||
|
if len(body[cursor:]) < 14 {
|
||||||
|
units = append(units, map[string]any{"type": "0x06", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parseExtremeData(body[cursor : cursor+14])
|
||||||
|
units = append(units, map[string]any{"type": "0x06", "name": "extreme", "value": unit})
|
||||||
|
cursor += 14
|
||||||
|
case 0x07:
|
||||||
|
size, ok := alarmDataSize(body[cursor:])
|
||||||
|
if !ok {
|
||||||
|
units = append(units, map[string]any{"type": "0x07", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parseAlarmData(body[cursor : cursor+size])
|
||||||
|
units = append(units, map[string]any{"type": "0x07", "name": "alarm", "value": unit})
|
||||||
|
cursor += size
|
||||||
|
case 0x08:
|
||||||
|
size, ok := voltageDataSize(body[cursor:])
|
||||||
|
if !ok {
|
||||||
|
units = append(units, map[string]any{"type": "0x08", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parseVoltageData(body[cursor : cursor+size])
|
||||||
|
units = append(units, map[string]any{"type": "0x08", "name": "voltage", "value": unit})
|
||||||
|
cursor += size
|
||||||
|
case 0x09:
|
||||||
|
size, ok := temperatureDataSize(body[cursor:])
|
||||||
|
if !ok {
|
||||||
|
units = append(units, map[string]any{"type": "0x09", "error": "truncated"})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
unit := parseTemperatureData(body[cursor : cursor+size])
|
||||||
|
units = append(units, map[string]any{"type": "0x09", "name": "temperature", "value": unit})
|
||||||
|
cursor += size
|
||||||
|
default:
|
||||||
|
units = append(units, map[string]any{
|
||||||
|
"type": fmt.Sprintf("0x%02X", unitType),
|
||||||
|
"raw_tail": strings.ToUpper(hex.EncodeToString(body[cursor:])),
|
||||||
|
"parse": "unknown_unit",
|
||||||
|
"byte_size": len(body) - cursor,
|
||||||
|
})
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return eventTime, units
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseVehicleData(data []byte) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"vehicle_status": int(data[0]),
|
||||||
|
"charge_status": int(data[1]),
|
||||||
|
"running_mode": int(data[2]),
|
||||||
|
"speed_kmh": float64(binary.BigEndian.Uint16(data[3:5])) / 10,
|
||||||
|
"total_mileage_km": float64(binary.BigEndian.Uint32(data[5:9])) / 10,
|
||||||
|
"total_voltage_v": float64(binary.BigEndian.Uint16(data[9:11])) / 10,
|
||||||
|
"total_current_a": float64(binary.BigEndian.Uint16(data[11:13]))/10 - 1000,
|
||||||
|
"soc_percent": int(data[13]),
|
||||||
|
"dc_dc_status": int(data[14]),
|
||||||
|
"gear": int(data[15]),
|
||||||
|
"insulation_kohm": binary.BigEndian.Uint16(data[16:18]),
|
||||||
|
"accelerator_pct": int(data[18]),
|
||||||
|
"brake_pct": int(data[19]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDriveMotorData(data []byte) map[string]any {
|
||||||
|
count := int(data[0])
|
||||||
|
motors := make([]map[string]any, 0, count)
|
||||||
|
cursor := 1
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
motors = append(motors, map[string]any{
|
||||||
|
"serial_no": int(data[cursor]),
|
||||||
|
"state": int(data[cursor+1]),
|
||||||
|
"controller_temperature_c": int(data[cursor+2]) - 40,
|
||||||
|
"speed_rpm": int(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) - 20000,
|
||||||
|
"torque_nm": float64(int(binary.BigEndian.Uint16(data[cursor+5:cursor+7]))-20000) / 10,
|
||||||
|
"motor_temperature_c": int(data[cursor+7]) - 40,
|
||||||
|
"controller_voltage_v": float64(binary.BigEndian.Uint16(data[cursor+8:cursor+10])) / 10,
|
||||||
|
"controller_current_a": float64(binary.BigEndian.Uint16(data[cursor+10:cursor+12]))/10 - 1000,
|
||||||
|
})
|
||||||
|
cursor += 12
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"count": count,
|
||||||
|
"motors": motors,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseFuelCellData(data []byte) map[string]any {
|
||||||
|
probeCount := int(binary.BigEndian.Uint16(data[6:8]))
|
||||||
|
probes := make([]int, 0, probeCount)
|
||||||
|
cursor := 8
|
||||||
|
for i := 0; i < probeCount; i++ {
|
||||||
|
probes = append(probes, int(data[cursor])-40)
|
||||||
|
cursor++
|
||||||
|
}
|
||||||
|
out := map[string]any{
|
||||||
|
"fuel_cell_voltage_v": float64(binary.BigEndian.Uint16(data[0:2])) / 10,
|
||||||
|
"fuel_cell_current_a": float64(binary.BigEndian.Uint16(data[2:4])) / 10,
|
||||||
|
"hydrogen_consumption_kg_per_100km": float64(binary.BigEndian.Uint16(data[4:6])) / 100,
|
||||||
|
"temperature_probe_count": probeCount,
|
||||||
|
"temperature_probe_values_c": probes,
|
||||||
|
"max_hydrogen_temperature_c": float64(binary.BigEndian.Uint16(data[cursor:cursor+2]))/10 - 40,
|
||||||
|
"max_hydrogen_temperature_probe_id": int(data[cursor+2]),
|
||||||
|
"max_hydrogen_concentration_fraction": float64(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) * 0.000001,
|
||||||
|
"max_hydrogen_concentration_probe_id": int(data[cursor+5]),
|
||||||
|
"max_hydrogen_pressure_mpa": float64(binary.BigEndian.Uint16(data[cursor+6:cursor+8])) / 10,
|
||||||
|
"max_hydrogen_pressure_probe_id": int(data[cursor+8]),
|
||||||
|
"dc_dc_status": int(data[cursor+9]),
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseEngineData(data []byte) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"engine_status": int(data[0]),
|
||||||
|
"crank_speed_rpm": binary.BigEndian.Uint16(data[1:3]),
|
||||||
|
"fuel_rate": binary.BigEndian.Uint16(data[3:5]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseExtremeData(data []byte) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"max_voltage_subsystem_no": int(data[0]),
|
||||||
|
"max_voltage_cell_no": int(data[1]),
|
||||||
|
"max_voltage_v": float64(binary.BigEndian.Uint16(data[2:4])) / 1000,
|
||||||
|
"min_voltage_subsystem_no": int(data[4]),
|
||||||
|
"min_voltage_cell_no": int(data[5]),
|
||||||
|
"min_voltage_v": float64(binary.BigEndian.Uint16(data[6:8])) / 1000,
|
||||||
|
"max_temp_subsystem_no": int(data[8]),
|
||||||
|
"max_temp_probe_no": int(data[9]),
|
||||||
|
"max_temp_c": int(data[10]) - 40,
|
||||||
|
"min_temp_subsystem_no": int(data[11]),
|
||||||
|
"min_temp_probe_no": int(data[12]),
|
||||||
|
"min_temp_c": int(data[13]) - 40,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func alarmDataSize(data []byte) (int, bool) {
|
||||||
|
if len(data) < 5 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
cursor := 5
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
if len(data[cursor:]) < 1 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
count := int(data[cursor])
|
||||||
|
cursor++
|
||||||
|
cursor += count * 4
|
||||||
|
if cursor > len(data) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cursor, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseAlarmData(data []byte) map[string]any {
|
||||||
|
cursor := 5
|
||||||
|
readList := func() []string {
|
||||||
|
count := int(data[cursor])
|
||||||
|
cursor++
|
||||||
|
values := make([]string, 0, count)
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
values = append(values, fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[cursor:cursor+4])))
|
||||||
|
cursor += 4
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"max_alarm_level": int(data[0]),
|
||||||
|
"general_alarm_flag": fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[1:5])),
|
||||||
|
"battery_faults": readList(),
|
||||||
|
"motor_faults": readList(),
|
||||||
|
"engine_faults": readList(),
|
||||||
|
"other_faults": readList(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func voltageDataSize(data []byte) (int, bool) {
|
||||||
|
if len(data) < 1 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
cursor := 1
|
||||||
|
for i := 0; i < int(data[0]); i++ {
|
||||||
|
if len(data[cursor:]) < 10 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
frameCellCount := int(data[cursor+9])
|
||||||
|
cursor += 10 + frameCellCount*2
|
||||||
|
if cursor > len(data) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cursor, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseVoltageData(data []byte) map[string]any {
|
||||||
|
subCount := int(data[0])
|
||||||
|
cursor := 1
|
||||||
|
totalVoltage := 0.0
|
||||||
|
maxCell := 0.0
|
||||||
|
minCell := 0.0
|
||||||
|
cellSeen := false
|
||||||
|
for i := 0; i < subCount; i++ {
|
||||||
|
totalVoltage += float64(binary.BigEndian.Uint16(data[cursor+1:cursor+3])) / 10
|
||||||
|
cellCount := int(binary.BigEndian.Uint16(data[cursor+5 : cursor+7]))
|
||||||
|
frameCellCount := int(data[cursor+9])
|
||||||
|
cursor += 10
|
||||||
|
for c := 0; c < frameCellCount; c++ {
|
||||||
|
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
|
||||||
|
if !cellSeen || voltage > maxCell {
|
||||||
|
maxCell = voltage
|
||||||
|
}
|
||||||
|
if !cellSeen || voltage < minCell {
|
||||||
|
minCell = voltage
|
||||||
|
}
|
||||||
|
cellSeen = true
|
||||||
|
cursor += 2
|
||||||
|
}
|
||||||
|
_ = cellCount
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"subsystem_count": subCount,
|
||||||
|
"total_voltage_v": totalVoltage,
|
||||||
|
"max_cell_v": maxCell,
|
||||||
|
"min_cell_v": minCell,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func temperatureDataSize(data []byte) (int, bool) {
|
||||||
|
if len(data) < 1 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
cursor := 1
|
||||||
|
for i := 0; i < int(data[0]); i++ {
|
||||||
|
if len(data[cursor:]) < 3 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
|
||||||
|
cursor += 3 + probeCount
|
||||||
|
if cursor > len(data) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cursor, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseTemperatureData(data []byte) map[string]any {
|
||||||
|
subCount := int(data[0])
|
||||||
|
cursor := 1
|
||||||
|
maxTemp := 0
|
||||||
|
minTemp := 0
|
||||||
|
seen := false
|
||||||
|
for i := 0; i < subCount; i++ {
|
||||||
|
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
|
||||||
|
cursor += 3
|
||||||
|
for p := 0; p < probeCount; p++ {
|
||||||
|
temp := int(data[cursor]) - 40
|
||||||
|
if !seen || temp > maxTemp {
|
||||||
|
maxTemp = temp
|
||||||
|
}
|
||||||
|
if !seen || temp < minTemp {
|
||||||
|
minTemp = temp
|
||||||
|
}
|
||||||
|
seen = true
|
||||||
|
cursor++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"subsystem_count": subCount,
|
||||||
|
"max_temp_c": maxTemp,
|
||||||
|
"min_temp_c": minTemp,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePositionData(data []byte) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"position_status": int(data[0]),
|
||||||
|
"longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
|
||||||
|
"latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexStart(data []byte) int {
|
||||||
|
for i := 0; i+1 < len(data); i++ {
|
||||||
|
if _, ok := version(data[i], data[i+1]); ok {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
func version(a byte, b byte) (string, bool) {
|
||||||
|
switch {
|
||||||
|
case a == '#' && b == '#':
|
||||||
|
return "V2016", true
|
||||||
|
case a == '$' && b == '$':
|
||||||
|
return "V2025", true
|
||||||
|
default:
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseBCDTime(data []byte) time.Time {
|
||||||
|
if len(data) != 6 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
year := 2000 + bcdByte(data[0])
|
||||||
|
month := time.Month(bcdByte(data[1]))
|
||||||
|
day := bcdByte(data[2])
|
||||||
|
hour := bcdByte(data[3])
|
||||||
|
minute := bcdByte(data[4])
|
||||||
|
second := bcdByte(data[5])
|
||||||
|
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||||
|
}
|
||||||
|
|
||||||
|
func bcdByte(value byte) int {
|
||||||
|
return int(value>>4)*10 + int(value&0x0f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func bcc(data []byte) byte {
|
||||||
|
var out byte
|
||||||
|
for _, value := range data {
|
||||||
|
out ^= value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
150
go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
Normal file
150
go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
package gb32960
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
|
||||||
|
|
||||||
|
func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
|
||||||
|
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57})
|
||||||
|
second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
|
||||||
|
stream := append([]byte{0x00, 0x01}, first...)
|
||||||
|
stream = append(stream, second[:len(second)-3]...)
|
||||||
|
|
||||||
|
frames, remainder, err := ExtractFrames(stream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExtractFrames() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(frames) != 1 {
|
||||||
|
t.Fatalf("expected one complete frame, got %d", len(frames))
|
||||||
|
}
|
||||||
|
if string(remainder) != string(second[:len(second)-3]) {
|
||||||
|
t.Fatalf("expected partial second frame as remainder, got %s", hex.EncodeToString(remainder))
|
||||||
|
}
|
||||||
|
|
||||||
|
env, err := ParseFrame(frames[0], 1782745114999, "115.29.187.205:32960")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseFrame() error = %v", err)
|
||||||
|
}
|
||||||
|
if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x02" {
|
||||||
|
t.Fatalf("unexpected envelope header: %#v", env)
|
||||||
|
}
|
||||||
|
if env.VIN != "LNBSCB3D4R1234567" {
|
||||||
|
t.Fatalf("unexpected vin: %q", env.VIN)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseFrameRejectsBadBCC(t *testing.T) {
|
||||||
|
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||||
|
frame[len(frame)-1] ^= 0xff
|
||||||
|
|
||||||
|
if _, err := ParseFrame(frame, 1782745114999, ""); err == nil {
|
||||||
|
t.Fatal("expected bad bcc error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
|
||||||
|
body := []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57}
|
||||||
|
body = append(body, 0x01)
|
||||||
|
body = append(body,
|
||||||
|
0x01, // vehicle status
|
||||||
|
0x03, // charge status
|
||||||
|
0x02, // running mode
|
||||||
|
0x01, 0x2c, // speed: 30.0km/h
|
||||||
|
0x00, 0x01, 0x86, 0xa0, // total mileage: 10000.0km
|
||||||
|
0x15, 0xe5, // total voltage: 560.5V
|
||||||
|
0x27, 0x10, // total current: 0A after -1000 offset
|
||||||
|
85, 0x01, 0x00,
|
||||||
|
0x27, 0x10,
|
||||||
|
0x00, 0x00)
|
||||||
|
body = append(body, 0x05)
|
||||||
|
body = append(body,
|
||||||
|
0x00,
|
||||||
|
0x07, 0x36, 0x50, 0x40, // longitude: 121.000000
|
||||||
|
0x01, 0xd2, 0x4f, 0x00) // latitude: 30.560000
|
||||||
|
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", body)
|
||||||
|
|
||||||
|
env, err := ParseFrame(frame, 1782745114999, "115.29.187.205:32960")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseFrame() error = %v", err)
|
||||||
|
}
|
||||||
|
assertFloatField(t, env, envelope.FieldSpeedKMH, 30.0)
|
||||||
|
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10000.0)
|
||||||
|
assertFloatField(t, env, envelope.FieldLongitude, 121.0)
|
||||||
|
assertFloatField(t, env, envelope.FieldLatitude, 30.56)
|
||||||
|
if env.Fields[envelope.FieldSOCPercent] != 85 {
|
||||||
|
t.Fatalf("unexpected soc: %#v", env.Fields[envelope.FieldSOCPercent])
|
||||||
|
}
|
||||||
|
if env.EventTimeMS == 1782745114999 {
|
||||||
|
t.Fatal("event time should come from GB32960 device timestamp")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testing.T) {
|
||||||
|
frame, err := hex.DecodeString(realFuelCellFrame)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
env, err := ParseFrame(frame, 1782914969584, "115.29.187.205:32960")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseFrame() error = %v", err)
|
||||||
|
}
|
||||||
|
if env.VIN != "LB9A32A21R0LS1707" {
|
||||||
|
t.Fatalf("vin = %q", env.VIN)
|
||||||
|
}
|
||||||
|
assertFloatField(t, env, envelope.FieldTotalMileageKM, 53490.9)
|
||||||
|
assertFloatField(t, env, "fuel_cell_hydrogen_consumption_kg_per_100km", 1.8)
|
||||||
|
assertFloatField(t, env, envelope.FieldLongitude, 120.800326)
|
||||||
|
assertFloatField(t, env, envelope.FieldLatitude, 31.634907)
|
||||||
|
|
||||||
|
units, ok := env.Parsed["data_units"].([]map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("data_units missing: %#v", env.Parsed["data_units"])
|
||||||
|
}
|
||||||
|
gotTypes := make([]string, 0, len(units))
|
||||||
|
for _, unit := range units {
|
||||||
|
gotTypes = append(gotTypes, fmt.Sprint(unit["type"]))
|
||||||
|
}
|
||||||
|
wantTypes := []string{"0x01", "0x02", "0x03", "0x04", "0x05", "0x06", "0x07", "0x08", "0x09", "0x30"}
|
||||||
|
if fmt.Sprint(gotTypes) != fmt.Sprint(wantTypes) {
|
||||||
|
t.Fatalf("unit types = %v, want %v", gotTypes, wantTypes)
|
||||||
|
}
|
||||||
|
if units[1]["name"] != "drive_motor" {
|
||||||
|
t.Fatalf("unexpected motor unit: %#v", units[1])
|
||||||
|
}
|
||||||
|
if units[2]["name"] != "fuel_cell" {
|
||||||
|
t.Fatalf("unexpected fuel cell unit: %#v", units[2])
|
||||||
|
}
|
||||||
|
if units[9]["parse"] != "unknown_unit" {
|
||||||
|
t.Fatalf("unexpected unknown unit: %#v", units[9])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||||
|
t.Helper()
|
||||||
|
got, ok := env.Fields[key].(float64)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildFrame(command byte, response byte, vin string, body []byte) []byte {
|
||||||
|
frame := []byte{'#', '#', command, response}
|
||||||
|
vinBytes := []byte(vin)
|
||||||
|
if len(vinBytes) < 17 {
|
||||||
|
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
|
||||||
|
}
|
||||||
|
frame = append(frame, vinBytes[:17]...)
|
||||||
|
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
|
||||||
|
frame = append(frame, body...)
|
||||||
|
return append(frame, bcc(frame[2:]))
|
||||||
|
}
|
||||||
321
go/vehicle-gateway/internal/protocol/jt808/parser.go
Normal file
321
go/vehicle-gateway/internal/protocol/jt808/parser.go
Normal file
@@ -0,0 +1,321 @@
|
|||||||
|
package jt808
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrFrameTooShort = errors.New("jt808 frame too short")
|
||||||
|
ErrChecksum = errors.New("jt808 checksum mismatch")
|
||||||
|
ErrEscape = errors.New("jt808 invalid escape sequence")
|
||||||
|
)
|
||||||
|
|
||||||
|
type Location struct {
|
||||||
|
AlarmFlag uint32
|
||||||
|
StatusFlag uint32
|
||||||
|
Latitude float64
|
||||||
|
Longitude float64
|
||||||
|
AltitudeM uint16
|
||||||
|
SpeedKMH float64
|
||||||
|
DirectionDeg uint16
|
||||||
|
Time time.Time
|
||||||
|
TotalMileageKM *float64
|
||||||
|
Additional []AdditionalItem
|
||||||
|
}
|
||||||
|
|
||||||
|
type AdditionalItem struct {
|
||||||
|
ID byte
|
||||||
|
Length int
|
||||||
|
ValueHex string
|
||||||
|
Parsed any
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractFrames splits JT/T 808 TCP streams by 0x7e delimiters and unescapes
|
||||||
|
// complete frame payloads. Returned frames exclude the leading and trailing 0x7e.
|
||||||
|
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
|
||||||
|
start := -1
|
||||||
|
for i, value := range stream {
|
||||||
|
if value != 0x7e {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if start < 0 {
|
||||||
|
start = i
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if i == start+1 {
|
||||||
|
start = i
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
frame, err := unescape(stream[start+1 : i])
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
frames = append(frames, frame)
|
||||||
|
start = i
|
||||||
|
}
|
||||||
|
if start >= 0 && start < len(stream)-1 {
|
||||||
|
return frames, append([]byte(nil), stream[start:]...), nil
|
||||||
|
}
|
||||||
|
return frames, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||||
|
if len(raw) < 13 {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||||
|
}
|
||||||
|
if got, want := checksum(raw[:len(raw)-1]), raw[len(raw)-1]; got != want {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrChecksum, got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
messageID := binary.BigEndian.Uint16(raw[0:2])
|
||||||
|
props := binary.BigEndian.Uint16(raw[2:4])
|
||||||
|
bodySize := props & 0x03ff
|
||||||
|
versioned := props&0x4000 != 0
|
||||||
|
headerLen := 12
|
||||||
|
phoneStart := 4
|
||||||
|
phoneEnd := 10
|
||||||
|
serialStart := 10
|
||||||
|
packageInfo := map[string]any(nil)
|
||||||
|
protocolVersion := byte(0)
|
||||||
|
if versioned {
|
||||||
|
headerLen = 17
|
||||||
|
if len(raw) < headerLen {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: versioned header len=%d", ErrFrameTooShort, len(raw))
|
||||||
|
}
|
||||||
|
protocolVersion = raw[4]
|
||||||
|
phoneStart = 5
|
||||||
|
phoneEnd = 15
|
||||||
|
serialStart = 15
|
||||||
|
}
|
||||||
|
if props&0x2000 != 0 {
|
||||||
|
headerLen += 4
|
||||||
|
packageInfo = map[string]any{
|
||||||
|
"total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]),
|
||||||
|
"index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(raw) < headerLen+int(bodySize)+1 {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
|
||||||
|
}
|
||||||
|
|
||||||
|
phoneBCD := raw[phoneStart:phoneEnd]
|
||||||
|
phone := bcdString(phoneBCD)
|
||||||
|
phoneTrimZero := strings.TrimLeft(phone, "0")
|
||||||
|
if versioned && phoneTrimZero != "" {
|
||||||
|
phone = phoneTrimZero
|
||||||
|
}
|
||||||
|
body := raw[headerLen : headerLen+int(bodySize)]
|
||||||
|
parsed := map[string]any{
|
||||||
|
"header": map[string]any{
|
||||||
|
"message_id": fmt.Sprintf("0x%04X", messageID),
|
||||||
|
"protocol_version": protocolVersion,
|
||||||
|
"versioned": versioned,
|
||||||
|
"properties": props,
|
||||||
|
"body_size": bodySize,
|
||||||
|
"phone_bcd_hex": strings.ToUpper(hex.EncodeToString(phoneBCD)),
|
||||||
|
"phone": phone,
|
||||||
|
"phone_trim_zero": phoneTrimZero,
|
||||||
|
"sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
|
||||||
|
"subpackage": props&0x2000 != 0,
|
||||||
|
"package_info": packageInfo,
|
||||||
|
"encryption_flags": (props >> 10) & 0x07,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fields := map[string]any{}
|
||||||
|
eventTimeMS := receivedAtMS
|
||||||
|
|
||||||
|
if messageID == 0x0200 {
|
||||||
|
location, err := parseLocation(body)
|
||||||
|
if err != nil {
|
||||||
|
return envelope.FrameEnvelope{}, err
|
||||||
|
}
|
||||||
|
parsed["location"] = locationToMap(location)
|
||||||
|
fields[envelope.FieldLatitude] = location.Latitude
|
||||||
|
fields[envelope.FieldLongitude] = location.Longitude
|
||||||
|
fields[envelope.FieldSpeedKMH] = location.SpeedKMH
|
||||||
|
fields["altitude_m"] = location.AltitudeM
|
||||||
|
fields["direction_deg"] = location.DirectionDeg
|
||||||
|
fields["alarm_flag"] = location.AlarmFlag
|
||||||
|
fields["status_flag"] = location.StatusFlag
|
||||||
|
if !location.Time.IsZero() {
|
||||||
|
eventTimeMS = location.Time.UnixMilli()
|
||||||
|
fields["device_time"] = location.Time.Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
if location.TotalMileageKM != nil {
|
||||||
|
fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
env := envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
MessageID: fmt.Sprintf("0x%04X", messageID),
|
||||||
|
Sequence: binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
|
||||||
|
Phone: phone,
|
||||||
|
SourceEndpoint: sourceEndpoint,
|
||||||
|
EventTimeMS: eventTimeMS,
|
||||||
|
ReceivedAtMS: receivedAtMS,
|
||||||
|
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||||
|
Parsed: parsed,
|
||||||
|
Fields: fields,
|
||||||
|
ParseStatus: envelope.ParseOK,
|
||||||
|
}
|
||||||
|
env.EventID = env.StableEventID()
|
||||||
|
return env, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseLocation(body []byte) (Location, error) {
|
||||||
|
if len(body) < 28 {
|
||||||
|
return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
|
||||||
|
}
|
||||||
|
location := Location{
|
||||||
|
AlarmFlag: binary.BigEndian.Uint32(body[0:4]),
|
||||||
|
StatusFlag: binary.BigEndian.Uint32(body[4:8]),
|
||||||
|
Latitude: float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000,
|
||||||
|
Longitude: float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000,
|
||||||
|
AltitudeM: binary.BigEndian.Uint16(body[16:18]),
|
||||||
|
SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10,
|
||||||
|
DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
|
||||||
|
Time: parseBCDTime(body[22:28]),
|
||||||
|
}
|
||||||
|
location.Additional = parseAdditional(body[28:], &location)
|
||||||
|
return location, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseAdditional(data []byte, location *Location) []AdditionalItem {
|
||||||
|
var out []AdditionalItem
|
||||||
|
for len(data) >= 2 {
|
||||||
|
id := data[0]
|
||||||
|
size := int(data[1])
|
||||||
|
data = data[2:]
|
||||||
|
if len(data) < size {
|
||||||
|
out = append(out, AdditionalItem{
|
||||||
|
ID: id,
|
||||||
|
Length: size,
|
||||||
|
ValueHex: strings.ToUpper(hex.EncodeToString(data)),
|
||||||
|
Parsed: "truncated",
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
value := data[:size]
|
||||||
|
item := AdditionalItem{
|
||||||
|
ID: id,
|
||||||
|
Length: size,
|
||||||
|
ValueHex: strings.ToUpper(hex.EncodeToString(value)),
|
||||||
|
}
|
||||||
|
// JT/T 808-2011 table 27: id 0x01 is GPS total mileage, DWORD, 0.1 km.
|
||||||
|
if id == 0x01 && size == 4 {
|
||||||
|
mileage := float64(binary.BigEndian.Uint32(value)) / 10
|
||||||
|
location.TotalMileageKM = &mileage
|
||||||
|
item.Parsed = map[string]any{
|
||||||
|
"name": envelope.FieldTotalMileageKM,
|
||||||
|
"value": mileage,
|
||||||
|
"unit": "km",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, item)
|
||||||
|
data = data[size:]
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func locationToMap(location Location) map[string]any {
|
||||||
|
out := map[string]any{
|
||||||
|
"alarm_flag": location.AlarmFlag,
|
||||||
|
"status_flag": location.StatusFlag,
|
||||||
|
"latitude": location.Latitude,
|
||||||
|
"longitude": location.Longitude,
|
||||||
|
"altitude_m": location.AltitudeM,
|
||||||
|
"speed_kmh": location.SpeedKMH,
|
||||||
|
"direction_deg": location.DirectionDeg,
|
||||||
|
"additional": additionalToMaps(location.Additional),
|
||||||
|
}
|
||||||
|
if !location.Time.IsZero() {
|
||||||
|
out["device_time"] = location.Time.Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
if location.TotalMileageKM != nil {
|
||||||
|
out[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func additionalToMaps(items []AdditionalItem) []map[string]any {
|
||||||
|
out := make([]map[string]any, 0, len(items))
|
||||||
|
for _, item := range items {
|
||||||
|
value := map[string]any{
|
||||||
|
"id": fmt.Sprintf("0x%02X", item.ID),
|
||||||
|
"length": item.Length,
|
||||||
|
"value_hex": item.ValueHex,
|
||||||
|
}
|
||||||
|
if item.Parsed != nil {
|
||||||
|
value["parsed"] = item.Parsed
|
||||||
|
}
|
||||||
|
out = append(out, value)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func bcdString(data []byte) string {
|
||||||
|
out := make([]byte, 0, len(data)*2)
|
||||||
|
for _, value := range data {
|
||||||
|
out = append(out, '0'+((value>>4)&0x0f), '0'+(value&0x0f))
|
||||||
|
}
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseBCDTime(data []byte) time.Time {
|
||||||
|
if len(data) != 6 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
year := 2000 + bcdByte(data[0])
|
||||||
|
month := time.Month(bcdByte(data[1]))
|
||||||
|
day := bcdByte(data[2])
|
||||||
|
hour := bcdByte(data[3])
|
||||||
|
minute := bcdByte(data[4])
|
||||||
|
second := bcdByte(data[5])
|
||||||
|
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||||
|
}
|
||||||
|
|
||||||
|
func bcdByte(value byte) int {
|
||||||
|
return int(value>>4)*10 + int(value&0x0f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func unescape(data []byte) ([]byte, error) {
|
||||||
|
out := make([]byte, 0, len(data))
|
||||||
|
for i := 0; i < len(data); i++ {
|
||||||
|
if data[i] != 0x7d {
|
||||||
|
out = append(out, data[i])
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if i+1 >= len(data) {
|
||||||
|
return nil, ErrEscape
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
switch data[i] {
|
||||||
|
case 0x01:
|
||||||
|
out = append(out, 0x7d)
|
||||||
|
case 0x02:
|
||||||
|
out = append(out, 0x7e)
|
||||||
|
default:
|
||||||
|
return nil, ErrEscape
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checksum(data []byte) byte {
|
||||||
|
var out byte
|
||||||
|
for _, value := range data {
|
||||||
|
out ^= value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
158
go/vehicle-gateway/internal/protocol/jt808/parser_test.go
Normal file
158
go/vehicle-gateway/internal/protocol/jt808/parser_test.go
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
package jt808
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
|
||||||
|
const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E"
|
||||||
|
|
||||||
|
func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
|
||||||
|
stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
frames, remainder, err := ExtractFrames(stream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExtractFrames() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(frames) != 1 {
|
||||||
|
t.Fatalf("expected one complete frame, got %d", len(frames))
|
||||||
|
}
|
||||||
|
if hex.EncodeToString(remainder) != "7e02000000" {
|
||||||
|
t.Fatalf("expected partial remainder, got %s", hex.EncodeToString(remainder))
|
||||||
|
}
|
||||||
|
|
||||||
|
env, err := ParseFrame(frames[0], 1782745114999, "115.231.168.135:43625")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseFrame() error = %v", err)
|
||||||
|
}
|
||||||
|
if env.Protocol != envelope.ProtocolJT808 {
|
||||||
|
t.Fatalf("protocol = %q", env.Protocol)
|
||||||
|
}
|
||||||
|
if env.MessageID != "0x0200" {
|
||||||
|
t.Fatalf("message id = %q", env.MessageID)
|
||||||
|
}
|
||||||
|
if env.Phone != "013307795425" {
|
||||||
|
t.Fatalf("phone = %q", env.Phone)
|
||||||
|
}
|
||||||
|
if env.Sequence != 1 {
|
||||||
|
t.Fatalf("sequence = %d", env.Sequence)
|
||||||
|
}
|
||||||
|
assertFloatField(t, env, envelope.FieldLatitude, 30.590151)
|
||||||
|
assertFloatField(t, env, envelope.FieldLongitude, 121.069881)
|
||||||
|
assertFloatField(t, env, envelope.FieldSpeedKMH, 23.0)
|
||||||
|
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10241.2)
|
||||||
|
if env.Fields["direction_deg"] != uint16(79) {
|
||||||
|
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
|
||||||
|
}
|
||||||
|
|
||||||
|
location, ok := env.Parsed["location"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("parsed location missing: %#v", env.Parsed)
|
||||||
|
}
|
||||||
|
additional, ok := location["additional"].([]map[string]any)
|
||||||
|
if !ok || len(additional) == 0 {
|
||||||
|
t.Fatalf("additional fields missing: %#v", location["additional"])
|
||||||
|
}
|
||||||
|
if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
|
||||||
|
t.Fatalf("unexpected first additional item: %#v", additional[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
|
||||||
|
stream, err := hex.DecodeString(real2019LocationFrame)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
frames, remainder, err := ExtractFrames(stream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExtractFrames() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(frames) != 1 || len(remainder) != 0 {
|
||||||
|
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||||
|
}
|
||||||
|
|
||||||
|
env, err := ParseFrame(frames[0], 1782914967713, "115.231.168.135:43625")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseFrame() error = %v", err)
|
||||||
|
}
|
||||||
|
if env.MessageID != "0x0200" {
|
||||||
|
t.Fatalf("message id = %q", env.MessageID)
|
||||||
|
}
|
||||||
|
if env.Phone != "14894135060" {
|
||||||
|
t.Fatalf("phone = %q", env.Phone)
|
||||||
|
}
|
||||||
|
if env.Sequence != 1995 {
|
||||||
|
t.Fatalf("sequence = %d", env.Sequence)
|
||||||
|
}
|
||||||
|
assertFloatField(t, env, envelope.FieldLatitude, 30.570155)
|
||||||
|
assertFloatField(t, env, envelope.FieldLongitude, 117.178187)
|
||||||
|
assertFloatField(t, env, envelope.FieldSpeedKMH, 33.4)
|
||||||
|
assertFloatField(t, env, envelope.FieldTotalMileageKM, 7868.9)
|
||||||
|
if env.Fields["altitude_m"] != uint16(13) {
|
||||||
|
t.Fatalf("altitude = %#v", env.Fields["altitude_m"])
|
||||||
|
}
|
||||||
|
if env.Fields["direction_deg"] != uint16(232) {
|
||||||
|
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
|
||||||
|
}
|
||||||
|
|
||||||
|
header, ok := env.Parsed["header"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("parsed header missing: %#v", env.Parsed)
|
||||||
|
}
|
||||||
|
if header["versioned"] != true || header["protocol_version"] != byte(1) {
|
||||||
|
t.Fatalf("unexpected versioned header: %#v", header)
|
||||||
|
}
|
||||||
|
if header["phone_bcd_hex"] != "00000000014894135060" {
|
||||||
|
t.Fatalf("phone bcd = %#v", header["phone_bcd_hex"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
|
||||||
|
payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
|
||||||
|
frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
|
||||||
|
frame = append(frame, 0x7e)
|
||||||
|
|
||||||
|
frames, remainder, err := ExtractFrames(frame)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExtractFrames() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(frames) != 1 || len(remainder) != 0 {
|
||||||
|
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||||
|
}
|
||||||
|
if hex.EncodeToString(frames[0][:len(payload)]) != hex.EncodeToString(payload) {
|
||||||
|
t.Fatalf("frame was not unescaped: %s", hex.EncodeToString(frames[0]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||||
|
t.Helper()
|
||||||
|
got, ok := env.Fields[key].(float64)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func escape(payload []byte) []byte {
|
||||||
|
var out []byte
|
||||||
|
for _, value := range payload {
|
||||||
|
switch value {
|
||||||
|
case 0x7e:
|
||||||
|
out = append(out, 0x7d, 0x02)
|
||||||
|
case 0x7d:
|
||||||
|
out = append(out, 0x7d, 0x01)
|
||||||
|
default:
|
||||||
|
out = append(out, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
189
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
189
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
package yutongmqtt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidJSON = errors.New("yutong mqtt invalid json")
|
||||||
|
|
||||||
|
func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS int64) (envelope.FrameEnvelope, error) {
|
||||||
|
root := map[string]any{}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||||
|
decoder.UseNumber()
|
||||||
|
if err := decoder.Decode(&root); err != nil {
|
||||||
|
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %v", ErrInvalidJSON, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data := objectValue(root, "data")
|
||||||
|
if data == nil {
|
||||||
|
data = root
|
||||||
|
}
|
||||||
|
deviceID := firstText(root, "device", "deviceId", "terminalId", "terminalID", "imei", "IMEI", "mqttDeviceId")
|
||||||
|
externalVIN := firstText(root, "vin", "VIN", "externalVin", "vehicleVin")
|
||||||
|
vin := externalVIN
|
||||||
|
if vin == "" && looksLikeVIN(deviceID) {
|
||||||
|
vin = deviceID
|
||||||
|
}
|
||||||
|
plate := firstText(root, "plateNo", "carNo", "plate", "vehicleNo")
|
||||||
|
phone := firstText(root, "sim", "phone", "mobile")
|
||||||
|
eventTimeMS := parseTimeMS(firstText(root, "time", "deviceTime", "eventTime", "gpsTime"), receivedAtMS)
|
||||||
|
fields := fieldsFromData(data)
|
||||||
|
|
||||||
|
parsed := map[string]any{
|
||||||
|
"endpoint": endpoint,
|
||||||
|
"topic": topic,
|
||||||
|
"root": root,
|
||||||
|
"data": data,
|
||||||
|
}
|
||||||
|
env := envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolYutongMQTT,
|
||||||
|
MessageID: "MQTT",
|
||||||
|
VIN: vin,
|
||||||
|
VehicleKeyHint: deviceID,
|
||||||
|
Phone: phone,
|
||||||
|
DeviceID: deviceID,
|
||||||
|
Plate: plate,
|
||||||
|
SourceEndpoint: "mqtt://" + endpoint + topic,
|
||||||
|
EventTimeMS: eventTimeMS,
|
||||||
|
ReceivedAtMS: receivedAtMS,
|
||||||
|
RawText: string(payload),
|
||||||
|
Parsed: parsed,
|
||||||
|
Fields: fields,
|
||||||
|
ParseStatus: envelope.ParseOK,
|
||||||
|
}
|
||||||
|
env.EventID = env.StableEventID()
|
||||||
|
return env, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fieldsFromData(data map[string]any) map[string]any {
|
||||||
|
fields := map[string]any{}
|
||||||
|
if speed, ok := firstFloat(data, "METER_SPEED", "speed", "speed_kmh"); ok {
|
||||||
|
fields[envelope.FieldSpeedKMH] = speed
|
||||||
|
}
|
||||||
|
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
|
||||||
|
fields[envelope.FieldTotalMileageKM] = mileage
|
||||||
|
}
|
||||||
|
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
|
||||||
|
fields[envelope.FieldSOCPercent] = soc
|
||||||
|
}
|
||||||
|
if longitude, ok := firstCoord(data, "LONGITUDE", "longitude", "lng"); ok {
|
||||||
|
fields[envelope.FieldLongitude] = longitude
|
||||||
|
}
|
||||||
|
if latitude, ok := firstCoord(data, "LATITUDE", "latitude", "lat"); ok {
|
||||||
|
fields[envelope.FieldLatitude] = latitude
|
||||||
|
}
|
||||||
|
if direction, ok := firstFloat(data, "GPSDirection", "direction", "direction_deg"); ok {
|
||||||
|
fields["direction_deg"] = direction
|
||||||
|
}
|
||||||
|
if gear, ok := firstFloat(data, "ON_GEAR", "gear"); ok {
|
||||||
|
fields["gear"] = gear
|
||||||
|
}
|
||||||
|
return fields
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectValue(root map[string]any, key string) map[string]any {
|
||||||
|
value, ok := root[key]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
typed, ok := value.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return typed
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstText(root map[string]any, keys ...string) string {
|
||||||
|
for _, key := range keys {
|
||||||
|
if value, ok := root[key]; ok && value != nil {
|
||||||
|
text := strings.TrimSpace(fmt.Sprint(value))
|
||||||
|
if text != "" && text != "<nil>" {
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstFloat(root map[string]any, keys ...string) (float64, bool) {
|
||||||
|
for _, key := range keys {
|
||||||
|
value, ok := root[key]
|
||||||
|
if !ok || value == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if parsed, ok := toFloat(value); ok {
|
||||||
|
return parsed, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstCoord(root map[string]any, keys ...string) (float64, bool) {
|
||||||
|
value, ok := firstFloat(root, keys...)
|
||||||
|
if !ok {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
if value == 0 || int64(value) == 999999 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
if value > 1000 || value < -1000 {
|
||||||
|
value = value / 1_000_000
|
||||||
|
}
|
||||||
|
return value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func toFloat(value any) (float64, bool) {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case json.Number:
|
||||||
|
parsed, err := typed.Float64()
|
||||||
|
return parsed, err == nil
|
||||||
|
case float64:
|
||||||
|
return typed, true
|
||||||
|
case float32:
|
||||||
|
return float64(typed), true
|
||||||
|
case int:
|
||||||
|
return float64(typed), true
|
||||||
|
case int64:
|
||||||
|
return float64(typed), true
|
||||||
|
case string:
|
||||||
|
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||||
|
return parsed, err == nil
|
||||||
|
default:
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseTimeMS(raw string, fallback int64) int64 {
|
||||||
|
raw = strings.TrimSpace(raw)
|
||||||
|
if raw == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
if len(raw) == 14 {
|
||||||
|
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
|
||||||
|
return parsed.UnixMilli()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if parsed, err := time.Parse(time.RFC3339, raw); err == nil {
|
||||||
|
return parsed.UnixMilli()
|
||||||
|
}
|
||||||
|
if epoch, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
||||||
|
if epoch > 10_000_000_000 {
|
||||||
|
return epoch
|
||||||
|
}
|
||||||
|
return epoch * 1000
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
func looksLikeVIN(value string) bool {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
return len(value) == 17
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package yutongmqtt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
|
||||||
|
payload := []byte(`{
|
||||||
|
"device": "LTEST000000000001",
|
||||||
|
"time": "20260413100000",
|
||||||
|
"plateNo": "豫A12345",
|
||||||
|
"data": {
|
||||||
|
"METER_SPEED": 52.3,
|
||||||
|
"TOTAL_MILEAGE": 123456.7,
|
||||||
|
"BATTERY_CAPACITY_SOC": 70,
|
||||||
|
"LONGITUDE": 116397128,
|
||||||
|
"LATITUDE": 39916527,
|
||||||
|
"GPSDirection": 88
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
|
||||||
|
env, err := ParseMessage("endpoint-a", "/ytforward/shln/dev1", payload, 1782745114999)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseMessage() error = %v", err)
|
||||||
|
}
|
||||||
|
if env.Protocol != envelope.ProtocolYutongMQTT {
|
||||||
|
t.Fatalf("protocol = %q", env.Protocol)
|
||||||
|
}
|
||||||
|
if env.VIN != "LTEST000000000001" || env.DeviceID != "LTEST000000000001" {
|
||||||
|
t.Fatalf("identity fields = vin:%q device:%q", env.VIN, env.DeviceID)
|
||||||
|
}
|
||||||
|
if env.Plate != "豫A12345" {
|
||||||
|
t.Fatalf("plate = %q", env.Plate)
|
||||||
|
}
|
||||||
|
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
|
||||||
|
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123456.7)
|
||||||
|
assertFloatField(t, env, envelope.FieldSOCPercent, 70)
|
||||||
|
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
|
||||||
|
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
|
||||||
|
assertFloatField(t, env, "direction_deg", 88)
|
||||||
|
if env.EventTimeMS == 1782745114999 {
|
||||||
|
t.Fatal("event time should come from device time")
|
||||||
|
}
|
||||||
|
if env.RawText == "" || env.Parsed["data"] == nil {
|
||||||
|
t.Fatalf("raw/parsed missing: %#v", env)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
|
||||||
|
payload := []byte(`{"device":"YT-DEVICE-001","time":"20260413100000","data":{"METER_SPEED":"12.3"}}`)
|
||||||
|
env, err := ParseMessage("endpoint-a", "/ytforward/shln/YT-DEVICE-001", payload, 1782745114999)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseMessage() error = %v", err)
|
||||||
|
}
|
||||||
|
if env.VIN != "" {
|
||||||
|
t.Fatalf("non-vin device should not be treated as vin: %q", env.VIN)
|
||||||
|
}
|
||||||
|
if env.VehicleKey() != "YT-DEVICE-001" {
|
||||||
|
t.Fatalf("vehicle key = %q", env.VehicleKey())
|
||||||
|
}
|
||||||
|
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseMessageRejectsMalformedJSON(t *testing.T) {
|
||||||
|
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
|
||||||
|
if !errors.Is(err, ErrInvalidJSON) {
|
||||||
|
t.Fatalf("error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||||
|
t.Helper()
|
||||||
|
got, ok := env.Fields[key].(float64)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
package realtime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Handler struct {
|
||||||
|
repository *Repository
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewHandler(repository *Repository) *Handler {
|
||||||
|
if repository == nil {
|
||||||
|
panic("realtime repository must not be nil")
|
||||||
|
}
|
||||||
|
return &Handler{repository: repository}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
path := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/realtime/vehicles/"), "/")
|
||||||
|
parts := strings.Split(path, "/")
|
||||||
|
if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" {
|
||||||
|
writeError(w, http.StatusNotFound, "vehicle not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
vin := parts[0]
|
||||||
|
switch {
|
||||||
|
case len(parts) == 1:
|
||||||
|
snapshot, err := h.repository.GetMerged(r.Context(), vin)
|
||||||
|
writeResult(w, snapshot, err)
|
||||||
|
case len(parts) == 2 && parts[1] == "online":
|
||||||
|
status, err := h.repository.IsOnline(r.Context(), vin)
|
||||||
|
writeResult(w, status, err)
|
||||||
|
case len(parts) == 3 && parts[1] == "protocols":
|
||||||
|
snapshot, err := h.repository.GetProtocol(r.Context(), vin, envelope.Protocol(strings.ToUpper(parts[2])))
|
||||||
|
writeResult(w, snapshot, err)
|
||||||
|
default:
|
||||||
|
writeError(w, http.StatusNotFound, "route not found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeResult(w http.ResponseWriter, value any, err error) {
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, redis.Nil) {
|
||||||
|
writeError(w, http.StatusNotFound, "not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeError(w http.ResponseWriter, status int, message string) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||||
|
}
|
||||||
38
go/vehicle-gateway/internal/realtime/model.go
Normal file
38
go/vehicle-gateway/internal/realtime/model.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package realtime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Snapshot struct {
|
||||||
|
VIN string `json:"vin"`
|
||||||
|
Protocol envelope.Protocol `json:"protocol,omitempty"`
|
||||||
|
EventID string `json:"event_id,omitempty"`
|
||||||
|
EventTimeMS int64 `json:"event_time_ms"`
|
||||||
|
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||||
|
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||||
|
Fields map[string]any `json:"fields,omitempty"`
|
||||||
|
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
|
||||||
|
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type OnlineStatus struct {
|
||||||
|
VIN string `json:"vin"`
|
||||||
|
Online bool `json:"online"`
|
||||||
|
LastSeenMS int64 `json:"last_seen_ms"`
|
||||||
|
Protocols []envelope.Protocol `json:"protocols,omitempty"`
|
||||||
|
TTLSeconds int64 `json:"ttl_seconds"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
OnlineTTL time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c Config) ttl() time.Duration {
|
||||||
|
if c.OnlineTTL <= 0 {
|
||||||
|
return 10 * time.Minute
|
||||||
|
}
|
||||||
|
return c.OnlineTTL
|
||||||
|
}
|
||||||
198
go/vehicle-gateway/internal/realtime/repository.go
Normal file
198
go/vehicle-gateway/internal/realtime/repository.go
Normal file
@@ -0,0 +1,198 @@
|
|||||||
|
package realtime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Repository struct {
|
||||||
|
client *redis.Client
|
||||||
|
cfg Config
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRepository(client *redis.Client, cfg Config) *Repository {
|
||||||
|
if client == nil {
|
||||||
|
panic("redis client must not be nil")
|
||||||
|
}
|
||||||
|
return &Repository{client: client, cfg: cfg}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
vin := strings.TrimSpace(env.VIN)
|
||||||
|
if vin == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
nowMS := time.Now().UnixMilli()
|
||||||
|
eventMS := env.EventTimeMS
|
||||||
|
if eventMS <= 0 {
|
||||||
|
eventMS = env.ReceivedAtMS
|
||||||
|
}
|
||||||
|
protocolSnapshot := Snapshot{
|
||||||
|
VIN: vin,
|
||||||
|
Protocol: env.Protocol,
|
||||||
|
EventID: env.StableEventID(),
|
||||||
|
EventTimeMS: eventMS,
|
||||||
|
ReceivedAtMS: env.ReceivedAtMS,
|
||||||
|
SourceEndpoint: env.SourceEndpoint,
|
||||||
|
Fields: cloneFields(env.Fields),
|
||||||
|
FieldTimesMS: fieldTimes(env.Fields, eventMS),
|
||||||
|
UpdatedAtMS: nowMS,
|
||||||
|
}
|
||||||
|
if err := r.setJSON(ctx, protocolKey(vin, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
merged, err := r.GetMerged(ctx, vin)
|
||||||
|
if err != nil && !errors.Is(err, redis.Nil) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if merged.VIN == "" {
|
||||||
|
merged = Snapshot{VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
|
||||||
|
}
|
||||||
|
mergeFields(&merged, env.Fields, eventMS)
|
||||||
|
if eventMS >= merged.EventTimeMS {
|
||||||
|
merged.EventTimeMS = eventMS
|
||||||
|
merged.EventID = env.StableEventID()
|
||||||
|
merged.ReceivedAtMS = env.ReceivedAtMS
|
||||||
|
merged.SourceEndpoint = env.SourceEndpoint
|
||||||
|
}
|
||||||
|
merged.UpdatedAtMS = nowMS
|
||||||
|
if err := r.setJSON(ctx, mergedKey(vin), merged, r.cfg.ttl()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
protocols, err := r.addProtocol(ctx, vin, env.Protocol)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
online := OnlineStatus{
|
||||||
|
VIN: vin,
|
||||||
|
Online: true,
|
||||||
|
LastSeenMS: env.ReceivedAtMS,
|
||||||
|
Protocols: protocols,
|
||||||
|
TTLSeconds: int64(r.cfg.ttl().Seconds()),
|
||||||
|
}
|
||||||
|
if err := r.setJSON(ctx, onlineKey(vin), online, r.cfg.ttl()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vin}).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) GetMerged(ctx context.Context, vin string) (Snapshot, error) {
|
||||||
|
return r.getSnapshot(ctx, mergedKey(vin))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) GetProtocol(ctx context.Context, vin string, protocol envelope.Protocol) (Snapshot, error) {
|
||||||
|
return r.getSnapshot(ctx, protocolKey(vin, protocol))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) IsOnline(ctx context.Context, vin string) (OnlineStatus, error) {
|
||||||
|
var status OnlineStatus
|
||||||
|
payload, err := r.client.Get(ctx, onlineKey(vin)).Bytes()
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, redis.Nil) {
|
||||||
|
return OnlineStatus{VIN: vin, Online: false}, nil
|
||||||
|
}
|
||||||
|
return status, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(payload, &status); err != nil {
|
||||||
|
return status, err
|
||||||
|
}
|
||||||
|
ttl := r.client.TTL(ctx, onlineKey(vin)).Val()
|
||||||
|
status.Online = ttl > 0
|
||||||
|
status.TTLSeconds = int64(ttl.Seconds())
|
||||||
|
return status, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) getSnapshot(ctx context.Context, key string) (Snapshot, error) {
|
||||||
|
var snapshot Snapshot
|
||||||
|
payload, err := r.client.Get(ctx, key).Bytes()
|
||||||
|
if err != nil {
|
||||||
|
return snapshot, err
|
||||||
|
}
|
||||||
|
return snapshot, json.Unmarshal(payload, &snapshot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl time.Duration) error {
|
||||||
|
payload, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return r.client.Set(ctx, key, payload, ttl).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) addProtocol(ctx context.Context, vin string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
|
||||||
|
key := protocolsKey(vin)
|
||||||
|
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = r.client.Expire(ctx, key, r.cfg.ttl()).Err()
|
||||||
|
values, err := r.client.SMembers(ctx, key).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(values)
|
||||||
|
out := make([]envelope.Protocol, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
out = append(out, envelope.Protocol(value))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeFields(snapshot *Snapshot, fields map[string]any, eventMS int64) {
|
||||||
|
if snapshot.Fields == nil {
|
||||||
|
snapshot.Fields = map[string]any{}
|
||||||
|
}
|
||||||
|
if snapshot.FieldTimesMS == nil {
|
||||||
|
snapshot.FieldTimesMS = map[string]int64{}
|
||||||
|
}
|
||||||
|
for key, value := range fields {
|
||||||
|
if eventMS >= snapshot.FieldTimesMS[key] {
|
||||||
|
snapshot.Fields[key] = value
|
||||||
|
snapshot.FieldTimesMS[key] = eventMS
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneFields(fields map[string]any) map[string]any {
|
||||||
|
if len(fields) == 0 {
|
||||||
|
return map[string]any{}
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(fields))
|
||||||
|
for key, value := range fields {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
|
||||||
|
out := make(map[string]int64, len(fields))
|
||||||
|
for key := range fields {
|
||||||
|
out[key] = eventMS
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergedKey(vin string) string {
|
||||||
|
return "vehicle:latest:" + strings.TrimSpace(vin)
|
||||||
|
}
|
||||||
|
|
||||||
|
func protocolKey(vin string, protocol envelope.Protocol) string {
|
||||||
|
return "vehicle:latest:" + strings.TrimSpace(vin) + ":" + string(protocol)
|
||||||
|
}
|
||||||
|
|
||||||
|
func onlineKey(vin string) string {
|
||||||
|
return "vehicle:online:" + strings.TrimSpace(vin)
|
||||||
|
}
|
||||||
|
|
||||||
|
func protocolsKey(vin string) string {
|
||||||
|
return "vehicle:protocols:" + strings.TrimSpace(vin)
|
||||||
|
}
|
||||||
136
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
136
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
@@ -0,0 +1,136 @@
|
|||||||
|
package realtime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/alicebob/miniredis/v2"
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
|
||||||
|
repo, closeFn := newTestRepository(t)
|
||||||
|
defer closeFn()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
VIN: "VIN001",
|
||||||
|
EventTimeMS: 1000,
|
||||||
|
ReceivedAtMS: 1100,
|
||||||
|
Fields: map[string]any{
|
||||||
|
envelope.FieldLongitude: 121.1,
|
||||||
|
envelope.FieldTotalMileageKM: 10.5,
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Update() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolGB32960,
|
||||||
|
VIN: "VIN001",
|
||||||
|
EventTimeMS: 900,
|
||||||
|
ReceivedAtMS: 1200,
|
||||||
|
Fields: map[string]any{
|
||||||
|
envelope.FieldLongitude: 120.0,
|
||||||
|
envelope.FieldLatitude: 30.5,
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Update() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
merged, err := repo.GetMerged(ctx, "VIN001")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetMerged() error = %v", err)
|
||||||
|
}
|
||||||
|
if merged.Fields[envelope.FieldLongitude] != 121.1 {
|
||||||
|
t.Fatalf("older longitude overwrote newer value: %#v", merged.Fields)
|
||||||
|
}
|
||||||
|
if merged.Fields[envelope.FieldLatitude] != 30.5 {
|
||||||
|
t.Fatalf("new latitude missing: %#v", merged.Fields)
|
||||||
|
}
|
||||||
|
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetProtocol() error = %v", err)
|
||||||
|
}
|
||||||
|
if protocol.Fields[envelope.FieldTotalMileageKM] != 10.5 {
|
||||||
|
t.Fatalf("protocol snapshot = %#v", protocol)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRepositoryOnlineStatus(t *testing.T) {
|
||||||
|
repo, closeFn := newTestRepository(t)
|
||||||
|
defer closeFn()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
status, err := repo.IsOnline(ctx, "VIN001")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("IsOnline() error = %v", err)
|
||||||
|
}
|
||||||
|
if status.Online {
|
||||||
|
t.Fatal("empty vin should be offline")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
VIN: "VIN001",
|
||||||
|
EventTimeMS: 1000,
|
||||||
|
ReceivedAtMS: 1100,
|
||||||
|
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Update() error = %v", err)
|
||||||
|
}
|
||||||
|
status, err = repo.IsOnline(ctx, "VIN001")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("IsOnline() error = %v", err)
|
||||||
|
}
|
||||||
|
if !status.Online || status.LastSeenMS != 1100 {
|
||||||
|
t.Fatalf("online status = %#v", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerReturnsMergedSnapshot(t *testing.T) {
|
||||||
|
repo, closeFn := newTestRepository(t)
|
||||||
|
defer closeFn()
|
||||||
|
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
VIN: "VIN001",
|
||||||
|
EventTimeMS: 1000,
|
||||||
|
ReceivedAtMS: 1100,
|
||||||
|
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Update() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/VIN001", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
NewHandler(repo).ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if !stringsContains(rec.Body.String(), `"vin":"VIN001"`) {
|
||||||
|
t.Fatalf("unexpected body: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestRepository(t *testing.T) (*Repository, func()) {
|
||||||
|
t.Helper()
|
||||||
|
server, err := miniredis.Run()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("miniredis.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||||
|
return NewRepository(client, Config{OnlineTTL: time.Minute}), func() {
|
||||||
|
_ = client.Close()
|
||||||
|
server.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func stringsContains(value string, pattern string) bool {
|
||||||
|
return strings.Contains(value, pattern)
|
||||||
|
}
|
||||||
159
go/vehicle-gateway/internal/stats/daily_metric.go
Normal file
159
go/vehicle-gateway/internal/stats/daily_metric.go
Normal file
@@ -0,0 +1,159 @@
|
|||||||
|
package stats
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
MetricDailyMileageKM = "daily_mileage_km"
|
||||||
|
MetricDailyTotalMileageKM = "daily_total_mileage_km"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Execer interface {
|
||||||
|
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type Writer struct {
|
||||||
|
exec Execer
|
||||||
|
loc *time.Location
|
||||||
|
}
|
||||||
|
|
||||||
|
type MetricSample struct {
|
||||||
|
VIN string
|
||||||
|
Protocol envelope.Protocol
|
||||||
|
StatDate string
|
||||||
|
MetricKey string
|
||||||
|
MetricValue float64
|
||||||
|
TotalMileageKM float64
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWriter(exec Execer, loc *time.Location) *Writer {
|
||||||
|
if exec == nil {
|
||||||
|
panic("stats execer must not be nil")
|
||||||
|
}
|
||||||
|
if loc == nil {
|
||||||
|
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||||
|
}
|
||||||
|
return &Writer{exec: exec, loc: loc}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) EnsureSchema(ctx context.Context) error {
|
||||||
|
_, err := w.exec.ExecContext(ctx, DailyMetricTableSQL)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
samples, err := SamplesFromEnvelope(env, w.loc)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, sample := range samples {
|
||||||
|
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL,
|
||||||
|
sample.VIN,
|
||||||
|
sample.StatDate,
|
||||||
|
string(sample.Protocol),
|
||||||
|
sample.MetricKey,
|
||||||
|
sample.MetricValue,
|
||||||
|
sample.TotalMileageKM,
|
||||||
|
sample.TotalMileageKM); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
|
||||||
|
vin := strings.TrimSpace(env.VIN)
|
||||||
|
if vin == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||||
|
if !ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if loc == nil {
|
||||||
|
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||||
|
}
|
||||||
|
eventMS := env.EventTimeMS
|
||||||
|
if eventMS <= 0 {
|
||||||
|
eventMS = env.ReceivedAtMS
|
||||||
|
}
|
||||||
|
if eventMS <= 0 {
|
||||||
|
return nil, errors.New("event or received time is required")
|
||||||
|
}
|
||||||
|
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
|
||||||
|
return []MetricSample{
|
||||||
|
{
|
||||||
|
VIN: vin,
|
||||||
|
Protocol: env.Protocol,
|
||||||
|
StatDate: statDate,
|
||||||
|
MetricKey: MetricDailyMileageKM,
|
||||||
|
MetricValue: 0,
|
||||||
|
TotalMileageKM: totalMileage,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
VIN: vin,
|
||||||
|
Protocol: env.Protocol,
|
||||||
|
StatDate: statDate,
|
||||||
|
MetricKey: MetricDailyTotalMileageKM,
|
||||||
|
MetricValue: totalMileage,
|
||||||
|
TotalMileageKM: totalMileage,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const upsertDailyMetricSQL = `
|
||||||
|
INSERT INTO vehicle_daily_metric
|
||||||
|
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||||
|
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
first_total_mileage_km = LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)),
|
||||||
|
latest_total_mileage_km = GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)),
|
||||||
|
metric_value = CASE
|
||||||
|
WHEN metric_key = 'daily_mileage_km'
|
||||||
|
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||||
|
- LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||||
|
WHEN metric_key = 'daily_total_mileage_km'
|
||||||
|
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||||
|
ELSE VALUES(metric_value)
|
||||||
|
END,
|
||||||
|
sample_count = sample_count + 1,
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
`
|
||||||
|
|
||||||
|
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||||
|
if env.Fields == nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
value, ok := env.Fields[key]
|
||||||
|
if !ok || value == nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case float64:
|
||||||
|
return typed, true
|
||||||
|
case float32:
|
||||||
|
return float64(typed), true
|
||||||
|
case int:
|
||||||
|
return float64(typed), true
|
||||||
|
case int64:
|
||||||
|
return float64(typed), true
|
||||||
|
case uint16:
|
||||||
|
return float64(typed), true
|
||||||
|
case uint32:
|
||||||
|
return float64(typed), true
|
||||||
|
case string:
|
||||||
|
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||||
|
return parsed, err == nil
|
||||||
|
default:
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
}
|
||||||
107
go/vehicle-gateway/internal/stats/daily_metric_test.go
Normal file
107
go/vehicle-gateway/internal/stats/daily_metric_test.go
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
package stats
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
|
||||||
|
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||||
|
env := envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
VIN: "LNBVIN00000000001",
|
||||||
|
EventTimeMS: time.Date(2026, 7, 1, 8, 30, 0, 0, loc).UnixMilli(),
|
||||||
|
Fields: map[string]any{
|
||||||
|
envelope.FieldTotalMileageKM: 10241.2,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
samples, err := SamplesFromEnvelope(env, loc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(samples) != 2 {
|
||||||
|
t.Fatalf("sample count = %d", len(samples))
|
||||||
|
}
|
||||||
|
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
|
||||||
|
t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
|
||||||
|
}
|
||||||
|
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 {
|
||||||
|
t.Fatalf("unexpected daily total sample: %#v", samples[1])
|
||||||
|
}
|
||||||
|
if samples[0].StatDate != "2026-07-01" {
|
||||||
|
t.Fatalf("stat date = %q", samples[0].StatDate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSamplesFromEnvelopeSkipsMissingVINOrMileage(t *testing.T) {
|
||||||
|
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(samples) != 0 {
|
||||||
|
t.Fatalf("expected no samples without vin, got %#v", samples)
|
||||||
|
}
|
||||||
|
|
||||||
|
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolGB32960,
|
||||||
|
VIN: "LNBVIN00000000002",
|
||||||
|
Fields: map[string]any{},
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(samples) != 0 {
|
||||||
|
t.Fatalf("expected no samples without mileage, got %#v", samples)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
|
||||||
|
exec := &recordingExec{}
|
||||||
|
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||||
|
if err := writer.EnsureSchema(context.Background()); err != nil {
|
||||||
|
t.Fatalf("EnsureSchema() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := writer.Append(context.Background(), envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolGB32960,
|
||||||
|
VIN: "LNBVIN00000000002",
|
||||||
|
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||||
|
Fields: map[string]any{
|
||||||
|
envelope.FieldTotalMileageKM: "10000.0",
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Append() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") {
|
||||||
|
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
|
||||||
|
}
|
||||||
|
if len(exec.calls) != 3 {
|
||||||
|
t.Fatalf("exec calls = %d", len(exec.calls))
|
||||||
|
}
|
||||||
|
if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") {
|
||||||
|
t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type execCall struct {
|
||||||
|
query string
|
||||||
|
args []any
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingExec struct {
|
||||||
|
calls []execCall
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||||
|
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
20
go/vehicle-gateway/internal/stats/schema.go
Normal file
20
go/vehicle-gateway/internal/stats/schema.go
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
package stats
|
||||||
|
|
||||||
|
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
vin VARCHAR(32) NOT NULL,
|
||||||
|
stat_date DATE NOT NULL,
|
||||||
|
protocol VARCHAR(32) NOT NULL,
|
||||||
|
metric_key VARCHAR(64) NOT NULL,
|
||||||
|
metric_value DECIMAL(18,3) NOT NULL,
|
||||||
|
metric_unit VARCHAR(16) NOT NULL,
|
||||||
|
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||||
|
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||||
|
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
calculation_method VARCHAR(64) NOT NULL,
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
|
||||||
|
KEY idx_stat_date (stat_date),
|
||||||
|
KEY idx_protocol_metric (protocol, metric_key)
|
||||||
|
)`
|
||||||
Reference in New Issue
Block a user