Compare commits
78 Commits
cde55e1ae6
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbb6f3b741 | ||
|
|
42de422935 | ||
|
|
318486b1e9 | ||
|
|
0a024f9a76 | ||
|
|
ea9d9f5d35 | ||
|
|
409f55b5ae | ||
|
|
8def635bf5 | ||
|
|
c211447747 | ||
|
|
1741aacaaa | ||
|
|
27f4ad3468 | ||
|
|
601150b364 | ||
|
|
5b7f386c5a | ||
|
|
f4d9c9ef5a | ||
|
|
96514898eb | ||
|
|
0ef88e1a4c | ||
|
|
0fd4d319f9 | ||
|
|
4e8434c511 | ||
|
|
576644eb65 | ||
|
|
9bd48c5781 | ||
|
|
2214abe9c5 | ||
|
|
1477b997c9 | ||
|
|
4ea8d18972 | ||
|
|
947e645090 | ||
|
|
78b805e3a2 | ||
|
|
a7b24f4c0b | ||
|
|
eb124d929b | ||
|
|
4711287655 | ||
|
|
2c224396c9 | ||
|
|
4f2cded760 | ||
|
|
7ad1a25e8e | ||
|
|
3ee1d6f52b | ||
|
|
6d1d0aa85e | ||
|
|
055373c405 | ||
|
|
e37dd39c89 | ||
|
|
dd73fdbb5b | ||
|
|
fb9ab8525a | ||
|
|
cc89e0d537 | ||
|
|
f3ecf430cc | ||
|
|
bcf7cdae2e | ||
|
|
9045b871d6 | ||
|
|
b0150a5503 | ||
|
|
8f3b337fd6 | ||
|
|
594ab2d9c7 | ||
|
|
229ffcf61f | ||
|
|
593713283c | ||
|
|
6bee0499ae | ||
|
|
59c1c9df0f | ||
|
|
56f4811c68 | ||
|
|
1a53011408 | ||
|
|
c2d70dad86 | ||
|
|
91db6c959e | ||
|
|
7b44f97ddb | ||
|
|
a72977cd79 | ||
|
|
dd95101499 | ||
|
|
d347525a75 | ||
|
|
19008e840c | ||
|
|
c803a19bf5 | ||
|
|
243de5c2b2 | ||
|
|
4a32e2181a | ||
|
|
7e7066dd86 | ||
|
|
e7c8c57296 | ||
|
|
6b55694c43 | ||
|
|
eb48d16181 | ||
|
|
a15de91912 | ||
|
|
796d79dd30 | ||
|
|
5f4f4fd124 | ||
|
|
e0068a750d | ||
|
|
665122b23b | ||
|
|
60ca42e5d5 | ||
|
|
80474eeccb | ||
|
|
17024651c8 | ||
|
|
d8c77d7882 | ||
|
|
b55b075a97 | ||
|
|
bac8864e79 | ||
|
|
5d844701ff | ||
|
|
967981c040 | ||
|
|
1fdbc29e27 | ||
|
|
d148a4e3af |
117
deploy/portainer/docker-compose-go.yml
Normal file
117
deploy/portainer/docker-compose-go.yml
Normal file
@@ -0,0 +1,117 @@
|
||||
version: "3.8"
|
||||
|
||||
# Go sidecar stack for the vehicle ingest redesign.
|
||||
# It is intentionally separate from deploy/portainer/docker-compose.yml so the
|
||||
# current Java production stack can keep running until Go evidence is captured.
|
||||
|
||||
x-common-env: &common-env
|
||||
TZ: Asia/Shanghai
|
||||
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.yutong-mqtt.v1}
|
||||
KAFKA_TOPIC_UNIFIED: ${KAFKA_TOPIC_UNIFIED:-vehicle.event.unified.v1}
|
||||
|
||||
x-restart: &restart-policy
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "100m"
|
||||
max-file: "5"
|
||||
|
||||
services:
|
||||
go-vehicle-gateway:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-vehicle-gateway
|
||||
command: ["/app/gateway"]
|
||||
mem_limit: ${GO_GATEWAY_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
GB32960_TCP_ADDR: ":32960"
|
||||
JT808_TCP_ADDR: ":808"
|
||||
TCP_READ_BUFFER_BYTES: ${TCP_READ_BUFFER_BYTES:-65536}
|
||||
TCP_IDLE_TIMEOUT_SECONDS: ${TCP_IDLE_TIMEOUT_SECONDS:-180}
|
||||
TCP_MAX_CONNECTIONS: ${TCP_MAX_CONNECTIONS:-20000}
|
||||
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
|
||||
YUTONG_MQTT_ENDPOINT: ${YUTONG_MQTT_ENDPOINT:-yutong}
|
||||
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
|
||||
YUTONG_MQTT_TOPICS: ${YUTONG_MQTT_TOPICS:-/ytforward/shln/+}
|
||||
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-2}
|
||||
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt}
|
||||
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
|
||||
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
|
||||
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
|
||||
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
|
||||
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
|
||||
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
|
||||
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
|
||||
IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-}
|
||||
VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding}
|
||||
ports:
|
||||
- "${GO_GB32960_TCP_PORT:-32960}:32960"
|
||||
- "${GO_JT808_TCP_PORT:-808}:808"
|
||||
volumes:
|
||||
- "${KAFKA_SPOOL_HOST_DIR:-/opt/lingniu-go/spool/gateway}:${KAFKA_SPOOL_DIR:-/data/spool/gateway}"
|
||||
- "${YUTONG_MQTT_CERT_HOST_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:${YUTONG_MQTT_CERT_CONTAINER_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:ro"
|
||||
|
||||
go-history-writer:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-history-writer
|
||||
command: ["/app/history-writer"]
|
||||
mem_limit: ${GO_HISTORY_WRITER_MEM_LIMIT:-768m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_HISTORY:-go-history-writer}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_HISTORY:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1}
|
||||
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
|
||||
TDENGINE_DSN: ${TDENGINE_DSN:?set TDENGINE_DSN, example root:password@ws(172.17.111.57:6041)/lingniu_vehicle_ts}
|
||||
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
|
||||
TDENGINE_ENSURE_SCHEMA: ${TDENGINE_ENSURE_SCHEMA:-true}
|
||||
|
||||
go-stat-writer:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-stat-writer
|
||||
command: ["/app/stat-writer"]
|
||||
mem_limit: ${GO_STAT_WRITER_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_STAT:-go-stat-writer}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_STAT:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1}
|
||||
MYSQL_DSN: ${MYSQL_DSN:?set MYSQL_DSN}
|
||||
MYSQL_ENSURE_SCHEMA: ${MYSQL_ENSURE_SCHEMA:-true}
|
||||
LOCAL_TZ: ${LOCAL_TZ:-Asia/Shanghai}
|
||||
|
||||
go-realtime-api:
|
||||
<<: *restart-policy
|
||||
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
|
||||
container_name: go-realtime-api
|
||||
command: ["/app/realtime-api"]
|
||||
mem_limit: ${GO_REALTIME_API_MEM_LIMIT:-512m}
|
||||
environment:
|
||||
<<: *common-env
|
||||
HTTP_ADDR: ":20210"
|
||||
KAFKA_GROUP: ${KAFKA_GROUP_GO_REALTIME:-go-realtime-api}
|
||||
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_REALTIME:-vehicle.event.unified.v1}
|
||||
REDIS_ADDR: ${REDIS_ADDR:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379}
|
||||
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
|
||||
REDIS_DB: ${REDIS_DB:-50}
|
||||
ONLINE_TTL_SECONDS: ${ONLINE_TTL_SECONDS:-600}
|
||||
MYSQL_DSN: ${MYSQL_DSN:-}
|
||||
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
|
||||
TDENGINE_DSN: ${TDENGINE_DSN:-}
|
||||
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
|
||||
ports:
|
||||
- "${GO_REALTIME_HTTP_PORT:-20210}:20210"
|
||||
|
||||
networks:
|
||||
vehicle-ingest:
|
||||
name: vehicle-ingest
|
||||
66
deploy/systemd/README.md
Normal file
66
deploy/systemd/README.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Go Vehicle Gateway Native Deployment
|
||||
|
||||
本目录用于 ECS 原生部署。当前 goal 后续不再使用 Docker/Portainer 作为 Go 接入链路的部署方式。
|
||||
|
||||
## Runtime Layout
|
||||
|
||||
```text
|
||||
/opt/lingniu-go-native/
|
||||
current -> /opt/lingniu-go-native/releases/<git-short-sha>
|
||||
releases/<git-short-sha>/
|
||||
gateway
|
||||
history-writer
|
||||
stat-writer
|
||||
realtime-api
|
||||
env/
|
||||
gateway.env
|
||||
history-writer.env
|
||||
stat-writer.env
|
||||
realtime-api.env
|
||||
spool/gateway/
|
||||
```
|
||||
|
||||
## Services
|
||||
|
||||
| systemd unit | Binary | Purpose |
|
||||
|---|---|---|
|
||||
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP `32960`、JT808 TCP `808`、宇通 MQTT 接入,写 Kafka RAW/unified |
|
||||
| `lingniu-go-history-writer.service` | `history-writer` | 消费 RAW topic,写 TDengine `raw_frames` 和核心时序表 |
|
||||
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 RAW topic,写 MySQL `vehicle_daily_metric` |
|
||||
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费 unified topic,写 Redis,并提供 realtime/raw/stat 查询 API |
|
||||
|
||||
## Build
|
||||
|
||||
在开发机或 CI 上构建 Linux amd64 二进制:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/gateway ./cmd/gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/history-writer ./cmd/history-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/stat-writer ./cmd/stat-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/realtime-api ./cmd/realtime-api
|
||||
```
|
||||
|
||||
## Cutover Notes
|
||||
|
||||
1. 先上传二进制到新 release 目录并更新 `current` symlink。
|
||||
2. 从现有生产 env 生成四个 systemd 专用 env 文件;不要把密钥写入仓库。
|
||||
3. 停止旧 Docker Go 容器或 Java 容器,释放 `808`、`32960`、`20210`。
|
||||
4. 执行:
|
||||
|
||||
```bash
|
||||
systemctl daemon-reload
|
||||
systemctl enable lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=JT808&limit=1'
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=GB32960&limit=1'
|
||||
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=YUTONG_MQTT&limit=1'
|
||||
```
|
||||
18
deploy/systemd/lingniu-go-gateway.service
Normal file
18
deploy/systemd/lingniu-go-gateway.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Vehicle Gateway
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/gateway.env
|
||||
ExecStart=/opt/lingniu-go-native/current/gateway
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-history-writer.service
Normal file
18
deploy/systemd/lingniu-go-history-writer.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go History Writer
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/history-writer.env
|
||||
ExecStart=/opt/lingniu-go-native/current/history-writer
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-realtime-api.service
Normal file
18
deploy/systemd/lingniu-go-realtime-api.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Realtime API
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/realtime-api.env
|
||||
ExecStart=/opt/lingniu-go-native/current/realtime-api
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/lingniu-go-stat-writer.service
Normal file
18
deploy/systemd/lingniu-go-stat-writer.service
Normal file
@@ -0,0 +1,18 @@
|
||||
[Unit]
|
||||
Description=Lingniu Go Stat Writer
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/lingniu-go-native/current
|
||||
EnvironmentFile=/opt/lingniu-go-native/env/stat-writer.env
|
||||
ExecStart=/opt/lingniu-go-native/current/stat-writer
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=1048576
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
227
docs/operations/current-ecs-deployment.md
Normal file
227
docs/operations/current-ecs-deployment.md
Normal file
@@ -0,0 +1,227 @@
|
||||
# 当前 ECS Go 原生部署说明
|
||||
|
||||
更新时间:2026-07-02 01:30 CST
|
||||
|
||||
本文档记录当前 `lingniu-vehicle-ingest` 的生产运行面。当前 goal 的接入链路已经切到 Go 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。密钥只维护在 ECS 环境文件或受控凭据中,不写入 Git。
|
||||
|
||||
## 部署范围
|
||||
|
||||
| systemd 服务 | 二进制 | 说明 | 对外端口 |
|
||||
| --- | --- | --- | --- |
|
||||
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP、JT808 TCP、宇通 MQTT 接入,解析后写 Kafka RAW 和统一事件 | TCP `32960`、TCP `808` |
|
||||
| `lingniu-go-history-writer.service` | `history-writer` | 消费 Kafka RAW,写 TDengine RAW、位置点、里程点核心表 | 无 HTTP |
|
||||
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 Kafka RAW,按总里程差值法写 MySQL 每日指标 | 无 HTTP |
|
||||
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费统一事件写 Redis,并提供实时、历史、统计查询 API | HTTP `20210` |
|
||||
|
||||
信达 Push 已废弃,不参与当前 Go 生产链路。旧 Java/Docker 服务不应占用 `808`、`32960`、`20210`。
|
||||
|
||||
## 应用 ECS
|
||||
|
||||
| 项 | 值 |
|
||||
| --- | --- |
|
||||
| 公网 IP | `115.29.187.205` |
|
||||
| 登录用户 | `root` |
|
||||
| 部署目录 | `/opt/lingniu-go-native` |
|
||||
| 当前 release | `/opt/lingniu-go-native/current` |
|
||||
| 环境文件目录 | `/opt/lingniu-go-native/env` |
|
||||
| systemd unit 目录 | `/etc/systemd/system` |
|
||||
| 对外服务 | `32960`、`808`、`20210` |
|
||||
|
||||
运行目录结构:
|
||||
|
||||
```text
|
||||
/opt/lingniu-go-native/
|
||||
current -> /opt/lingniu-go-native/releases/<git-short-sha>
|
||||
releases/<git-short-sha>/
|
||||
gateway
|
||||
history-writer
|
||||
stat-writer
|
||||
realtime-api
|
||||
env/
|
||||
gateway.env
|
||||
history-writer.env
|
||||
stat-writer.env
|
||||
realtime-api.env
|
||||
spool/gateway/
|
||||
```
|
||||
|
||||
## 中间件
|
||||
|
||||
| 组件 | 内网地址 | 公网地址 | 用途 |
|
||||
| --- | --- | --- | --- |
|
||||
| Kafka | `172.17.111.56:9092` | `114.55.58.251:9092` | RAW 和统一事件消息总线 |
|
||||
| TDengine | `172.17.111.57:6041` | `115.29.185.82:6041` | RAW、位置、里程点时序热存储 |
|
||||
| MySQL RDS | `rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306` | `rm-bp179zbv481rnw3e2no.mysql.rds.aliyuncs.com:3306` | 身份映射、JT808 注册、每日指标 |
|
||||
| Redis RDS | `r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379` | 无 | 准实时车辆状态缓存 |
|
||||
|
||||
生产应用之间访问中间件优先使用内网地址。RDS 白名单需要允许应用 ECS 私网访问。
|
||||
|
||||
## Kafka Topic
|
||||
|
||||
| Topic | 生产者 | 消费者 | 内容 |
|
||||
| --- | --- | --- | --- |
|
||||
| `vehicle.raw.go.gb32960.v1` | `gateway` | `history-writer`、`stat-writer` | GB32960 完整 RAW 记录 |
|
||||
| `vehicle.raw.go.jt808.v1` | `gateway` | `history-writer`、`stat-writer` | JT808 完整 RAW 记录 |
|
||||
| `vehicle.raw.go.yutong-mqtt.v1` | `gateway` | `history-writer` | 宇通 MQTT 完整 RAW 记录 |
|
||||
| `vehicle.event.go.unified.v1` | `gateway` | `realtime-api` | 三类协议统一事件,用于 Redis 实时状态 |
|
||||
|
||||
`gateway` 配置了 Kafka retry;生产 env 中启用 `KAFKA_SPOOL_DIR` 时,本地 spool 可在 Kafka 短暂不可用后回放。
|
||||
|
||||
Kafka 消费组:
|
||||
|
||||
| Consumer Group | Topic | 当前验证 |
|
||||
| --- | --- | --- |
|
||||
| `go-history-writer` | 三个 `vehicle.raw.go.*` topic | 2026-07-02 验证 lag 为 `0` |
|
||||
| `go-stat-writer` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1` | 2026-07-02 验证 lag 为 `0` |
|
||||
| `go-realtime-api` | `vehicle.event.go.unified.v1` | 2026-07-02 验证 lag 为 `0` |
|
||||
|
||||
可重复 smoke:
|
||||
|
||||
```bash
|
||||
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
|
||||
```
|
||||
|
||||
该脚本通过 SSH 登录 Kafka ECS,检查 Go 生产 topic 是否存在,并校验 `go-history-writer`、`go-stat-writer`、`go-realtime-api` 的消费组 lag。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
## TDengine 数据层
|
||||
|
||||
默认数据库:`lingniu_vehicle_ts`
|
||||
|
||||
| 表 | 类型 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `raw_frames` | stable | 完整 RAW 帧,包含 `raw_hex`、`parsed_json`、`fields_json`、`source_endpoint` 和协议/车辆 tags |
|
||||
| `vehicle_locations` | stable | 最小化位置历史,保留 `frame_id` 回链 RAW |
|
||||
| `vehicle_mileage_points` | stable | 最小化总里程点,供历史查询和统计复核 |
|
||||
|
||||
设计原则:
|
||||
|
||||
- 完整结构化 payload 只落在 `raw_frames.parsed_json` / `fields_json`。
|
||||
- `vehicle_locations` 和 `vehicle_mileage_points` 只保留核心查询字段,不重复保存完整 JSON。
|
||||
- `telemetry_fields` 不由当前服务维护,字段配置解析由独立子服务处理。
|
||||
- API 传入东八区时间时,服务会转换为 TDengine 当前存储使用的 UTC 字面量再查询。
|
||||
|
||||
## MySQL 数据层
|
||||
|
||||
默认业务库:`lingniu_vehicle_data`
|
||||
|
||||
| 表 | 维护方 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `vehicle_identity_binding` | 人工/外部主数据 | VIN 映射主表,用 `phone`、`device_id`、`plate` 降级定位 VIN |
|
||||
| `jt808_registration` | `gateway` 自动写入 | 仅 JT808 注册和鉴权记录,以 `phone` 作为主键,记录设备、车牌、厂家、鉴权、来源端点 |
|
||||
| `vehicle_daily_metric` | `stat-writer` 自动写入 | 每日指标表,保存 `daily_mileage_km` 和 `daily_total_mileage_km` |
|
||||
|
||||
每日里程算法:
|
||||
|
||||
```text
|
||||
daily_mileage_km = 当日最大 total_mileage_km - 当日最小 total_mileage_km
|
||||
daily_total_mileage_km = 当日最大 total_mileage_km
|
||||
```
|
||||
|
||||
统计按 `Asia/Shanghai` 自然日归档,当前支持 GB32960、JT808、宇通 MQTT 中能解析出 `total_mileage_km` 的数据。JT808 的总里程来自位置附加信息 `0x01`,单位按协议转换为 km;宇通 MQTT 的 `TOTAL_MILEAGE` 原始单位为米,入库前转换为 km。
|
||||
|
||||
## Redis 实时层
|
||||
|
||||
`realtime-api` 消费 `vehicle.event.go.unified.v1`,按 VIN/vehicle key 维护准实时快照。主要能力:
|
||||
|
||||
| API | 说明 |
|
||||
| --- | --- |
|
||||
| `/api/realtime/vehicles/{vin}` | 查询车辆合并实时快照 |
|
||||
| `/api/realtime/vehicles/{vin}/online` | 查询车辆是否在线 |
|
||||
| `/api/realtime/vehicles/{vin}/protocols/{protocol}` | 查询指定协议的实时快照 |
|
||||
|
||||
Redis 只作为实时缓存;历史和统计以 TDengine/MySQL 为准。
|
||||
|
||||
## API 入口
|
||||
|
||||
公网基础地址:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210
|
||||
```
|
||||
|
||||
常用查询:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=YUTONG_MQTT&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/locations?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/mileage-points?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/stats/daily-metrics?dateFrom=2026-07-02&dateTo=2026-07-02&limit=20'
|
||||
```
|
||||
|
||||
分页参数统一使用 `limit` 和 `offset`。
|
||||
|
||||
部署后推荐直接运行 Go 原生生产 smoke:
|
||||
|
||||
```bash
|
||||
python3 tools/go_prod_acceptance.py --date 2026-07-02
|
||||
```
|
||||
|
||||
该聚合脚本会串行执行 systemd/端口检查、Kafka topic/consumer lag 检查、HTTP 数据链路检查,任一子检查失败时整体退出码为非 0。需要 SSH 可登录应用 ECS 和 Kafka ECS。
|
||||
|
||||
也可以单独运行子检查:
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
|
||||
```
|
||||
|
||||
`go_systemd_prod_smoke.py` 通过 SSH 检查四个 Go systemd 服务是否 `active` 且 `enabled`,并确认 `808`、`32960` 由 `gateway` 监听、`20210` 由 `realtime-api` 监听,同时要求 `/opt/lingniu-go-native/current` 指向有效 release、四个生产二进制均可执行、`/opt/lingniu-go-native/spool/gateway` 没有待回放文件,避免旧 Java/Docker 进程占用生产端口、部署目录错乱或 Kafka spool 静默堆积。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
`go_native_prod_smoke.py` 通过 `20210` HTTP API 验证 GB32960、JT808、宇通 MQTT 的 RAW 查询及结构化 `parsed_json`,三类协议的位置和里程点查询及 `frame_id` RAW 回链,GB32960/JT808 的 `daily_mileage_km`、`daily_total_mileage_km` 统计公式,以及三类协议的 Redis realtime snapshot、online、protocol 查询。实时 snapshot/protocol 查询会校验 `updated_at_ms` 新鲜度;默认检查今天东八区数据时,还会要求三类 RAW 最新样本不超过 15 分钟,避免旧数据误判为接收正常;任一检查不达标时退出码为非 0。
|
||||
|
||||
## 部署命令
|
||||
|
||||
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS,切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收:
|
||||
|
||||
```bash
|
||||
python3 tools/go_native_deploy.py --date 2026-07-02
|
||||
```
|
||||
|
||||
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout` 和 `--spool-poll-interval` 调整。
|
||||
|
||||
脚本执行的发布动作等价于:
|
||||
|
||||
```bash
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/history-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/stat-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/realtime-api
|
||||
scp release.tar.gz root@115.29.187.205:/tmp/
|
||||
ln -sfn /opt/lingniu-go-native/releases/<git-short-sha> /opt/lingniu-go-native/current
|
||||
systemctl daemon-reload
|
||||
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
```
|
||||
|
||||
只更新查询 API 时,可以仅替换 `/opt/lingniu-go-native/current/realtime-api` 并重启:
|
||||
|
||||
```bash
|
||||
systemctl restart lingniu-go-realtime-api.service
|
||||
```
|
||||
|
||||
## 运行检查
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-history-writer --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-stat-writer --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-realtime-api --since '5 minutes ago' --no-pager
|
||||
```
|
||||
|
||||
2026-07-02 01:30 CST 已验证:
|
||||
|
||||
- 四个 Go systemd 服务均为 `active` 且 `enabled`。
|
||||
- `/opt/lingniu-go-native/current` 指向有效 release,四个生产二进制均存在且可执行。
|
||||
- `gateway` 正在监听 `32960` 和 `808`。
|
||||
- `realtime-api` 正在监听 `20210`。
|
||||
- gateway Kafka spool 当前无待回放文件。
|
||||
- GB32960、JT808、YUTONG_MQTT 的东八区 RAW 查询均可命中生产数据,且最新 RAW 样本包含结构化 `parsed_json`。
|
||||
- GB32960、JT808、YUTONG_MQTT 的最新 RAW 样本均在 15 分钟内。
|
||||
- GB32960、JT808、YUTONG_MQTT 的 `vehicle_locations`、`vehicle_mileage_points` 查询均可命中生产数据,且样本包含 `frame_id` 回链 RAW。
|
||||
- `vehicle_daily_metric` 可查到 `daily_mileage_km` 和 `daily_total_mileage_km`,且公式满足 `daily_mileage_km = latest_total_mileage_km - first_total_mileage_km`、`daily_total_mileage_km = latest_total_mileage_km`。
|
||||
- Redis realtime 可查到 GB32960、JT808、YUTONG_MQTT 的在线状态和协议快照,且 snapshot/protocol 快照最近刷新。
|
||||
@@ -0,0 +1,888 @@
|
||||
# Go Vehicle Gateway 旁路验证记录
|
||||
|
||||
验证时间:2026-07-01 22:00-22:10 CST
|
||||
|
||||
## 部署版本
|
||||
|
||||
- Git commit:`60ca42e`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-60ca42e-20260701220622`
|
||||
- 部署方式:ECS `115.29.187.205` 上 Docker Compose 旁路部署
|
||||
- Compose 目录:`/opt/lingniu-go`
|
||||
|
||||
## 旁路端口和 topic
|
||||
|
||||
为了不影响现有 Java 生产服务,本次没有占用生产端口和生产 topic。
|
||||
|
||||
| 项 | 值 |
|
||||
|---|---|
|
||||
| GB32960 旁路 TCP | `115.29.187.205:23296 -> container :32960` |
|
||||
| JT808 旁路 TCP | `115.29.187.205:18080 -> container :808` |
|
||||
| Realtime API | `115.29.187.205:20210` |
|
||||
| GB32960 RAW topic | `vehicle.raw.go.gb32960.v1` |
|
||||
| JT808 RAW topic | `vehicle.raw.go.jt808.v1` |
|
||||
| Yutong MQTT RAW topic | `vehicle.raw.go.yutong-mqtt.v1` |
|
||||
| Unified topic | `vehicle.event.go.unified.v1` |
|
||||
|
||||
## 容器状态
|
||||
|
||||
ECS 上四个 Go 容器均已启动:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway Up
|
||||
go-history-writer Up
|
||||
go-stat-writer Up
|
||||
go-realtime-api Up
|
||||
```
|
||||
|
||||
## Kafka 验证
|
||||
|
||||
向 `18080` 注入 JT808 样例帧后,`vehicle.raw.go.jt808.v1` 和
|
||||
`vehicle.event.go.unified.v1` 均消费到统一 JSON envelope。
|
||||
|
||||
样例解析结果包含:
|
||||
|
||||
- `protocol=JT808`
|
||||
- `message_id=0x0200`
|
||||
- `phone=013307795425`
|
||||
- `longitude=121.069881`
|
||||
- `latitude=30.590151`
|
||||
- `speed_kmh=23`
|
||||
- `total_mileage_km=10241.2`
|
||||
- 表 27 附加项 `0x01` 原始值 `0001900C`
|
||||
|
||||
## TDengine 验证
|
||||
|
||||
目标库:`lingniu_vehicle_ts`
|
||||
|
||||
```text
|
||||
raw_frames count = 3
|
||||
vehicle_locations count = 2
|
||||
vehicle_mileage_points count = 2
|
||||
```
|
||||
|
||||
最新 RAW 标识:
|
||||
|
||||
```text
|
||||
GB32960 | LNBSCB3D4R1234567 | LNBSCB3D4R1234567 |
|
||||
JT808 | JT808:013307795425 | | 013307795425
|
||||
JT808 | JT808:013307795425 | | 013307795425
|
||||
```
|
||||
|
||||
说明:JT808 样例 phone 当前没有在 `vehicle_identity_binding` 中解析到 VIN,因此
|
||||
TDengine tag 使用 `vehicle_key=JT808:013307795425`。
|
||||
|
||||
## Redis 验证
|
||||
|
||||
向 `23296` 注入带 VIN 的 GB32960 合成帧后,Realtime API 可查询:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LNBSCB3D4R1234567
|
||||
```
|
||||
|
||||
返回字段包含:
|
||||
|
||||
- `vin=LNBSCB3D4R1234567`
|
||||
- `protocols=["GB32960"]`
|
||||
- `online=true`
|
||||
- `speed_kmh=30`
|
||||
- `total_mileage_km=10000`
|
||||
- `soc_percent=85`
|
||||
- `longitude=121`
|
||||
- `latitude=30.56`
|
||||
|
||||
## MySQL 统计验证
|
||||
|
||||
向 `23296` 注入带 VIN 的 GB32960 合成帧后,`vehicle_daily_metric` 写入:
|
||||
|
||||
```text
|
||||
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_mileage_km | 0.000 | 10000.000 | 10000.000 | 1
|
||||
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_total_mileage_km | 10000.000 | 10000.000 | 10000.000 | 1
|
||||
```
|
||||
|
||||
说明:本次 GB32960 合成帧的设备时间为测试值,因此统计日期为 `2020-07-01`。
|
||||
|
||||
## 已修复问题
|
||||
|
||||
- TDengine WebSocket 普通 `INSERT` 使用参数占位符时,字符串未按预期加引号,导致语法错误。已在 `60ca42e` 中改为显式 SQL literal 转义。
|
||||
- MySQL Go DSN 中 `charset=utf8mb4%2Cutf8` 会被 driver 传给 MySQL,导致语法错误;ECS 旁路 env 已改为 `charset=utf8mb4`。
|
||||
- TDengine ECS 当前可用 root 密码与早先记录不一致,本次旁路 env 已按实际可认证值修正。
|
||||
|
||||
## 未完成
|
||||
|
||||
- 未切换真实生产端口 `32960/808`。
|
||||
- 未将真实外部 32960/808 流量转发到 Go 旁路端口。
|
||||
- 未开启宇通 MQTT 正式订阅。
|
||||
- JT808 样例未解析到 VIN,需维护 `vehicle_identity_binding` 后再验证 808 每日统计。
|
||||
|
||||
## 2026-07-01 22:22 真实归档帧复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`5f4f4fd`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5f4f4fd-20260701222128`
|
||||
|
||||
本轮补充验证内容:
|
||||
|
||||
- JT808 支持 2019 版 versioned header,真实帧 `0x0200` 可解析出 `phone=14894135060`、`longitude=117.178187`、`latitude=30.570155`、`speed_kmh=33.4`、`total_mileage_km=7868.9`。
|
||||
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 可解析 `0x01` 整车、`0x02` 驱动电机、`0x03` 燃料电池、`0x04` 发动机、`0x05` 位置、`0x06` 极值、`0x07` 报警、`0x08` 电压、`0x09` 温度;后续 `0x30` 广东燃料电池扩展暂以 unknown raw 保留。
|
||||
- GB32960 真实帧字段包含 `fuel_cell_hydrogen_consumption_kg_per_100km=1.8`、`longitude=120.800326`、`latitude=31.634907`、`total_mileage_km=53490.9`。
|
||||
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回上述经纬度和氢耗字段。
|
||||
- TDengine `vehicle_locations` 和 `vehicle_mileage_points` 已写入真实 GB32960 位置和里程点:
|
||||
|
||||
```text
|
||||
2020-07-01 08:09:22 | GB32960 | LB9A32A21R0LS1707 | lon=120.800326 | lat=31.634907 | mileage=53490.9
|
||||
```
|
||||
|
||||
后续仍需补齐:
|
||||
|
||||
- JT808 真实 phone `14894135060` 仍未解析到 VIN,需维护身份绑定后再验证 VIN 维度实时和统计。
|
||||
|
||||
## 2026-07-01 22:28 广东燃料电池扩展复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`a15de91`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-a15de91-20260701222743`
|
||||
|
||||
本轮补充验证内容:
|
||||
|
||||
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 的 vendor 尾部已从 `unknown_unit` 补为结构化块:
|
||||
`0x30 gd_fc_stack`、`0x31 gd_fc_auxiliary`、`0x32 gd_fc_dcdc`、`0x33 gd_fc_air_conditioner`、
|
||||
`0x34 gd_fc_vehicle_info`、`0x80 gd_fc_demo_extension`、`0x83 gd_fc_vendor_tlv`。
|
||||
- Kafka `vehicle.raw.go.gb32960.v1` 最新消息包含上述 block,并输出关键扁平字段:
|
||||
`gd_fc_stack_hydrogen_inlet_pressure_kpa=97`、`gd_fc_stack_cell_count=108`、
|
||||
`gd_fc_vehicle_hydrogen_mass_kg=4.3`、`gd_fc_demo_stack_temp_c=65`。
|
||||
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回 vendor 扁平字段:
|
||||
|
||||
```text
|
||||
gd_fc_dcdc_controller_temp_c=47
|
||||
gd_fc_demo_stack_temp_c=65
|
||||
gd_fc_stack_cell_count=108
|
||||
gd_fc_stack_frame_cell_count=108
|
||||
gd_fc_stack_hydrogen_inlet_pressure_kpa=97
|
||||
gd_fc_stack_water_outlet_temp_c=65
|
||||
gd_fc_vehicle_hydrogen_mass_kg=4.3
|
||||
```
|
||||
|
||||
- TDengine `raw_frames` 中 GB32960 行数增加到 `4`,最新 RAW 标识:
|
||||
|
||||
```text
|
||||
2026-07-01 14:28:24.869 | 172.20.0.1:47082 | GB32960 | LB9A32A21R0LS1707 | OK
|
||||
```
|
||||
|
||||
后续仍需补齐:
|
||||
|
||||
- JT808 真实 phone `14894135060` 到 VIN 的绑定数据维护与统计复验。
|
||||
- Go 旁路尚未切换到生产 `32960/808` 端口;仍运行在 `23296/18080`。
|
||||
|
||||
## 2026-07-01 22:43 JT808 前导零 phone 身份解析复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`6b55694`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-6b55694-20260701224300`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- JT808 2011 老版包头的终端手机号按 BCD[6] 解析会保留前导 `0`,例如包头为 `013079963379`。
|
||||
- `vehicle_identity_binding.phone` 中实际维护的是去前导零后的手机号 `13079963379`。
|
||||
- 身份解析候选键已调整为先查原始 phone,再查去前导零 phone,之后再按 `device_id`、`plate` 降级。
|
||||
|
||||
真实帧复验样本:
|
||||
|
||||
- 归档帧:`/opt/lingniu/vehicle-ingest/gb32960/archive/2026/07/01/JT808/unknown/1782903942845000.bin`
|
||||
- 消息:`0x0200` 位置信息汇报
|
||||
- 包头 phone:`013079963379`
|
||||
- 绑定表 phone:`13079963379`
|
||||
- 解析 VIN:`LKLG7C4E3NA774736`
|
||||
|
||||
Kafka `vehicle.raw.go.jt808.v1` 最新消息确认:
|
||||
|
||||
```text
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
parsed.header.phone_trim_zero=13079963379
|
||||
parsed.identity.resolved=true
|
||||
parsed.identity.source=phone
|
||||
parsed.identity.value=13079963379
|
||||
```
|
||||
|
||||
Realtime API 确认:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
speed_kmh=0
|
||||
total_mileage_km=0
|
||||
device_time=2026-07-01T19:05:40+08:00
|
||||
```
|
||||
|
||||
TDengine 确认:
|
||||
|
||||
```text
|
||||
raw_frames:
|
||||
2026-07-01 14:49:10.913 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | message_id=512
|
||||
|
||||
vehicle_locations:
|
||||
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | lon=119.557997 | lat=29.049092 | speed=0 | mileage=0
|
||||
|
||||
vehicle_mileage_points:
|
||||
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | mileage=0
|
||||
```
|
||||
|
||||
MySQL `vehicle_daily_metric` 确认:
|
||||
|
||||
```text
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
说明:该真实位置帧携带的表 27 附加项 `0x01` 总里程值为 `0`,因此本次只验证身份解析、链路落地和差值统计写入;要验证非零日里程,需要回放或等待同一 VIN 的非零总里程连续位置点。
|
||||
|
||||
## 2026-07-01 22:55 Kafka 发布重试部署复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`7e7066d`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7e7066d-20260701225503`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- gateway 的 Kafka 发布增加同步重试封装。
|
||||
- 默认配置:
|
||||
- `KAFKA_PUBLISH_ATTEMPTS=3`
|
||||
- `KAFKA_PUBLISH_BACKOFF_MS=100`
|
||||
- `KAFKA_PUBLISH_TIMEOUT_MS=3000`
|
||||
- RAW 仍先于 unified event 写入;RAW 写失败不会继续写 unified。
|
||||
- 单元测试覆盖首次失败后成功、重试耗尽返回错误、上下文取消时停止重试。
|
||||
|
||||
部署后容器状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway | go-7e7066d-20260701225503 | Up
|
||||
go-history-writer | go-7e7066d-20260701225503 | Up
|
||||
go-stat-writer | go-7e7066d-20260701225503 | Up
|
||||
go-realtime-api | go-7e7066d-20260701225503 | Up
|
||||
```
|
||||
|
||||
真实 808 帧回放复验:
|
||||
|
||||
```text
|
||||
source_endpoint=172.20.0.1:51966
|
||||
protocol=JT808
|
||||
message_id=0x0200
|
||||
phone=013079963379
|
||||
vin=LKLG7C4E3NA774736
|
||||
parsed.identity.value=13079963379
|
||||
```
|
||||
|
||||
Kafka `vehicle.raw.go.jt808.v1` 确认新消息写入,partition 6 offset 从 `1` 增加到 `2`。最新消息包含:
|
||||
|
||||
```text
|
||||
received_at_ms=1782917751903
|
||||
source_endpoint=172.20.0.1:51966
|
||||
vin=LKLG7C4E3NA774736
|
||||
fields.longitude=119.557997
|
||||
fields.latitude=29.049092
|
||||
fields.total_mileage_km=0
|
||||
```
|
||||
|
||||
TDengine `raw_frames` 确认新 RAW 落地:
|
||||
|
||||
```text
|
||||
2026-07-01 14:55:51.903 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | source=172.20.0.1:51966
|
||||
```
|
||||
|
||||
Realtime API 确认 Redis 已刷新:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
|
||||
received_at_ms=1782917751903
|
||||
source_endpoint=172.20.0.1:51966
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=0
|
||||
```
|
||||
|
||||
说明:当前重试层只能覆盖 Kafka 短暂抖动;如果 Kafka 长时间不可用,仍需后续增加本地磁盘 spool/WAL 和恢复补发。
|
||||
|
||||
## 2026-07-01 23:02 切换 Go 到生产 32960/808 端口
|
||||
|
||||
本轮操作:
|
||||
|
||||
- 停止 ECS 上原有 Java Docker 容器:
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
- `vehicle-history-app`
|
||||
- `vehicle-analytics-app`
|
||||
- `vehicle-state-app`
|
||||
- `telemetry-field-parser-app`
|
||||
- 保留 Go sidecar 和 Portainer agent。
|
||||
- 修改 `/opt/lingniu-go/.env`:
|
||||
- `GO_GB32960_TCP_PORT=32960`
|
||||
- `GO_JT808_TCP_PORT=808`
|
||||
- Go gateway 使用镜像:
|
||||
- `crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-243de5c-20260701230028`
|
||||
- 开启 Kafka 本地 spool:
|
||||
- host:`/opt/lingniu-go/spool/gateway`
|
||||
- container:`/data/spool/gateway`
|
||||
|
||||
端口验证:
|
||||
|
||||
```text
|
||||
0.0.0.0:32960 -> go-vehicle-gateway:32960
|
||||
0.0.0.0:808 -> go-vehicle-gateway:808
|
||||
18080/23296 -> 已释放
|
||||
```
|
||||
|
||||
启动日志确认:
|
||||
|
||||
```text
|
||||
kafka durable spool enabled dir=/data/spool/gateway replay_interval_ms=1000
|
||||
tcp listener started protocol=GB32960 addr=[::]:32960
|
||||
tcp listener started protocol=JT808 addr=[::]:808
|
||||
```
|
||||
|
||||
真实连接确认:
|
||||
|
||||
```text
|
||||
GB32960 external remote: 8.134.95.166:50084
|
||||
JT808 external remote: 115.231.168.135:13801
|
||||
```
|
||||
|
||||
生产 `808` 回放复验:
|
||||
|
||||
```text
|
||||
source_endpoint=172.20.0.1:54680
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=0
|
||||
```
|
||||
|
||||
TDengine 最新 RAW 复验显示生产 808 已有真实外部数据持续落地:
|
||||
|
||||
```text
|
||||
2026-07-01 15:03:02.960 | JT808 | LKLG7C4E1NA774802 | 013079963291 | 222.66.200.68:28394
|
||||
2026-07-01 15:03:02.114 | JT808 | | 14894135583 | 122.152.221.156:33871
|
||||
2026-07-01 15:03:01.352 | JT808 | | 013307795519 | 115.231.168.135:13801
|
||||
2026-07-01 15:03:01.330 | JT808 | LKLG7C4E3NA774736 | 013079963379 | 222.66.200.68:28393
|
||||
```
|
||||
|
||||
Kafka spool 目录确认:
|
||||
|
||||
```text
|
||||
/opt/lingniu-go/spool/gateway file count = 0
|
||||
```
|
||||
|
||||
说明:生产 808 中仍有部分 phone 未解析出 VIN,后续需要继续维护 `vehicle_identity_binding` 的 phone/device_id/plate 到 VIN 映射,或者增强 808 注册/鉴权帧中的 identity 回写。
|
||||
|
||||
## 2026-07-01 23:09 808 里程统计防 0 污染复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`d347525`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-d347525-20260701230845`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- `stat-writer` 不再把 `total_mileage_km <= 0` 的采样写入 MySQL 每日指标。
|
||||
- MySQL upsert 对历史 `first_total_mileage_km=0` / `latest_total_mileage_km=0` 做纠偏;后续首次有效总里程会替换掉历史 0。
|
||||
- `realtime-api` 合并实时快照时,非正 `total_mileage_km` 不再覆盖已有正值;速度、位置、状态等其他字段仍按事件时间更新。
|
||||
|
||||
测试验证:
|
||||
|
||||
```text
|
||||
go test ./...
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
生产回放验证:
|
||||
|
||||
- 第一条 808 帧:`device_time=2026-07-01T23:09:35+08:00`,`total_mileage_km=12345.6`
|
||||
- 第二条 808 帧:`device_time=2026-07-01T23:09:36+08:00`,`total_mileage_km=0`
|
||||
- 两条帧使用同一 VIN:`LKLG7C4E3NA774736`
|
||||
|
||||
Redis merged 快照确认:第二条 0 里程帧没有覆盖已有有效总里程。
|
||||
|
||||
```text
|
||||
event_time_ms=1782918576000
|
||||
source_endpoint=172.20.0.1:40946
|
||||
speed_kmh=0
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=12345.6
|
||||
field_times_ms.total_mileage_km=1782918575000
|
||||
```
|
||||
|
||||
MySQL `vehicle_daily_metric` 确认:0 里程采样未进入统计,sample_count 只因有效总里程采样增加一次。
|
||||
|
||||
```text
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=13 | first=12345.600 | latest=12345.600
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 12345.600 | sample_count=13 | first=12345.600 | latest=12345.600
|
||||
```
|
||||
|
||||
说明:生产 TDengine 中已经存在大量 808 非零 `total_mileage_km` 采样,但许多 phone 尚未映射到 VIN,因此 MySQL 按 VIN 的统计只会覆盖已解析 VIN 的车辆。下一步应继续补齐 `vehicle_identity_binding`,让更多 808 车辆进入统计。
|
||||
|
||||
## 2026-07-01 23:23 808 注册身份自动维护复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`7b44f97`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7b44f97-20260701232200`
|
||||
|
||||
本轮修复内容:
|
||||
|
||||
- 808 `0x0100` 注册帧解析 `province`、`city`、`manufacturer`、`device_type`、`device_id`、`plate_color`、`plate`。
|
||||
- 808 `0x0102` 鉴权帧解析 `auth_token`。
|
||||
- Go gateway 在 identity resolve 后写入 `jt808_registration`,普通上行也会刷新 `latest_seen_at`。
|
||||
- 注册帧通过车牌解析到 VIN 后,会尝试把 phone/device/plate 反写到 `vehicle_identity_binding`。
|
||||
- 注册帧车牌支持 GBK 解码,避免 `Incorrect string value` 写入错误。
|
||||
|
||||
验证命令:
|
||||
|
||||
```text
|
||||
go test ./...
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
ECS 运行状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway | go-7b44f97-20260701232200 | 0.0.0.0:808->808, 0.0.0.0:32960->32960
|
||||
go-history-writer | go-7b44f97-20260701232200 | Up
|
||||
go-stat-writer | go-7b44f97-20260701232200 | Up
|
||||
go-realtime-api | go-7b44f97-20260701232200 | 0.0.0.0:20210->20210
|
||||
```
|
||||
|
||||
生产日志确认:
|
||||
|
||||
- 新版本启动后真实 808、32960 连接持续进入。
|
||||
- GBK 车牌注册帧不再出现 `Incorrect string value`。
|
||||
|
||||
MySQL `jt808_registration` 确认:
|
||||
|
||||
```text
|
||||
registration_rows=214
|
||||
distinct_phone=214
|
||||
vin_rows=74
|
||||
plate_rows=74
|
||||
device_rows=72
|
||||
|
||||
013079963379 | device_id=9963379 | plate=沪A06788F | vin=LKLG7C4E3NA774736 | latest_registered_at=2026-07-01 23:23:01
|
||||
013079963321 | device_id=9963321 | plate=沪A59613F | vin=LKLG7C4EXNA774796 | latest_registered_at=2026-07-01 23:23:04
|
||||
```
|
||||
|
||||
说明:`jt808_registration.phone` 保存 808 包头原始 phone;`vehicle_identity_binding.phone` 中已有部分去前导零 phone。Go resolver 会同时查询原始 phone 和去前导零 phone。
|
||||
|
||||
## 2026-07-01 23:28 生产链路与指标清理复验
|
||||
|
||||
本轮复验目标:
|
||||
|
||||
- 确认 Go gateway 的 Kafka durable spool 是重启窗口残留,而不是持续发布失败。
|
||||
- 确认 stat-writer / history-writer / realtime-api 的 Kafka consumer group 无 lag。
|
||||
- 确认真实 808 车辆 registration、TDengine RAW/里程点、Redis 实时查询均在持续更新。
|
||||
- 清理之前生产合成回放产生的 JT808 当日指标测试数据。
|
||||
|
||||
Kafka topic / consumer group:
|
||||
|
||||
```text
|
||||
vehicle.raw.go.jt808.v1 有持续 offset
|
||||
vehicle.raw.go.gb32960.v1 有持续 offset
|
||||
vehicle.event.go.unified.v1 有持续 offset
|
||||
|
||||
go-stat-writer LAG=0
|
||||
go-history-writer LAG=0
|
||||
go-realtime-api LAG=0
|
||||
```
|
||||
|
||||
TDengine:
|
||||
|
||||
```text
|
||||
raw_frames:
|
||||
JT808 2922
|
||||
GB32960 27
|
||||
|
||||
vehicle_mileage_points:
|
||||
JT808 2050
|
||||
GB32960 2
|
||||
```
|
||||
|
||||
Redis 实时查询样例:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
vin=LKLG7C4E3NA774736
|
||||
source_endpoint=222.66.200.68:29646
|
||||
plate=沪A06788F
|
||||
longitude=119.556866
|
||||
latitude=29.047911
|
||||
total_mileage_km=12345.6
|
||||
```
|
||||
|
||||
说明:该 Redis merged 快照保留了此前有效 `total_mileage_km=12345.6`,后续真实 808 位置帧上报的 `total_mileage_km=0` 不会覆盖该字段。TDengine 中该 VIN 后续真实里程点的 `total_mileage_km` 均为 0,因此 stat-writer 按“只统计正总里程”的规则不再写 JT808 每日指标。
|
||||
|
||||
MySQL 指标清理:
|
||||
|
||||
```text
|
||||
删除前:
|
||||
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_mileage_km | first=0 | latest=0 | sample_count=7
|
||||
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_total_mileage_km | first=0 | latest=0 | sample_count=7
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
|
||||
|
||||
删除后等待真实流量窗口:
|
||||
JT808 2026-07-01 metric rows = 0
|
||||
```
|
||||
|
||||
清理原因:
|
||||
|
||||
- `LKLG7C4E3NA774736` 的 12345.6 来自此前生产连通性合成回放,不是真实平台上报。
|
||||
- `LKLG7C4E2NA774775` 的 0 指标来自旧逻辑污染,当前 stat-writer 已跳过非正总里程。
|
||||
|
||||
Kafka spool:
|
||||
|
||||
```text
|
||||
23:24 spool_count=904
|
||||
23:25 spool_count=862
|
||||
23:28 spool_count=722
|
||||
new_spool_1min=0
|
||||
```
|
||||
|
||||
说明:spool 正在回放下降,且最近 1 分钟没有新增文件;生产 gateway 日志未见 `Incorrect string value`、Kafka publish error 或 replay error。
|
||||
|
||||
## 2026-07-01 23:34 每日指标查询 API 复验
|
||||
|
||||
部署版本已更新:
|
||||
|
||||
- Git commit:`56f4811`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-56f4811-20260701233311`
|
||||
|
||||
新增接口:
|
||||
|
||||
```text
|
||||
GET /api/stats/daily-metrics
|
||||
```
|
||||
|
||||
查询参数:
|
||||
|
||||
```text
|
||||
vin 可选
|
||||
protocol 可选,GB32960 / JT808 / YUTONG_MQTT
|
||||
metricKey 可选,daily_mileage_km / daily_total_mileage_km
|
||||
dateFrom 可选,YYYY-MM-DD
|
||||
dateTo 可选,YYYY-MM-DD
|
||||
limit 可选,默认 50,最大 1000
|
||||
offset 可选,默认 0
|
||||
```
|
||||
|
||||
生产可访问样例:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01&limit=10
|
||||
```
|
||||
|
||||
返回确认:
|
||||
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"vin": "LB9A32A21R0LS1707",
|
||||
"stat_date": "2020-07-01",
|
||||
"protocol": "GB32960",
|
||||
"metric_key": "daily_mileage_km",
|
||||
"metric_value": 0,
|
||||
"metric_unit": "km",
|
||||
"first_total_mileage_km": 53490.9,
|
||||
"latest_total_mileage_km": 53490.9,
|
||||
"sample_count": 3,
|
||||
"calculation_method": "TOTAL_MILEAGE_DIFF",
|
||||
"created_at": "2026-07-01 22:14:13",
|
||||
"updated_at": "2026-07-01 22:28:25"
|
||||
},
|
||||
{
|
||||
"vin": "LB9A32A21R0LS1707",
|
||||
"stat_date": "2020-07-01",
|
||||
"protocol": "GB32960",
|
||||
"metric_key": "daily_total_mileage_km",
|
||||
"metric_value": 53490.9,
|
||||
"metric_unit": "km",
|
||||
"first_total_mileage_km": 53490.9,
|
||||
"latest_total_mileage_km": 53490.9,
|
||||
"sample_count": 3,
|
||||
"calculation_method": "TOTAL_MILEAGE_DIFF",
|
||||
"created_at": "2026-07-01 22:14:13",
|
||||
"updated_at": "2026-07-01 22:28:25"
|
||||
}
|
||||
],
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
"total": 2
|
||||
}
|
||||
```
|
||||
|
||||
JT808 真实生产查询:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/stats/daily-metrics?protocol=JT808&dateFrom=2026-07-01&dateTo=2026-07-01&limit=10
|
||||
```
|
||||
|
||||
返回 `total=0`。原因是此前合成回放产生的测试指标已清理,当前真实 808 已绑定 VIN 的车辆仍上报 `total_mileage_km=0`,stat-writer 按规则不写非正总里程指标。
|
||||
|
||||
实时接口回归:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
```
|
||||
|
||||
返回 `online=true`,说明新增 stats API 没有影响 Redis 实时查询。
|
||||
|
||||
发布后 spool 观察:
|
||||
|
||||
```text
|
||||
发布后立即:spool_count=1101,new_spool_1min=102
|
||||
等待 60 秒:spool_count=1033,new_spool_1min=0
|
||||
```
|
||||
|
||||
说明:本次发布重启窗口产生的 spool 正在回放下降,且最近 1 分钟没有新增;gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。
|
||||
|
||||
## 2026-07-01 23:50 生产端口接管与 RAW 查询 API 复验
|
||||
|
||||
部署版本:
|
||||
|
||||
- Git commit:`5937132`
|
||||
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5937132-20260701234220`
|
||||
|
||||
生产端口状态:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway 0.0.0.0:808->808/tcp
|
||||
go-vehicle-gateway 0.0.0.0:32960->32960/tcp
|
||||
```
|
||||
|
||||
旧 Java 容器状态:
|
||||
|
||||
```text
|
||||
gb32960-ingest-app restart=no status=exited
|
||||
jt808-ingest-app restart=no status=exited
|
||||
yutong-mqtt-app restart=no status=exited
|
||||
vehicle-history-app restart=no status=exited
|
||||
vehicle-analytics-app restart=no status=exited
|
||||
vehicle-state-app restart=no status=exited
|
||||
telemetry-field-parser-app restart=no status=exited
|
||||
```
|
||||
|
||||
生产连接确认:
|
||||
|
||||
- JT808 端口 `808` 已收到外部连接:`222.66.200.68`、`115.231.168.135`。
|
||||
- GB32960 端口 `32960` 已收到外部连接:`8.134.95.166`、`117.160.0.65`。
|
||||
- ECS 当前没有 `8089` 监听。
|
||||
|
||||
RAW 查询 API:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames
|
||||
```
|
||||
|
||||
生产可访问样例:
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
|
||||
http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
|
||||
```
|
||||
|
||||
接口复验结果:
|
||||
|
||||
```text
|
||||
JT808_RAW:
|
||||
total=1
|
||||
protocol=JT808
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
message_id=512
|
||||
message_id_hex=0x0200
|
||||
parse_status=OK
|
||||
raw_hex_len=94
|
||||
parsed_json_len=651
|
||||
fields_json_len=186
|
||||
|
||||
GB32960_RAW:
|
||||
total=1
|
||||
protocol=GB32960
|
||||
vin=LB9A32A21R0LS1707
|
||||
message_id=2
|
||||
message_id_hex=0x0002
|
||||
parse_status=OK
|
||||
raw_hex_len=1548
|
||||
parsed_json_len=3698
|
||||
fields_json_len=563
|
||||
```
|
||||
|
||||
同轮回归验证:
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
online=true
|
||||
protocols=["JT808"]
|
||||
|
||||
GET /api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2026-07-01&limit=1
|
||||
total=1
|
||||
protocol=GB32960
|
||||
metric_key=daily_mileage_km
|
||||
calculation_method=TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- RAW 查询已返回 `raw_hex`、`parsed_json`、`fields_json`,可以用于查看完整解析结果。
|
||||
- 日统计和实时查询在生产端口接管后仍可用。
|
||||
|
||||
## 2026-07-02 00:00 原生 systemd 部署与宇通 MQTT 复验
|
||||
|
||||
根据本次 goal 的新要求,Go 接入链路后续不再使用 Docker 部署。本轮已中断 Docker 镜像构建路径,改为 ECS 原生 Linux 二进制 + systemd。
|
||||
|
||||
部署版本:
|
||||
|
||||
- Git commit:`594ab2d`
|
||||
- 部署目录:`/opt/lingniu-go-native`
|
||||
- 当前 release:`/opt/lingniu-go-native/releases/594ab2d`
|
||||
- 当前指针:`/opt/lingniu-go-native/current -> /opt/lingniu-go-native/releases/594ab2d`
|
||||
|
||||
systemd 服务:
|
||||
|
||||
```text
|
||||
lingniu-go-gateway.service active
|
||||
lingniu-go-history-writer.service active
|
||||
lingniu-go-stat-writer.service active
|
||||
lingniu-go-realtime-api.service active
|
||||
```
|
||||
|
||||
端口确认:
|
||||
|
||||
```text
|
||||
[::]:32960 gateway
|
||||
[::]:808 gateway
|
||||
[::]:20210 realtime-api
|
||||
```
|
||||
|
||||
Go Docker 容器已停止:
|
||||
|
||||
```text
|
||||
go-vehicle-gateway Exited
|
||||
go-history-writer Exited
|
||||
go-stat-writer Exited
|
||||
go-realtime-api Exited
|
||||
```
|
||||
|
||||
宇通 MQTT:
|
||||
|
||||
```text
|
||||
yutong mqtt client started
|
||||
mqtt subscribed broker=ssl://cpxlm.axxc.cn:38883 topic=/ytforward/shln/+ qos=1
|
||||
```
|
||||
|
||||
说明:MQTT broker 存在短连接 `EOF` 后自动重连现象,但订阅后已收到真实数据并落 RAW。
|
||||
|
||||
RAW 查询复验:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=3
|
||||
|
||||
total=3
|
||||
protocol=YUTONG_MQTT
|
||||
vin=LMRKH9AC1R1004131
|
||||
vehicle_key=LMRKH9AC1R1004131
|
||||
parse_status=OK
|
||||
parsed_json_len=1104
|
||||
fields_json_len=116
|
||||
```
|
||||
|
||||
同轮回归:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
|
||||
total=1
|
||||
parse_status=OK
|
||||
|
||||
GET /api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
|
||||
total=1
|
||||
parse_status=OK
|
||||
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
online=true
|
||||
protocols=["JT808"]
|
||||
```
|
||||
|
||||
本轮落地文件:
|
||||
|
||||
- `deploy/systemd/README.md`
|
||||
- `deploy/systemd/lingniu-go-gateway.service`
|
||||
- `deploy/systemd/lingniu-go-history-writer.service`
|
||||
- `deploy/systemd/lingniu-go-stat-writer.service`
|
||||
- `deploy/systemd/lingniu-go-realtime-api.service`
|
||||
|
||||
## 2026-07-02 00:04 宇通 MQTT EOF 诊断
|
||||
|
||||
现象:
|
||||
|
||||
Go 原生部署后,`lingniu-go-gateway` 日志中出现多次:
|
||||
|
||||
```text
|
||||
mqtt connection lost error=EOF
|
||||
mqtt connection lost error=write: broken pipe
|
||||
mqtt subscribed topic=/ytforward/shln/+ qos=1
|
||||
```
|
||||
|
||||
10 分钟窗口统计:
|
||||
|
||||
```text
|
||||
mqtt subscribed = 33
|
||||
mqtt connection lost = 32
|
||||
```
|
||||
|
||||
对照旧 Java `yutong-mqtt-app` 历史日志,停止前也存在同类行为:
|
||||
|
||||
```text
|
||||
mqtt endpoint [yutong] received topic=/ytforward/shln/1 bytes=574
|
||||
mqtt endpoint [yutong] connection lost
|
||||
Caused by: java.io.EOFException
|
||||
mqtt endpoint [yutong] re-subscribed topic=/ytforward/shln/+ qos=1
|
||||
mqtt endpoint [yutong] received topic=/ytforward/shln/3 bytes=586
|
||||
```
|
||||
|
||||
判断:
|
||||
|
||||
- EOF/短连接不是 Go TLS 支持或 systemd 原生部署新引入的问题。
|
||||
- 旧 Java 和新 Go 都表现为:订阅、收到数据、broker 断开、自动重连。
|
||||
- 当前链路可持续收到宇通 MQTT 数据并写入 TDengine RAW。
|
||||
|
||||
复验:
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=1
|
||||
|
||||
total=1
|
||||
protocol=YUTONG_MQTT
|
||||
vin=LMRKH9ACXR1004094
|
||||
parse_status=OK
|
||||
parsed_json_len=1080
|
||||
fields_json_len=115
|
||||
```
|
||||
|
||||
运维建议:
|
||||
|
||||
- 单条 EOF 不作为不可用判断。
|
||||
- 以 `YUTONG_MQTT` RAW 入库增长、最近成功 `mqtt subscribed` 时间、服务 `systemctl is-active` 作为可用性判断。
|
||||
- 如果 EOF 频率继续升高且 RAW 不再增长,再联系宇通侧确认 broker 长连接策略、同一 clientId 并发限制或网络出口策略。
|
||||
158
docs/operations/go-vehicle-gateway-verification.md
Normal file
158
docs/operations/go-vehicle-gateway-verification.md
Normal file
@@ -0,0 +1,158 @@
|
||||
# Go Vehicle Gateway 验证手册
|
||||
|
||||
本文档用于验证 Go 重构运行面:
|
||||
|
||||
- `go-vehicle-gateway`:GB32960 TCP、JT808 TCP、宇通 MQTT 接入。
|
||||
- `go-history-writer`:Kafka RAW 写 TDengine。
|
||||
- `go-stat-writer`:Kafka RAW 写 MySQL 每日指标。
|
||||
- `go-realtime-api`:Kafka unified event 写 Redis,并提供实时查询 API。
|
||||
|
||||
## 本地构建验证
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./...
|
||||
go build ./cmd/gateway
|
||||
go build ./cmd/history-writer
|
||||
go build ./cmd/stat-writer
|
||||
go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
## 生产原生部署验证
|
||||
|
||||
```bash
|
||||
python3 tools/go_prod_acceptance.py --date 2026-07-02
|
||||
```
|
||||
|
||||
该命令会聚合执行:
|
||||
|
||||
- `go_systemd_prod_smoke.py`:应用 ECS systemd 服务 active/enabled、端口归属和 gateway spool
|
||||
- `go_kafka_prod_smoke.py`:Kafka topic 和消费组 lag
|
||||
- `go_native_prod_smoke.py`:HTTP API、RAW `parsed_json`、历史核心表 `frame_id` 回链、每日指标公式、Redis 实时新鲜度、TDengine/MySQL/Redis 数据链路
|
||||
|
||||
如需单独检查 systemd:
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
```
|
||||
|
||||
脚本通过 SSH 检查:
|
||||
|
||||
- `lingniu-go-gateway.service`
|
||||
- `lingniu-go-history-writer.service`
|
||||
- `lingniu-go-stat-writer.service`
|
||||
- `lingniu-go-realtime-api.service`
|
||||
- 上述服务均为 `active` 且 `enabled`
|
||||
- `808`、`32960` 由 `gateway` 监听
|
||||
- `20210` 由 `realtime-api` 监听
|
||||
- `/opt/lingniu-go-native/spool/gateway` 无待回放文件
|
||||
|
||||
## 生产环境变量
|
||||
|
||||
当前 goal 的生产面使用 ECS 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。以下环境变量维护在 `/opt/lingniu-go-native/env/*.env`:
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=172.17.111.56:9092
|
||||
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
|
||||
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
|
||||
IDENTITY_MYSQL_DSN=${MYSQL_DSN}
|
||||
VEHICLE_IDENTITY_TABLE=vehicle_identity_binding
|
||||
REDIS_ADDR=r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379
|
||||
REDIS_PASSWORD=<password>
|
||||
REDIS_DB=50
|
||||
```
|
||||
|
||||
宇通 MQTT 开启时再配置:
|
||||
|
||||
```bash
|
||||
YUTONG_MQTT_ENABLED=true
|
||||
YUTONG_MQTT_URI=<mqtt-uri>
|
||||
YUTONG_MQTT_TOPICS=/ytforward/shln/+
|
||||
YUTONG_MQTT_CLIENT_ID=lingniu-go-yutong-mqtt
|
||||
YUTONG_MQTT_USERNAME=<username>
|
||||
YUTONG_MQTT_PASSWORD=<password>
|
||||
```
|
||||
|
||||
## ECS 进程检查
|
||||
|
||||
```bash
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
```
|
||||
|
||||
## Kafka 验证
|
||||
|
||||
优先使用 smoke 脚本做可重复检查:
|
||||
|
||||
```bash
|
||||
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
|
||||
```
|
||||
|
||||
脚本会检查 `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1`、`vehicle.event.go.unified.v1` 是否存在,并汇总 `go-history-writer`、`go-stat-writer`、`go-realtime-api` 的消费 lag。运行环境需要 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
手工核对命令:
|
||||
|
||||
```bash
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.jt808.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.gb32960.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.yutong-mqtt.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.event.go.unified.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-history-writer
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-stat-writer
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-realtime-api
|
||||
```
|
||||
|
||||
## TDengine 验证
|
||||
|
||||
```sql
|
||||
USE lingniu_vehicle_ts;
|
||||
SHOW STABLES;
|
||||
SELECT COUNT(*) FROM raw_frames;
|
||||
SELECT COUNT(*) FROM vehicle_locations;
|
||||
SELECT COUNT(*) FROM vehicle_mileage_points;
|
||||
```
|
||||
|
||||
## MySQL 验证
|
||||
|
||||
```sql
|
||||
SELECT protocol, metric_key, COUNT(*)
|
||||
FROM vehicle_daily_metric
|
||||
GROUP BY protocol, metric_key;
|
||||
|
||||
SELECT *
|
||||
FROM vehicle_daily_metric
|
||||
WHERE metric_key IN ('daily_mileage_km', 'daily_total_mileage_km')
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
## Redis 实时查询
|
||||
|
||||
```bash
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>'
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/online'
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/protocols/JT808'
|
||||
```
|
||||
|
||||
## 验收证据
|
||||
|
||||
正式切换前至少记录:
|
||||
|
||||
- 一个真实 GB32960 VIN 的 RAW、location、mileage point、Redis snapshot。
|
||||
- 一个真实 JT808 VIN/phone 的 RAW、location、mileage point、daily metric、Redis snapshot。
|
||||
- 一个真实宇通 MQTT VIN 或 device_id 的 RAW、Redis snapshot。
|
||||
- `go-history-writer` 和 `go-stat-writer` 重启后 Kafka offset 能继续推进。
|
||||
720
docs/superpowers/plans/2026-07-01-go-ingest-redesign-phase1.md
Normal file
720
docs/superpowers/plans/2026-07-01-go-ingest-redesign-phase1.md
Normal file
@@ -0,0 +1,720 @@
|
||||
# Go Vehicle Ingest Redesign Phase 1 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build the first production-capable Go runtime for GB32960, JT808, and Yutong MQTT ingestion with unified Kafka, TDengine, MySQL statistics, and Redis realtime state boundaries.
|
||||
|
||||
**Architecture:** Create a new single Go module at `go/vehicle-gateway` and migrate only the useful pieces from the existing `go/ingest-edge` and `go/vehicle-state` prototypes. The gateway produces unified envelopes to Kafka; independent consumers write TDengine history, MySQL daily metrics, and Redis realtime state.
|
||||
|
||||
**Tech Stack:** Go 1.26+, Kafka (`segmentio/kafka-go`), Redis (`redis/go-redis`), MySQL (`go-sql-driver/mysql`), TDengine official Go connector, MQTT (`eclipse/paho.mqtt.golang`), standard library TCP.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- Create: `go/vehicle-gateway/go.mod`
|
||||
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/*`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/*`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/yutongmqtt/*`
|
||||
- Create: `go/vehicle-gateway/internal/gateway/*`
|
||||
- Create: `go/vehicle-gateway/internal/identity/*`
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/*`
|
||||
- Create: `go/vehicle-gateway/internal/history/*`
|
||||
- Create: `go/vehicle-gateway/internal/stats/*`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/*`
|
||||
- Create: `go/vehicle-gateway/internal/observability/*`
|
||||
- Modify: `README.md`
|
||||
- Modify: `docs/target-architecture.md`
|
||||
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||
|
||||
The existing `go/ingest-edge` and `go/vehicle-state` directories are migration sources only. After phase 1 is verified, remove or mark them superseded.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Create Single Go Module
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/go.mod`
|
||||
- Create: `go/vehicle-gateway/internal/observability/logger.go`
|
||||
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
|
||||
- [ ] **Step 1: Create module manifest**
|
||||
|
||||
Add `go/vehicle-gateway/go.mod`:
|
||||
|
||||
```go
|
||||
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/taosdata/driver-go/v3 v3.8.1
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add logger helper**
|
||||
|
||||
Add `go/vehicle-gateway/internal/observability/logger.go`:
|
||||
|
||||
```go
|
||||
package observability
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
func NewLogger(service string) *slog.Logger {
|
||||
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||
return slog.New(handler).With("service", service)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add temporary gateway entrypoint**
|
||||
|
||||
Add `go/vehicle-gateway/cmd/gateway/main.go`:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-gateway")
|
||||
logger.Info("vehicle gateway scaffold started")
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify scaffold builds**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go mod tidy
|
||||
go test ./...
|
||||
go build ./cmd/gateway
|
||||
```
|
||||
|
||||
Expected: all commands exit `0`.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Define Unified Envelope
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||
|
||||
- [ ] **Step 1: Write envelope tests**
|
||||
|
||||
Add `go/vehicle-gateway/internal/envelope/envelope_test.go`:
|
||||
|
||||
```go
|
||||
package envelope
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||
e := FrameEnvelope{
|
||||
Protocol: ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013307795425",
|
||||
Sequence: 1,
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7e02000000ff7e",
|
||||
}
|
||||
a := e.StableEventID()
|
||||
b := e.StableEventID()
|
||||
if a == "" || a != b {
|
||||
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests and confirm failure**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/envelope
|
||||
```
|
||||
|
||||
Expected: fail because `FrameEnvelope` is not defined.
|
||||
|
||||
- [ ] **Step 3: Implement envelope**
|
||||
|
||||
Add `go/vehicle-gateway/internal/envelope/envelope.go`:
|
||||
|
||||
```go
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Protocol string
|
||||
|
||||
const (
|
||||
ProtocolGB32960 Protocol = "GB32960"
|
||||
ProtocolJT808 Protocol = "JT808"
|
||||
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||
)
|
||||
|
||||
type ParseStatus string
|
||||
|
||||
const (
|
||||
ParseOK ParseStatus = "OK"
|
||||
ParsePartial ParseStatus = "PARTIAL"
|
||||
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||
)
|
||||
|
||||
type FrameEnvelope struct {
|
||||
EventID string `json:"event_id"`
|
||||
TraceID string `json:"trace_id"`
|
||||
Protocol Protocol `json:"protocol"`
|
||||
MessageID string `json:"message_id"`
|
||||
Sequence uint16 `json:"sequence"`
|
||||
VIN string `json:"vin,omitempty"`
|
||||
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
Parsed map[string]any `json:"parsed,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
ParseStatus ParseStatus `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) VehicleKey() string {
|
||||
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
return string(e.Protocol) + ":unknown"
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) StableEventID() string {
|
||||
if strings.TrimSpace(e.EventID) != "" {
|
||||
return e.EventID
|
||||
}
|
||||
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||
sum := sha256.Sum256([]byte(input))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||
if e.EventID == "" {
|
||||
e.EventID = e.StableEventID()
|
||||
}
|
||||
if e.ParseStatus == "" {
|
||||
e.ParseStatus = ParseOK
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify tests pass**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/envelope
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Implement JT808 Frame and 0200 Parser
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser_test.go`
|
||||
|
||||
- [ ] **Step 1: Add sample frame tests**
|
||||
|
||||
Use the production sample provided in the thread:
|
||||
|
||||
```text
|
||||
7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E
|
||||
```
|
||||
|
||||
Expected assertions:
|
||||
|
||||
- message id is `0x0200`
|
||||
- phone keeps normalized BCD string `013307795425`
|
||||
- sequence is `1`
|
||||
- `fields.total_mileage_km` is parsed from additional item `0x01`
|
||||
- latitude, longitude, speed, direction, alarm, status, and device time are present
|
||||
|
||||
- [ ] **Step 2: Implement parser**
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- strip `0x7e` start/end delimiters
|
||||
- unescape `0x7d 0x02 -> 0x7e`
|
||||
- unescape `0x7d 0x01 -> 0x7d`
|
||||
- verify XOR checksum
|
||||
- parse 2011/2013 common header
|
||||
- parse BCD phone from 6 bytes and keep both raw and normalized values in `parsed.header`
|
||||
- parse 0200 fixed body
|
||||
- parse additional item list as `id,length,value_hex`
|
||||
- parse additional `0x01` as `total_mileage_km = uint32 / 10`
|
||||
|
||||
- [ ] **Step 3: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/protocol/jt808
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Implement GB32960 Frame and Data Unit Parser
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser_test.go`
|
||||
|
||||
- [ ] **Step 1: Add parser tests**
|
||||
|
||||
Tests must cover:
|
||||
|
||||
- `##` frame boundary
|
||||
- command id
|
||||
- response flag
|
||||
- 17-byte VIN
|
||||
- encryption flag
|
||||
- payload length
|
||||
- BCC verification
|
||||
- realtime data command `0x02`
|
||||
- reissue data command `0x03`
|
||||
- data unit `0x01` vehicle status fields
|
||||
- data unit `0x05` position fields
|
||||
|
||||
- [ ] **Step 2: Implement parser**
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- parse header without allocating large temporary buffers
|
||||
- keep RAW as hex in envelope
|
||||
- write all recognized data units to `Parsed`
|
||||
- write core fields to `Fields`
|
||||
- unsupported data unit stays in `Parsed["unknown_units"]`
|
||||
|
||||
- [ ] **Step 3: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/protocol/gb32960
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Implement Kafka Sink
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink.go`
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink_test.go`
|
||||
|
||||
- [ ] **Step 1: Add sink interface**
|
||||
|
||||
Create a sink interface:
|
||||
|
||||
```go
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Sink interface {
|
||||
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||
Close() error
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Implement Kafka topic routing**
|
||||
|
||||
Rules:
|
||||
|
||||
- `GB32960 -> vehicle.raw.gb32960.v1`
|
||||
- `JT808 -> vehicle.raw.jt808.v1`
|
||||
- `YUTONG_MQTT -> vehicle.raw.yutong-mqtt.v1`
|
||||
- unified topic is `vehicle.event.unified.v1`
|
||||
- message key is `env.VehicleKey()`
|
||||
|
||||
- [ ] **Step 3: Verify routing with unit tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/eventbus
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Implement TDengine History Writer
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/history/schema.go`
|
||||
- Create: `go/vehicle-gateway/internal/history/writer.go`
|
||||
- Create: `go/vehicle-gateway/internal/history/writer_test.go`
|
||||
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||
|
||||
- [ ] **Step 1: Add schema bootstrap SQL**
|
||||
|
||||
Implement schema strings for:
|
||||
|
||||
- database `lingniu_vehicle_ts`
|
||||
- stable `raw_frames`
|
||||
- stable `vehicle_locations`
|
||||
- stable `vehicle_mileage_points`
|
||||
|
||||
- [ ] **Step 2: Implement writer**
|
||||
|
||||
Rules:
|
||||
|
||||
- `AppendRawFrame` always writes one raw row.
|
||||
- `AppendLocation` writes only when longitude and latitude exist.
|
||||
- `AppendMileagePoint` writes only when `total_mileage_km` exists.
|
||||
- child table name is deterministic hash of `protocol + vehicle_key`.
|
||||
- escape tag values.
|
||||
|
||||
- [ ] **Step 3: Use TDengine official driver**
|
||||
|
||||
Import WebSocket driver in command:
|
||||
|
||||
```go
|
||||
import _ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
```
|
||||
|
||||
Default driver name:
|
||||
|
||||
```text
|
||||
TDENGINE_DRIVER=taosWS
|
||||
```
|
||||
|
||||
Short-term compatibility:
|
||||
|
||||
```text
|
||||
TDENGINE_DRIVER=taosSql
|
||||
```
|
||||
|
||||
Only use `taosSql` when ECS TDengine WebSocket is not available.
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/history
|
||||
go build ./cmd/history-writer
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Implement MySQL Daily Metric Writer
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/stats/schema.go`
|
||||
- Create: `go/vehicle-gateway/internal/stats/daily_metric.go`
|
||||
- Create: `go/vehicle-gateway/internal/stats/daily_metric_test.go`
|
||||
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||
|
||||
- [ ] **Step 1: Add schema bootstrap**
|
||||
|
||||
Implement `vehicle_daily_metric` schema from the design spec.
|
||||
|
||||
- [ ] **Step 2: Add metric derivation tests**
|
||||
|
||||
Test cases:
|
||||
|
||||
- no `total_mileage_km` produces no metric
|
||||
- one sample produces:
|
||||
- `daily_mileage_km = 0`
|
||||
- `daily_total_mileage_km = sample`
|
||||
- later larger sample updates:
|
||||
- `latest_total_mileage_km`
|
||||
- `daily_mileage_km`
|
||||
- `daily_total_mileage_km`
|
||||
- out-of-order smaller sample updates:
|
||||
- `first_total_mileage_km`
|
||||
- `daily_mileage_km`
|
||||
|
||||
- [ ] **Step 3: Implement MySQL upsert**
|
||||
|
||||
Use one table and one idempotent upsert:
|
||||
|
||||
```sql
|
||||
INSERT INTO vehicle_daily_metric
|
||||
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
first_total_mileage_km = LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)),
|
||||
latest_total_mileage_km = GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)),
|
||||
metric_value = CASE
|
||||
WHEN metric_key = 'daily_mileage_km'
|
||||
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
- LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
WHEN metric_key = 'daily_total_mileage_km'
|
||||
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
ELSE VALUES(metric_value)
|
||||
END,
|
||||
sample_count = sample_count + 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/stats
|
||||
go build ./cmd/stat-writer
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Implement Redis Realtime API
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/realtime/repository.go`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/repository_test.go`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/http.go`
|
||||
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||
|
||||
- [ ] **Step 1: Add repository contract**
|
||||
|
||||
Methods:
|
||||
|
||||
- `Update(ctx, envelope.FrameEnvelope) error`
|
||||
- `GetMerged(ctx, vin string) (Snapshot, error)`
|
||||
- `GetProtocol(ctx, vin string, protocol envelope.Protocol) (Snapshot, error)`
|
||||
- `IsOnline(ctx, vin string) (OnlineStatus, error)`
|
||||
|
||||
- [ ] **Step 2: Implement merge logic**
|
||||
|
||||
Rules:
|
||||
|
||||
- update `vehicle:latest:{vin}:{protocol}`
|
||||
- update `vehicle:latest:{vin}` with newest fields
|
||||
- update `vehicle:online:{vin}`
|
||||
- update sorted set `vehicle:last_seen`
|
||||
|
||||
- [ ] **Step 3: Implement HTTP API**
|
||||
|
||||
Routes:
|
||||
|
||||
- `GET /api/realtime/vehicles/{vin}`
|
||||
- `GET /api/realtime/vehicles/{vin}/online`
|
||||
- `GET /api/realtime/vehicles/{vin}/protocols/{protocol}`
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/realtime
|
||||
go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Wire Gateway Runtime
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/gateway/tcp_server.go`
|
||||
- Create: `go/vehicle-gateway/internal/gateway/mqtt_client.go`
|
||||
- Modify: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
|
||||
- [ ] **Step 1: Implement TCP server**
|
||||
|
||||
Rules:
|
||||
|
||||
- one goroutine per accepted connection
|
||||
- bounded max connections
|
||||
- read timeout and idle timeout
|
||||
- protocol-specific frame extractor
|
||||
- structured peer endpoint
|
||||
- graceful shutdown on SIGTERM
|
||||
|
||||
- [ ] **Step 2: Implement MQTT client**
|
||||
|
||||
Rules:
|
||||
|
||||
- connect with official production config from environment
|
||||
- subscribe configured topic list
|
||||
- convert each message into envelope
|
||||
- publish raw and unified events
|
||||
- reconnect with backoff
|
||||
|
||||
- [ ] **Step 3: Verify local JSON mode**
|
||||
|
||||
Run without Kafka:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
GB32960_TCP_ADDR=:132960 JT808_TCP_ADDR=:18080 go run ./cmd/gateway
|
||||
```
|
||||
|
||||
Expected: service starts and logs configured listeners.
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Docker and ECS Deployment
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/Dockerfile`
|
||||
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||
- Modify: `docs/operations/current-ecs-deployment.md`
|
||||
|
||||
- [ ] **Step 1: Add multi-stage Dockerfile**
|
||||
|
||||
Build all commands:
|
||||
|
||||
- `gateway`
|
||||
- `history-writer`
|
||||
- `stat-writer`
|
||||
- `realtime-api`
|
||||
|
||||
- [ ] **Step 2: Add Portainer compose**
|
||||
|
||||
Services:
|
||||
|
||||
- `go-vehicle-gateway`
|
||||
- `go-history-writer`
|
||||
- `go-stat-writer`
|
||||
- `go-realtime-api`
|
||||
|
||||
Each service must include:
|
||||
|
||||
- restart policy
|
||||
- memory limit
|
||||
- Kafka env
|
||||
- MySQL/TDengine/Redis env as needed
|
||||
- logging options
|
||||
|
||||
- [ ] **Step 3: Verify Linux build**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
Expected: all commands exit `0`.
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Production Verification
|
||||
|
||||
**Files:**
|
||||
- Create: `docs/operations/go-vehicle-gateway-verification.md`
|
||||
|
||||
- [ ] **Step 1: Record test commands**
|
||||
|
||||
Document commands to verify:
|
||||
|
||||
- gateway process health
|
||||
- Kafka topic consumption
|
||||
- TDengine row counts
|
||||
- MySQL daily metric rows
|
||||
- Redis realtime lookup
|
||||
|
||||
- [ ] **Step 2: Validate real traffic**
|
||||
|
||||
Evidence required:
|
||||
|
||||
- one real 32960 VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||
- one real JT808 phone/VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||
- one real Yutong MQTT VIN with RAW and Redis snapshot
|
||||
|
||||
- [ ] **Step 3: Keep old Java services until evidence is captured**
|
||||
|
||||
Only disable Java equivalents after the evidence file contains successful command outputs and timestamps.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review Checklist
|
||||
|
||||
- The plan creates one new Go module instead of extending scattered prototypes.
|
||||
- The plan covers GB32960, JT808, and Yutong MQTT ingress.
|
||||
- The plan covers Kafka, TDengine, MySQL, and Redis.
|
||||
- The plan includes 32960 and 808 daily mileage and daily total mileage.
|
||||
- The plan includes local and ECS verification.
|
||||
- The plan does not restore Xinda Push.
|
||||
- The plan keeps Java services untouched until Go evidence exists.
|
||||
498
docs/superpowers/specs/2026-07-01-go-ingest-redesign-design.md
Normal file
498
docs/superpowers/specs/2026-07-01-go-ingest-redesign-design.md
Normal file
@@ -0,0 +1,498 @@
|
||||
# Go 车辆接入重构设计规格
|
||||
|
||||
## 目标
|
||||
|
||||
使用 Go 重新设计车辆数据接入运行面,让系统回到第一性原则:
|
||||
|
||||
- 接入层只负责可靠收包、协议解析、应答、身份解析和投递。
|
||||
- Kafka 是唯一在线消息通道。
|
||||
- TDengine 保存高吞吐时序明细和完整 RAW 解析 JSON。
|
||||
- MySQL 保存低频配置、身份映射和每日统计窄表。
|
||||
- Redis 保存可重建的准实时状态。
|
||||
- 32960、JT808、宇通 MQTT 使用同一套数据层接口,避免每个协议各自落库。
|
||||
|
||||
第一阶段只做生产链路的骨架和核心数据闭环:接收、解析、Kafka、TDengine、MySQL 统计、Redis 实时查询、ECS 验证。历史查询 API 和业务侧复杂分析放在后续阶段。
|
||||
|
||||
## 参考原则
|
||||
|
||||
本设计只吸收开源项目的边界思想,不复制外部实现代码。
|
||||
|
||||
- GB32960 参考 `DarkInno/gb32960-go-sdk`:连接管理、Handler、Forwarder、鉴权接口分离。
|
||||
- JT808 参考 `cuteLittleDevil/go-jt808`:高并发 Go 原生 TCP、协议核心小而可扩展、事件/适配器机制。
|
||||
- JT808 参考 `fakeyanss/jt808-server-go`:`FramePayload -> PacketData -> JT808Msg -> Reply` 的分层处理、Gateway 模式、保活和版本兼容。
|
||||
- TDengine 参考官方数据模型:按数据采集点建立 supertable,静态维度做 tags,明细数据按子表写入;Go 连接优先使用 WebSocket 驱动。
|
||||
|
||||
## 非目标
|
||||
|
||||
- 第一阶段不继续扩展 Java 服务。
|
||||
- 第一阶段不做统一大查询 API。
|
||||
- 第一阶段不恢复信达 Push。
|
||||
- 第一阶段不把所有协议字段展开成一张超宽 TDengine 表。
|
||||
- Redis 不作为历史事实源。
|
||||
- MySQL 不保存高频原始明细。
|
||||
|
||||
## 目标目录
|
||||
|
||||
新的 Go 运行面放在 `go/vehicle-gateway`,不继续沿用实验性的多 module 分散结构。现有 `go/ingest-edge` 和 `go/vehicle-state` 可以作为迁移素材,最终归并或删除。
|
||||
|
||||
```text
|
||||
go/vehicle-gateway/
|
||||
cmd/
|
||||
gateway/ # 协议接入进程:32960 TCP、JT808 TCP、宇通 MQTT
|
||||
history-writer/ # Kafka -> TDengine RAW/location/mileage points
|
||||
stat-writer/ # Kafka -> MySQL 每日统计窄表
|
||||
realtime-api/ # Redis 准实时查询 API
|
||||
internal/
|
||||
config/ # 环境变量、配置文件、校验
|
||||
gateway/ # TCP/MQTT 生命周期、连接限制、优雅停机
|
||||
protocol/
|
||||
gb32960/ # 32960 编解码
|
||||
jt808/ # 808 编解码
|
||||
yutongmqtt/ # 宇通 MQTT payload 解析
|
||||
identity/ # VIN、phone、device_id、plate 绑定
|
||||
envelope/ # 统一事件模型
|
||||
eventbus/ # Kafka producer/consumer
|
||||
history/ # TDengine adapter
|
||||
stats/ # 每日里程聚合
|
||||
realtime/ # Redis adapter 与快照合并
|
||||
observability/ # 日志、metrics、健康检查
|
||||
```
|
||||
|
||||
## 协议接入设计
|
||||
|
||||
### GB32960
|
||||
|
||||
职责:
|
||||
|
||||
- 监听 TCP `32960`。
|
||||
- 支持车辆登入、实时信息上报、补发信息上报、车辆登出、平台登入/登出、心跳、校时。
|
||||
- 正确返回协议应答。
|
||||
- 完整解析 2016 数据单元:
|
||||
- 整车数据
|
||||
- 驱动电机数据
|
||||
- 燃料电池数据
|
||||
- 发动机数据
|
||||
- 车辆位置数据
|
||||
- 极值数据
|
||||
- 报警数据
|
||||
- 可充电储能装置电压数据
|
||||
- 可充电储能装置温度数据
|
||||
- RAW 中保存完整 parsed JSON。
|
||||
- 关键字段标准化为统一 field:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `soc_percent`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `vehicle_status`
|
||||
- `charge_status`
|
||||
|
||||
### JT808
|
||||
|
||||
职责:
|
||||
|
||||
- 监听 TCP `808`。
|
||||
- 支持 2011/2013 基础兼容,预留 2019 版本标记。
|
||||
- 分层处理:
|
||||
- Frame:`0x7e` 包边界、转义、校验。
|
||||
- Packet:消息头、手机号 BCD、流水号、分包。
|
||||
- Message:注册、鉴权、心跳、注销、位置、批量位置、未知上行。
|
||||
- Reply:注册应答、平台通用应答。
|
||||
- 终端手机号按协议头 BCD 解析,内部同时保留原始 BCD、规范化 phone、去前导零 phone。
|
||||
- 注册帧和鉴权帧写入 808 registration 表。
|
||||
- 如果设备直接上报位置帧,也要走 identity 解析,尝试用 phone、device_id、plate 找到 VIN。
|
||||
- 0200 附加信息必须解析表 27:
|
||||
- `0x01` GPS 总里程,DWORD,单位 0.1 km。
|
||||
- 其他已知附加项结构化放入 `parsed_json.additional`。
|
||||
- 标准化字段:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `altitude_m`
|
||||
- `direction_deg`
|
||||
- `alarm_flag`
|
||||
- `status_flag`
|
||||
|
||||
### 宇通 MQTT
|
||||
|
||||
职责:
|
||||
|
||||
- 使用正式 MQTT 配置订阅生产 topic。
|
||||
- 接收 payload 后解析为统一 envelope。
|
||||
- 按 payload 中的车辆标识解析 VIN、车牌、设备号。
|
||||
- 保存完整 payload 和 parsed JSON。
|
||||
- 标准化字段向 32960/808 对齐:
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `soc_percent`
|
||||
- `vehicle_status`
|
||||
|
||||
## 统一 Envelope
|
||||
|
||||
所有协议接入后输出同一结构。
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "string",
|
||||
"trace_id": "string",
|
||||
"protocol": "GB32960|JT808|YUTONG_MQTT",
|
||||
"message_id": "string",
|
||||
"vin": "string",
|
||||
"vehicle_key": "string",
|
||||
"phone": "string",
|
||||
"device_id": "string",
|
||||
"plate": "string",
|
||||
"source_endpoint": "ip:port or mqtt://broker/topic",
|
||||
"event_time_ms": 0,
|
||||
"received_at_ms": 0,
|
||||
"raw_hex": "string",
|
||||
"raw_text": "string",
|
||||
"parsed": {},
|
||||
"fields": {
|
||||
"speed_kmh": 0,
|
||||
"total_mileage_km": 0,
|
||||
"longitude": 0,
|
||||
"latitude": 0
|
||||
},
|
||||
"parse_status": "OK|PARTIAL|BAD_FRAME",
|
||||
"parse_error": ""
|
||||
}
|
||||
```
|
||||
|
||||
规则:
|
||||
|
||||
- `event_id` 使用协议、设备标识、消息流水、事件时间、RAW hash 生成,保证幂等。
|
||||
- Kafka partition key 优先使用 VIN;没有 VIN 时使用 `protocol:phone/device_id`。
|
||||
- `parsed` 保存协议完整结构化字段。
|
||||
- `fields` 只保存跨协议核心字段,用于统计、位置和实时快照。
|
||||
|
||||
## Kafka Topic
|
||||
|
||||
生产只支持 Kafka。
|
||||
|
||||
| Topic | 内容 | Key |
|
||||
|---|---|---|
|
||||
| `vehicle.raw.gb32960.v1` | 32960 完整 RAW envelope | VIN 或 vehicle_key |
|
||||
| `vehicle.raw.jt808.v1` | 808 完整 RAW envelope | VIN 或 phone |
|
||||
| `vehicle.raw.yutong-mqtt.v1` | 宇通 MQTT 完整 RAW envelope | VIN 或 vehicle_key |
|
||||
| `vehicle.event.unified.v1` | 统一轻量事件,用于业务消费 | VIN 或 vehicle_key |
|
||||
|
||||
接入层必须先写 RAW topic,再写 unified event。后续消费者只依赖 Kafka,不直接依赖接入进程内存。
|
||||
|
||||
发布可靠性:
|
||||
|
||||
- Kafka 生产端必须开启同步写入,RAW 写成功后才允许写 unified event。
|
||||
- Kafka 写入必须支持可配置重试、单次写超时和短退避,默认值为 `KAFKA_PUBLISH_ATTEMPTS=3`、`KAFKA_PUBLISH_TIMEOUT_MS=3000`、`KAFKA_PUBLISH_BACKOFF_MS=100`。
|
||||
- gateway 支持本地磁盘 spool/WAL,配置 `KAFKA_SPOOL_DIR` 后启用;Kafka 长时间不可用时,发布失败的 envelope 先原子写入本地 JSON 文件。
|
||||
- spool 补发按文件名顺序执行,补发成功后删除文件;默认补发间隔由 `KAFKA_SPOOL_REPLAY_INTERVAL_MS=1000` 控制。
|
||||
- 如果 RAW 写 Kafka 失败并已落盘,同一个 event 的 unified event 不允许抢先写 Kafka,也必须进入 spool,确保恢复后按 RAW -> unified 顺序补发。
|
||||
- Kafka topic 不允许自动创建,topic 和分区数由部署脚本或运维初始化,避免生产拼写错误造成隐性分流。
|
||||
|
||||
## TDengine 数据库设计
|
||||
|
||||
如果现有 TDengine 库不符合目标,可以新建库并迁移。建议库名:
|
||||
|
||||
```sql
|
||||
CREATE DATABASE IF NOT EXISTS lingniu_vehicle_ts KEEP 7300 DURATION 10 BUFFER 256;
|
||||
```
|
||||
|
||||
### raw_frames
|
||||
|
||||
保存完整 RAW 和完整 parsed JSON。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
event_id NCHAR(64),
|
||||
message_id INT,
|
||||
event_time TIMESTAMP,
|
||||
received_at TIMESTAMP,
|
||||
raw_size_bytes INT,
|
||||
raw_hex BINARY(16374),
|
||||
raw_text BINARY(16374),
|
||||
parsed_json BINARY(16374),
|
||||
fields_json BINARY(4096),
|
||||
parse_status NCHAR(16),
|
||||
parse_error BINARY(1024),
|
||||
source_endpoint NCHAR(128)
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- 子表按 `protocol + vehicle_key hash` 创建。
|
||||
- RAW 查询优先按 tags 和时间范围过滤。
|
||||
- `parsed_json` 是完整协议字段;查询 API 可选择是否返回,避免默认大字段拖慢。
|
||||
|
||||
### vehicle_locations
|
||||
|
||||
只保存位置查询核心字段。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg INT,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
### vehicle_mileage_points
|
||||
|
||||
保存总里程采样点,用于回放重算和查询。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
total_mileage_km DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
## MySQL 数据库设计
|
||||
|
||||
MySQL 只保存配置、身份映射、808 注册鉴权和每日统计窄表。
|
||||
|
||||
### vehicle_identity_binding
|
||||
|
||||
用于把外部标识定位到 VIN。
|
||||
|
||||
```sql
|
||||
CREATE TABLE vehicle_identity_binding (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
plate VARCHAR(32) NULL,
|
||||
phone VARCHAR(32) NULL,
|
||||
device_id VARCHAR(64) NULL,
|
||||
remark VARCHAR(255) NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_vin (vin),
|
||||
KEY idx_plate (plate),
|
||||
KEY idx_phone (phone),
|
||||
KEY idx_device_id (device_id)
|
||||
);
|
||||
```
|
||||
|
||||
### jt808_registration
|
||||
|
||||
只记录 808 独有的注册、鉴权和在线身份信息。
|
||||
|
||||
```sql
|
||||
CREATE TABLE jt808_registration (
|
||||
phone VARCHAR(32) PRIMARY KEY,
|
||||
vin VARCHAR(32) NULL,
|
||||
plate VARCHAR(32) NULL,
|
||||
device_id VARCHAR(64) NULL,
|
||||
manufacturer_id VARCHAR(32) NULL,
|
||||
terminal_model VARCHAR(64) NULL,
|
||||
terminal_id VARCHAR(64) NULL,
|
||||
auth_code VARCHAR(128) NULL,
|
||||
source_endpoint VARCHAR(128) NULL,
|
||||
first_registered_at DATETIME NULL,
|
||||
latest_registered_at DATETIME NULL,
|
||||
latest_authed_at DATETIME NULL,
|
||||
latest_seen_at DATETIME NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
KEY idx_vin (vin),
|
||||
KEY idx_plate (plate),
|
||||
KEY idx_device_id (device_id)
|
||||
);
|
||||
```
|
||||
|
||||
### vehicle_daily_metric
|
||||
|
||||
32960 和 808 的每日里程、每日总里程都进入同一张窄表。
|
||||
|
||||
```sql
|
||||
CREATE TABLE vehicle_daily_metric (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,3) NOT NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
|
||||
KEY idx_stat_date (stat_date),
|
||||
KEY idx_protocol_metric (protocol, metric_key)
|
||||
);
|
||||
```
|
||||
|
||||
指标:
|
||||
|
||||
- `daily_mileage_km = max(total_mileage_km) - min(total_mileage_km)`
|
||||
- `daily_total_mileage_km = max(total_mileage_km)`
|
||||
|
||||
统计消费者可幂等重放。每个样本 upsert 时只维护当天 `min/max total_mileage_km`,不依赖进程内状态。
|
||||
|
||||
## Redis 实时数据设计
|
||||
|
||||
Redis 保存可从 Kafka 重建的数据。
|
||||
|
||||
| Key | 内容 |
|
||||
|---|---|
|
||||
| `vehicle:online:{vin}` | 在线状态、最后活跃时间、协议列表 |
|
||||
| `vehicle:latest:{vin}` | 合并后的 VIN 最新快照 |
|
||||
| `vehicle:latest:{vin}:{protocol}` | 单协议最新快照 |
|
||||
| `vehicle:last_seen` | VIN 到最后接收时间的 sorted set |
|
||||
|
||||
合并规则:
|
||||
|
||||
- 位置以最新 event_time 为准。
|
||||
- 总里程取最新上报值,不做递增假设。
|
||||
- 32960 多帧字段按 field timestamp 合并。
|
||||
- 808 和 MQTT 不覆盖其他协议独有字段。
|
||||
- Redis TTL 用于在线判断,不作为数据删除依据。
|
||||
|
||||
## 数据流
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
source["车辆 / 平台"] --> gateway["Go gateway"]
|
||||
gateway --> parser["protocol parser"]
|
||||
parser --> identity["identity resolver"]
|
||||
identity --> kafkaRaw["Kafka RAW topics"]
|
||||
identity --> kafkaEvent["Kafka unified event"]
|
||||
kafkaRaw --> history["history-writer"]
|
||||
kafkaRaw --> stats["stat-writer"]
|
||||
kafkaEvent --> realtime["realtime consumer"]
|
||||
history --> td["TDengine raw_frames / locations / mileage_points"]
|
||||
stats --> mysql["MySQL vehicle_daily_metric"]
|
||||
realtime --> redis["Redis latest state"]
|
||||
```
|
||||
|
||||
## 错误处理
|
||||
|
||||
- 协议坏帧仍写 RAW topic,`parse_status=BAD_FRAME`。
|
||||
- 可部分解析的帧写 `parse_status=PARTIAL`,保留 parse error。
|
||||
- Kafka 写失败时接入层先按配置重试;重试耗尽后不写 unified event,必须打错误日志和 metrics。
|
||||
- 启用 `KAFKA_SPOOL_DIR` 后,Kafka 重试耗尽会落本地 spool;如果 spool 写失败,才视为接入层最终失败。
|
||||
- TDengine 写失败不提交 Kafka offset。
|
||||
- MySQL 统计写失败不提交 Kafka offset。
|
||||
- Redis 写失败不影响历史和统计,但要通过 metrics 暴露。
|
||||
- 808 相同 phone 新连接时,旧连接应被替换并记录 latest_seen。
|
||||
|
||||
## 部署设计
|
||||
|
||||
生产 ECS 运行 Go 二进制或容器:
|
||||
|
||||
- `vehicle-gateway`
|
||||
- `vehicle-history-writer`
|
||||
- `vehicle-stat-writer`
|
||||
- `vehicle-realtime-api`
|
||||
|
||||
配置来源:
|
||||
|
||||
- 环境变量优先。
|
||||
- Nacos 可作为后续配置中心,但 Go 第一阶段必须能只靠环境变量启动,便于 Portainer 部署和故障恢复。
|
||||
|
||||
TDengine 连接:
|
||||
|
||||
- 优先 WebSocket `taosWS`。
|
||||
- 若当前 ECS 只开放原生端口,可短期兼容 `taosSql`,但代码配置必须显式标记为兼容模式。
|
||||
|
||||
## 验证计划
|
||||
|
||||
### 本地验证
|
||||
|
||||
- 用样例 808 报文验证:
|
||||
- phone BCD 解析。
|
||||
- 0200 经纬度、速度、方向、时间。
|
||||
- 表 27 `0x01` GPS 总里程。
|
||||
- 用真实 32960 报文验证:
|
||||
- 登录应答正确。
|
||||
- 实时帧和补发帧都能写 RAW。
|
||||
- 位置和总里程字段进入统一 fields。
|
||||
- 用宇通 MQTT payload 验证:
|
||||
- payload 完整保存。
|
||||
- 核心字段归一化。
|
||||
- 不连 Kafka 时可输出 JSON log。
|
||||
- 连接 Kafka 时 RAW topic 可消费到 envelope。
|
||||
|
||||
### ECS 验证
|
||||
|
||||
- 部署前先旁路接收或短窗口切流。
|
||||
- 验证端口:
|
||||
- `32960`
|
||||
- `808`
|
||||
- MQTT 订阅连接。
|
||||
- 验证 Kafka topic 有持续写入。
|
||||
- 验证 TDengine:
|
||||
- `raw_frames` 有三种协议数据。
|
||||
- `vehicle_locations` 能按 VIN、协议、时间分页查询。
|
||||
- `vehicle_mileage_points` 有 32960 和 808 总里程采样。
|
||||
- 验证 MySQL:
|
||||
- `vehicle_daily_metric` 有 32960 和 808 的 `daily_mileage_km`、`daily_total_mileage_km`。
|
||||
- 验证 Redis:
|
||||
- 查询 VIN 在线。
|
||||
- 查询 VIN 合并实时快照。
|
||||
- 查询 VIN 分协议实时快照。
|
||||
|
||||
## 验收标准
|
||||
|
||||
- Go gateway 能在 ECS 上接收真实 32960、808、宇通 MQTT 数据。
|
||||
- 三种协议 RAW 都进入 Kafka。
|
||||
- Kafka 短暂写失败时,gateway 按配置重试;单元测试覆盖首次失败后成功和重试耗尽返回错误。
|
||||
- Kafka 长时间不可用时,gateway 可把 RAW/unified 写入本地 spool;单元测试覆盖 RAW 已落盘时 unified 也落盘、恢复后按顺序补发并删除文件。
|
||||
- 三种协议 RAW 和 parsed JSON 都进入 TDengine。
|
||||
- 32960 和 808 的位置进入 `vehicle_locations`。
|
||||
- 32960 和 808 的总里程采样进入 `vehicle_mileage_points`。
|
||||
- 32960 和 808 的每日里程、每日总里程进入 MySQL 窄表。
|
||||
- Redis 可以查询 VIN 在线和实时数据。
|
||||
- 任一消费者宕机后可通过 Kafka offset 继续恢复。
|
||||
- 查询和统计验证使用 ECS 上的真实数据完成。
|
||||
|
||||
## 实施顺序
|
||||
|
||||
1. 创建 `go/vehicle-gateway` 单 module,迁移现有 Go 原型中可用的解析、Kafka、TDengine、MySQL、Redis 代码。
|
||||
2. 先写协议解析单元测试,覆盖 808 样例报文和 32960 核心字段。
|
||||
3. 完成统一 envelope 和 Kafka sink。
|
||||
4. 完成 TDengine schema bootstrap 与 history writer。
|
||||
5. 完成 MySQL schema bootstrap 与 stat writer。
|
||||
6. 完成 Redis realtime writer/API。
|
||||
7. 本地启动 gateway,用 ECS 转发流量验证。
|
||||
8. 构建 Linux 镜像,部署 ECS。
|
||||
9. 按验收标准逐项验证并记录证据。
|
||||
29
go/vehicle-gateway/Dockerfile
Normal file
29
go/vehicle-gateway/Dockerfile
Normal file
@@ -0,0 +1,29 @@
|
||||
FROM golang:1.26-bookworm AS build
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/gateway ./cmd/gateway \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/history-writer ./cmd/history-writer \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/stat-writer ./cmd/stat-writer \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/realtime-api ./cmd/realtime-api
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/gateway /app/gateway
|
||||
COPY --from=build /out/history-writer /app/history-writer
|
||||
COPY --from=build /out/stat-writer /app/stat-writer
|
||||
COPY --from=build /out/realtime-api /app/realtime-api
|
||||
|
||||
ENV TZ=Asia/Shanghai
|
||||
|
||||
EXPOSE 808 32960 20210
|
||||
|
||||
CMD ["/app/gateway"]
|
||||
223
go/vehicle-gateway/cmd/gateway/main.go
Normal file
223
go/vehicle-gateway/cmd/gateway/main.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/gateway"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-gateway")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
sink, err := buildSink(ctx, logger)
|
||||
if err != nil {
|
||||
logger.Error("build sink failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer sink.Close()
|
||||
resolver, closeResolver, err := buildIdentityResolver(ctx, logger)
|
||||
if err != nil {
|
||||
logger.Error("build identity resolver failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer closeResolver()
|
||||
|
||||
protocols := []gateway.TCPProtocol{
|
||||
{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: env("GB32960_TCP_ADDR", ":32960"),
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
Respond: gb32960.AutoResponse,
|
||||
},
|
||||
{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Addr: env("JT808_TCP_ADDR", ":808"),
|
||||
Extract: jt808.ExtractFrames,
|
||||
Parse: jt808.ParseFrame,
|
||||
Respond: jt808.NewAutoResponder(env("JT808_REGISTER_AUTH_CODE", "g7gps")).Respond,
|
||||
},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, len(protocols))
|
||||
for _, protocol := range protocols {
|
||||
server, err := gateway.NewTCPServer(gateway.TCPServerConfig{
|
||||
Protocol: protocol,
|
||||
Sink: sink,
|
||||
Resolver: resolver,
|
||||
Logger: logger,
|
||||
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
|
||||
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
|
||||
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := server.ListenAndServe(ctx); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}()
|
||||
}
|
||||
if envBool("YUTONG_MQTT_ENABLED", false) {
|
||||
client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{
|
||||
EndpointName: env("YUTONG_MQTT_ENDPOINT", env("YUTONG_MQTT_ENDPOINT_NAME", "yutong")),
|
||||
Broker: env("YUTONG_MQTT_URI", ""),
|
||||
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
|
||||
Username: env("YUTONG_MQTT_USERNAME", ""),
|
||||
Password: env("YUTONG_MQTT_PASSWORD", ""),
|
||||
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
|
||||
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
|
||||
CleanSession: envBool("YUTONG_MQTT_CLEAN_SESSION", false),
|
||||
KeepAlive: time.Duration(envInt("YUTONG_MQTT_KEEP_ALIVE_SECONDS", 20)) * time.Second,
|
||||
ConnectTimeout: time.Duration(envInt("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", 10)) * time.Second,
|
||||
TLSCACertPath: env("YUTONG_MQTT_TLS_CA_PEM", ""),
|
||||
TLSClientCertPath: env("YUTONG_MQTT_TLS_CLIENT_PEM", ""),
|
||||
TLSClientKeyPath: env("YUTONG_MQTT_TLS_CLIENT_KEY", ""),
|
||||
TLSHostnameVerification: envBool("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", true),
|
||||
Sink: sink,
|
||||
Resolver: resolver,
|
||||
Logger: logger,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("build yutong mqtt client failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := client.Start(ctx); err != nil {
|
||||
logger.Error("start yutong mqtt client failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
logger.Info("yutong mqtt client started")
|
||||
}
|
||||
|
||||
logger.Info("vehicle gateway started")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case err := <-errs:
|
||||
logger.Error("vehicle gateway listener failed", "error", err)
|
||||
stop()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.Resolver, func(), error) {
|
||||
dsn := env("IDENTITY_MYSQL_DSN", strings.TrimSpace(os.Getenv("MYSQL_DSN")))
|
||||
if strings.TrimSpace(dsn) == "" {
|
||||
logger.Warn("identity mysql dsn is empty; using noop identity resolver")
|
||||
return identity.NoopResolver{}, func() {}, nil
|
||||
}
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding")
|
||||
logger.Info("identity mysql resolver enabled", "table", table)
|
||||
return identity.NewMySQLResolver(db, table), func() { _ = db.Close() }, nil
|
||||
}
|
||||
|
||||
func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error) {
|
||||
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
|
||||
if len(brokers) == 0 {
|
||||
logger.Warn("KAFKA_BROKERS is empty; using log sink")
|
||||
return eventbus.NewLogSink(logger), nil
|
||||
}
|
||||
sink, err := eventbus.NewKafkaSink(eventbus.KafkaConfig{
|
||||
Brokers: brokers,
|
||||
RawTopics: map[envelope.Protocol]string{
|
||||
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1"),
|
||||
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1"),
|
||||
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1"),
|
||||
},
|
||||
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1"),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out eventbus.Sink = eventbus.NewRetryingSink(sink, eventbus.RetryConfig{
|
||||
Attempts: envInt("KAFKA_PUBLISH_ATTEMPTS", 3),
|
||||
Backoff: time.Duration(envInt("KAFKA_PUBLISH_BACKOFF_MS", 100)) * time.Millisecond,
|
||||
AttemptTimeout: time.Duration(envInt("KAFKA_PUBLISH_TIMEOUT_MS", 3000)) * time.Millisecond,
|
||||
})
|
||||
spoolDir := strings.TrimSpace(os.Getenv("KAFKA_SPOOL_DIR"))
|
||||
if spoolDir != "" {
|
||||
durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{Directory: spoolDir})
|
||||
interval := time.Duration(envInt("KAFKA_SPOOL_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond
|
||||
go durable.ReplayLoop(ctx, interval, func(err error) {
|
||||
logger.Warn("kafka spool replay failed", "error", err)
|
||||
})
|
||||
logger.Info("kafka durable spool enabled", "dir", spoolDir, "replay_interval_ms", interval.Milliseconds())
|
||||
out = durable
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
var out int
|
||||
for _, r := range value {
|
||||
if r < '0' || r > '9' {
|
||||
return fallback
|
||||
}
|
||||
out = out*10 + int(r-'0')
|
||||
}
|
||||
if out <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func envBool(key string, fallback bool) bool {
|
||||
value := strings.ToLower(strings.TrimSpace(os.Getenv(key)))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value == "1" || value == "true" || value == "yes" || value == "on"
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
123
go/vehicle-gateway/cmd/history-writer/main.go
Normal file
123
go/vehicle-gateway/cmd/history-writer/main.go
Normal file
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
_ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-history-writer")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
cfg := loadConfig()
|
||||
db, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
|
||||
if err != nil {
|
||||
logger.Error("tdengine open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
logger.Error("tdengine ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
writer := history.NewWriter(db)
|
||||
if cfg.EnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
|
||||
logger.Error("tdengine schema bootstrap failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
GroupID: cfg.KafkaGroup,
|
||||
GroupTopics: cfg.KafkaTopics,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
|
||||
logger.Info("history writer started",
|
||||
"driver", cfg.TDengineDriver,
|
||||
"group", cfg.KafkaGroup,
|
||||
"topics", strings.Join(cfg.KafkaTopics, ","))
|
||||
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := writer.AppendAll(ctx, env); err != nil {
|
||||
logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
KafkaBrokers []string
|
||||
KafkaTopics []string
|
||||
KafkaGroup string
|
||||
TDengineDriver string
|
||||
TDengineDSN string
|
||||
TDengineDatabase string
|
||||
EnsureSchema bool
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
return config{
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1")),
|
||||
KafkaGroup: env("KAFKA_GROUP", "go-history-writer"),
|
||||
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
|
||||
TDengineDSN: env("TDENGINE_DSN", ""),
|
||||
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
|
||||
EnsureSchema: env("TDENGINE_ENSURE_SCHEMA", "true") != "false",
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
198
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
198
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
@@ -0,0 +1,198 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/segmentio/kafka-go"
|
||||
_ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-realtime-api")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: env("REDIS_ADDR", "127.0.0.1:6379"),
|
||||
Username: env("REDIS_USERNAME", ""),
|
||||
Password: env("REDIS_PASSWORD", ""),
|
||||
DB: envInt("REDIS_DB", 0),
|
||||
})
|
||||
defer client.Close()
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
logger.Error("redis ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
repository := realtime.NewRepository(client, realtime.Config{
|
||||
OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", 600)) * time.Second,
|
||||
})
|
||||
if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
|
||||
go consumeKafka(ctx, logger, repository, brokers)
|
||||
} else {
|
||||
logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository))
|
||||
closeStats := func() {}
|
||||
if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" {
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
logger.Error("stats mysql open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
logger.Error("stats mysql ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
closeStats = func() { _ = db.Close() }
|
||||
mux.Handle("/api/stats/daily-metrics", stats.NewMetricHandler(stats.NewMetricRepository(db)))
|
||||
logger.Info("stats mysql query enabled")
|
||||
} else {
|
||||
mux.HandleFunc("/api/stats/daily-metrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "MYSQL_DSN is not configured"})
|
||||
})
|
||||
logger.Warn("MYSQL_DSN is empty; stats query api disabled")
|
||||
}
|
||||
defer closeStats()
|
||||
closeHistory := func() {}
|
||||
if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" {
|
||||
driver := env("TDENGINE_DRIVER", "taosWS")
|
||||
db, err := sql.Open(driver, dsn)
|
||||
if err != nil {
|
||||
logger.Error("history tdengine open failed", "driver", driver, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
logger.Error("history tdengine ping failed", "driver", driver, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
closeHistory = func() { _ = db.Close() }
|
||||
database := env("TDENGINE_DATABASE", history.DefaultDatabase)
|
||||
mux.Handle("/api/history/raw-frames", history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)))
|
||||
mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database)))
|
||||
mux.Handle("/api/history/mileage-points", history.NewMileagePointHandler(history.NewMileagePointRepository(db, database)))
|
||||
logger.Info("history query enabled", "driver", driver, "database", database)
|
||||
} else {
|
||||
historyUnavailable := func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"})
|
||||
}
|
||||
mux.HandleFunc("/api/history/raw-frames", historyUnavailable)
|
||||
mux.HandleFunc("/api/history/locations", historyUnavailable)
|
||||
mux.HandleFunc("/api/history/mileage-points", historyUnavailable)
|
||||
logger.Warn("TDENGINE_DSN is empty; history query api disabled")
|
||||
}
|
||||
defer closeHistory()
|
||||
|
||||
server := &http.Server{
|
||||
Addr: env("HTTP_ADDR", ":20210"),
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = server.Shutdown(shutdownCtx)
|
||||
}()
|
||||
|
||||
logger.Info("realtime api started", "addr", server.Addr)
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
logger.Error("http server failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func consumeKafka(ctx context.Context, logger interface {
|
||||
Info(string, ...any)
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, repository *realtime.Repository, brokers []string) {
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: brokers,
|
||||
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
|
||||
GroupTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.event.unified.v1")),
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
logger.Info("realtime kafka consumer started", "topics", env("KAFKA_TOPICS", "vehicle.event.unified.v1"))
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := repository.Update(ctx, env); err != nil {
|
||||
logger.Error("redis realtime update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
122
go/vehicle-gateway/cmd/stat-writer/main.go
Normal file
122
go/vehicle-gateway/cmd/stat-writer/main.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-stat-writer")
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
cfg := loadConfig()
|
||||
db, err := sql.Open("mysql", cfg.MySQLDSN)
|
||||
if err != nil {
|
||||
logger.Error("mysql open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
logger.Error("mysql ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
writer := stats.NewWriter(db, cfg.Location)
|
||||
if cfg.EnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx); err != nil {
|
||||
logger.Error("mysql schema bootstrap failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
GroupID: cfg.KafkaGroup,
|
||||
GroupTopics: cfg.KafkaTopics,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
})
|
||||
defer reader.Close()
|
||||
|
||||
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","))
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = reader.CommitMessages(ctx, message)
|
||||
continue
|
||||
}
|
||||
if err := writer.Append(ctx, env); err != nil {
|
||||
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
continue
|
||||
}
|
||||
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
KafkaBrokers []string
|
||||
KafkaTopics []string
|
||||
KafkaGroup string
|
||||
MySQLDSN string
|
||||
EnsureSchema bool
|
||||
Location *time.Location
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
loc, err := time.LoadLocation(env("LOCAL_TZ", "Asia/Shanghai"))
|
||||
if err != nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
return config{
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1")),
|
||||
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
|
||||
MySQLDSN: env("MYSQL_DSN", ""),
|
||||
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",
|
||||
Location: loc,
|
||||
}
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
30
go/vehicle-gateway/go.mod
Normal file
30
go/vehicle-gateway/go.mod
Normal file
@@ -0,0 +1,30 @@
|
||||
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/alicebob/miniredis/v2 v2.35.0
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/taosdata/driver-go/v3 v3.8.1
|
||||
golang.org/x/text v0.29.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.15.9 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
github.com/yuin/gopher-lua v1.1.1 // indirect
|
||||
golang.org/x/net v0.44.0 // indirect
|
||||
golang.org/x/sync v0.17.0 // indirect
|
||||
)
|
||||
73
go/vehicle-gateway/go.sum
Normal file
73
go/vehicle-gateway/go.sum
Normal file
@@ -0,0 +1,73 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE=
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
|
||||
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
||||
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/taosdata/driver-go/v3 v3.8.1 h1:kkd4ABsGiU+oXDbsw/sic985LKAvnpF9Gb/TEunTnLE=
|
||||
github.com/taosdata/driver-go/v3 v3.8.1/go.mod h1:S6OGOinfR0xxxaMGsvBi9cLkYxEIW1p6qqr8QJATTlg=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
|
||||
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
95
go/vehicle-gateway/internal/envelope/envelope.go
Normal file
95
go/vehicle-gateway/internal/envelope/envelope.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Protocol string
|
||||
|
||||
const (
|
||||
ProtocolGB32960 Protocol = "GB32960"
|
||||
ProtocolJT808 Protocol = "JT808"
|
||||
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||
)
|
||||
|
||||
type ParseStatus string
|
||||
|
||||
const (
|
||||
ParseOK ParseStatus = "OK"
|
||||
ParsePartial ParseStatus = "PARTIAL"
|
||||
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||
)
|
||||
|
||||
const (
|
||||
FieldSpeedKMH = "speed_kmh"
|
||||
FieldTotalMileageKM = "total_mileage_km"
|
||||
FieldLongitude = "longitude"
|
||||
FieldLatitude = "latitude"
|
||||
FieldSOCPercent = "soc_percent"
|
||||
)
|
||||
|
||||
type FrameEnvelope struct {
|
||||
EventID string `json:"event_id"`
|
||||
TraceID string `json:"trace_id"`
|
||||
Protocol Protocol `json:"protocol"`
|
||||
MessageID string `json:"message_id"`
|
||||
Sequence uint16 `json:"sequence"`
|
||||
VIN string `json:"vin,omitempty"`
|
||||
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
Parsed map[string]any `json:"parsed,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
ParseStatus ParseStatus `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) VehicleKey() string {
|
||||
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
return string(e.Protocol) + ":unknown"
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) StableEventID() string {
|
||||
if strings.TrimSpace(e.EventID) != "" {
|
||||
return e.EventID
|
||||
}
|
||||
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||
sum := sha256.Sum256([]byte(input))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) KafkaKey() []byte {
|
||||
return []byte(e.VehicleKey())
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||
if e.EventID == "" {
|
||||
e.EventID = e.StableEventID()
|
||||
}
|
||||
if e.ParseStatus == "" {
|
||||
e.ParseStatus = ParseOK
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
55
go/vehicle-gateway/internal/envelope/envelope_test.go
Normal file
55
go/vehicle-gateway/internal/envelope/envelope_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||
e := FrameEnvelope{
|
||||
Protocol: ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013307795425",
|
||||
Sequence: 1,
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7e02000000ff7e",
|
||||
}
|
||||
a := e.StableEventID()
|
||||
b := e.StableEventID()
|
||||
if a == "" || a != b {
|
||||
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeMarshalDefaults(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolGB32960, VIN: "LNBVIN00000000002", MessageID: "0x02"}
|
||||
payload, err := e.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalJSONBytes() error = %v", err)
|
||||
}
|
||||
var decoded FrameEnvelope
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if decoded.EventID == "" {
|
||||
t.Fatal("event id should be filled")
|
||||
}
|
||||
if decoded.ParseStatus != ParseOK {
|
||||
t.Fatalf("parse status = %q", decoded.ParseStatus)
|
||||
}
|
||||
}
|
||||
187
go/vehicle-gateway/internal/eventbus/durable_sink.go
Normal file
187
go/vehicle-gateway/internal/eventbus/durable_sink.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type DurableConfig struct {
|
||||
Directory string
|
||||
}
|
||||
|
||||
type DurableSink struct {
|
||||
delegate Sink
|
||||
dir string
|
||||
|
||||
mu sync.Mutex
|
||||
seq uint64
|
||||
rawPending map[string]struct{}
|
||||
}
|
||||
|
||||
type durableRecord struct {
|
||||
Kind string `json:"kind"`
|
||||
Envelope envelope.FrameEnvelope `json:"envelope"`
|
||||
}
|
||||
|
||||
func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
|
||||
if delegate == nil {
|
||||
panic("durable delegate sink must not be nil")
|
||||
}
|
||||
return &DurableSink{
|
||||
delegate: delegate,
|
||||
dir: strings.TrimSpace(cfg.Directory),
|
||||
rawPending: map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if err := s.delegate.PublishRaw(ctx, env); err == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.spool("raw", env); err != nil {
|
||||
return err
|
||||
}
|
||||
s.markRawPending(env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DurableSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if s.isRawPending(env) {
|
||||
return s.spool("unified", env)
|
||||
}
|
||||
if err := s.delegate.PublishUnified(ctx, env); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.spool("unified", env)
|
||||
}
|
||||
|
||||
func (s *DurableSink) ReplayOnce(ctx context.Context) error {
|
||||
files, err := filepath.Glob(filepath.Join(s.dir, "*.json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(files)
|
||||
for _, file := range files {
|
||||
record, err := readDurableRecord(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.publishRecord(ctx, record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(file); err != nil {
|
||||
return err
|
||||
}
|
||||
if record.Kind == "raw" {
|
||||
s.clearRawPending(record.Envelope)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DurableSink) ReplayLoop(ctx context.Context, interval time.Duration, onError func(error)) {
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := s.ReplayOnce(ctx); err != nil && onError != nil {
|
||||
onError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) Close() error {
|
||||
return s.delegate.Close()
|
||||
}
|
||||
|
||||
func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) error {
|
||||
switch record.Kind {
|
||||
case "raw":
|
||||
return s.delegate.PublishRaw(ctx, record.Envelope)
|
||||
case "unified":
|
||||
return s.delegate.PublishUnified(ctx, record.Envelope)
|
||||
default:
|
||||
return fmt.Errorf("unknown durable record kind %q", record.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
|
||||
if s.dir == "" {
|
||||
return fmt.Errorf("durable spool directory is empty")
|
||||
}
|
||||
if env.EventID == "" {
|
||||
env.EventID = env.StableEventID()
|
||||
}
|
||||
if env.ParseStatus == "" {
|
||||
env.ParseStatus = envelope.ParseOK
|
||||
}
|
||||
if err := os.MkdirAll(s.dir, 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := s.nextFileName(env, kind)
|
||||
path := filepath.Join(s.dir, name)
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, payload, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.seq++
|
||||
eventID := env.StableEventID()
|
||||
if len(eventID) > 12 {
|
||||
eventID = eventID[:12]
|
||||
}
|
||||
return fmt.Sprintf("%020d-%06d-%s-%s.json", time.Now().UnixNano(), s.seq, kind, eventID)
|
||||
}
|
||||
|
||||
func (s *DurableSink) markRawPending(env envelope.FrameEnvelope) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.rawPending[env.StableEventID()] = struct{}{}
|
||||
}
|
||||
|
||||
func (s *DurableSink) clearRawPending(env envelope.FrameEnvelope) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.rawPending, env.StableEventID())
|
||||
}
|
||||
|
||||
func (s *DurableSink) isRawPending(env envelope.FrameEnvelope) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, ok := s.rawPending[env.StableEventID()]
|
||||
return ok
|
||||
}
|
||||
|
||||
func readDurableRecord(path string) (durableRecord, error) {
|
||||
var record durableRecord
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return record, err
|
||||
}
|
||||
return record, json.Unmarshal(payload, &record)
|
||||
}
|
||||
153
go/vehicle-gateway/internal/eventbus/durable_sink_test.go
Normal file
153
go/vehicle-gateway/internal/eventbus/durable_sink_test.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
|
||||
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
|
||||
env := durableTestEnvelope()
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
|
||||
if delegate.rawCalls != 1 {
|
||||
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
|
||||
}
|
||||
if delegate.unifiedCalls != 0 {
|
||||
t.Fatalf("unified should not be delegated while raw is spooled, calls = %d", delegate.unifiedCalls)
|
||||
}
|
||||
files := spoolFiles(t, dir)
|
||||
if len(files) != 2 {
|
||||
t.Fatalf("spool files = %d, want 2: %#v", len(files), files)
|
||||
}
|
||||
assertSpoolKind(t, files[0], "raw")
|
||||
assertSpoolKind(t, files[1], "unified")
|
||||
}
|
||||
|
||||
func TestDurableSinkReplayPublishesInFileOrderAndDeletesFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
|
||||
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
|
||||
env := durableTestEnvelope()
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
delegate.rawErrors = nil
|
||||
|
||||
if err := sink.ReplayOnce(context.Background()); err != nil {
|
||||
t.Fatalf("ReplayOnce() error = %v", err)
|
||||
}
|
||||
|
||||
got := delegate.calls
|
||||
want := []string{"raw", "raw", "unified"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("calls = %#v, want %#v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("calls[%d] = %q, want %q; all=%#v", i, got[i], want[i], got)
|
||||
}
|
||||
}
|
||||
if files := spoolFiles(t, dir); len(files) != 0 {
|
||||
t.Fatalf("spool files after replay = %#v, want none", files)
|
||||
}
|
||||
}
|
||||
|
||||
func durableTestEnvelope() envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013079963379",
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
Sequence: 183,
|
||||
EventTimeMS: 1782903940000,
|
||||
ReceivedAtMS: 1782917751903,
|
||||
RawHex: "0200002201307996337900B7",
|
||||
}
|
||||
}
|
||||
|
||||
func spoolFiles(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
matches, err := filepath.Glob(filepath.Join(dir, "*.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob spool files: %v", err)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
func assertSpoolKind(t *testing.T, path string, kind string) {
|
||||
t.Helper()
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read spool file: %v", err)
|
||||
}
|
||||
if !containsString(string(payload), `"kind":"`+kind+`"`) {
|
||||
t.Fatalf("spool file %s payload = %s, want kind %q", path, payload, kind)
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(value string, needle string) bool {
|
||||
return len(needle) == 0 || (len(value) >= len(needle) && indexString(value, needle) >= 0)
|
||||
}
|
||||
|
||||
func indexString(value string, needle string) int {
|
||||
for i := 0; i+len(needle) <= len(value); i++ {
|
||||
if value[i:i+len(needle)] == needle {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
var errSpoolTest = errors.New("delegate unavailable")
|
||||
|
||||
type scriptedSink struct {
|
||||
rawErrors []error
|
||||
unifiedErrors []error
|
||||
rawCalls int
|
||||
unifiedCalls int
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (s *scriptedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
s.rawCalls++
|
||||
s.calls = append(s.calls, "raw")
|
||||
if len(s.rawErrors) > 0 {
|
||||
err := s.rawErrors[0]
|
||||
s.rawErrors = s.rawErrors[1:]
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scriptedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
s.unifiedCalls++
|
||||
s.calls = append(s.calls, "unified")
|
||||
if len(s.unifiedErrors) > 0 {
|
||||
err := s.unifiedErrors[0]
|
||||
s.unifiedErrors = s.unifiedErrors[1:]
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scriptedSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
94
go/vehicle-gateway/internal/eventbus/kafka_sink.go
Normal file
94
go/vehicle-gateway/internal/eventbus/kafka_sink.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Sink interface {
|
||||
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type KafkaSink struct {
|
||||
writer kafkaWriter
|
||||
rawTopics map[envelope.Protocol]string
|
||||
unifiedTopic string
|
||||
}
|
||||
|
||||
type kafkaWriter interface {
|
||||
WriteMessages(context.Context, ...kafka.Message) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type KafkaConfig struct {
|
||||
Brokers []string
|
||||
RawTopics map[envelope.Protocol]string
|
||||
UnifiedTopic string
|
||||
}
|
||||
|
||||
func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) {
|
||||
if len(cfg.Brokers) == 0 {
|
||||
return nil, errors.New("kafka brokers are required")
|
||||
}
|
||||
return newKafkaSinkWithWriter(&kafka.Writer{
|
||||
Addr: kafka.TCP(cfg.Brokers...),
|
||||
Balancer: &kafka.Hash{},
|
||||
AllowAutoTopicCreation: false,
|
||||
}, cfg), nil
|
||||
}
|
||||
|
||||
func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
|
||||
rawTopics := map[envelope.Protocol]string{
|
||||
envelope.ProtocolGB32960: "vehicle.raw.gb32960.v1",
|
||||
envelope.ProtocolJT808: "vehicle.raw.jt808.v1",
|
||||
envelope.ProtocolYutongMQTT: "vehicle.raw.yutong-mqtt.v1",
|
||||
}
|
||||
for protocol, topic := range cfg.RawTopics {
|
||||
if topic != "" {
|
||||
rawTopics[protocol] = topic
|
||||
}
|
||||
}
|
||||
unifiedTopic := cfg.UnifiedTopic
|
||||
if unifiedTopic == "" {
|
||||
unifiedTopic = "vehicle.event.unified.v1"
|
||||
}
|
||||
return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic}
|
||||
}
|
||||
|
||||
func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
topic, ok := s.rawTopics[env.Protocol]
|
||||
if !ok || topic == "" {
|
||||
return fmt.Errorf("raw topic not configured for protocol %s", env.Protocol)
|
||||
}
|
||||
return s.publish(ctx, topic, env)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.publish(ctx, s.unifiedTopic, env)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) Close() error {
|
||||
if s == nil || s.writer == nil {
|
||||
return nil
|
||||
}
|
||||
return s.writer.Close()
|
||||
}
|
||||
|
||||
func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.writer.WriteMessages(ctx, kafka.Message{
|
||||
Topic: topic,
|
||||
Key: env.KafkaKey(),
|
||||
Value: payload,
|
||||
})
|
||||
}
|
||||
77
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
Normal file
77
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestKafkaSinkRoutesRawTopics(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "013307795425", MessageID: "0x0200"}
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if len(writer.messages) != 1 {
|
||||
t.Fatalf("messages = %d", len(writer.messages))
|
||||
}
|
||||
msg := writer.messages[0]
|
||||
if msg.Topic != "vehicle.raw.jt808.v1" {
|
||||
t.Fatalf("topic = %q", msg.Topic)
|
||||
}
|
||||
if string(msg.Key) != "JT808:013307795425" {
|
||||
t.Fatalf("key = %q", string(msg.Key))
|
||||
}
|
||||
var decoded envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(msg.Value, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if decoded.EventID == "" || decoded.ParseStatus != envelope.ParseOK {
|
||||
t.Fatalf("unexpected payload defaults: %#v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKafkaSinkRoutesUnifiedTopic(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{UnifiedTopic: "custom.unified"})
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000001", MessageID: "0x02"}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
if writer.messages[0].Topic != "custom.unified" {
|
||||
t.Fatalf("topic = %q", writer.messages[0].Topic)
|
||||
}
|
||||
if string(writer.messages[0].Key) != "LNBVIN00000000001" {
|
||||
t.Fatalf("key = %q", string(writer.messages[0].Key))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKafkaSinkRejectsUnknownProtocol(t *testing.T) {
|
||||
writer := &recordingWriter{}
|
||||
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")})
|
||||
if err == nil {
|
||||
t.Fatal("expected unknown protocol error")
|
||||
}
|
||||
}
|
||||
|
||||
type recordingWriter struct {
|
||||
messages []kafka.Message
|
||||
}
|
||||
|
||||
func (w *recordingWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
|
||||
w.messages = append(w.messages, messages...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *recordingWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
33
go/vehicle-gateway/internal/eventbus/log_sink.go
Normal file
33
go/vehicle-gateway/internal/eventbus/log_sink.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type LogSink struct {
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewLogSink(logger *slog.Logger) *LogSink {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &LogSink{logger: logger}
|
||||
}
|
||||
|
||||
func (s *LogSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.logger.Info("raw envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID, "status", env.ParseStatus)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LogSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.logger.Info("unified envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LogSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
80
go/vehicle-gateway/internal/eventbus/retry_sink.go
Normal file
80
go/vehicle-gateway/internal/eventbus/retry_sink.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type RetryConfig struct {
|
||||
Attempts int
|
||||
Backoff time.Duration
|
||||
AttemptTimeout time.Duration
|
||||
}
|
||||
|
||||
type RetryingSink struct {
|
||||
delegate Sink
|
||||
cfg RetryConfig
|
||||
}
|
||||
|
||||
func NewRetryingSink(delegate Sink, cfg RetryConfig) *RetryingSink {
|
||||
if delegate == nil {
|
||||
panic("retry delegate sink must not be nil")
|
||||
}
|
||||
if cfg.Attempts <= 0 {
|
||||
cfg.Attempts = 1
|
||||
}
|
||||
return &RetryingSink{delegate: delegate, cfg: cfg}
|
||||
}
|
||||
|
||||
func (s *RetryingSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.withRetry(ctx, func(attemptCtx context.Context) error {
|
||||
return s.delegate.PublishRaw(attemptCtx, env)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *RetryingSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
return s.withRetry(ctx, func(attemptCtx context.Context) error {
|
||||
return s.delegate.PublishUnified(attemptCtx, env)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *RetryingSink) Close() error {
|
||||
return s.delegate.Close()
|
||||
}
|
||||
|
||||
func (s *RetryingSink) withRetry(ctx context.Context, publish func(context.Context) error) error {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= s.cfg.Attempts; attempt++ {
|
||||
attemptCtx := ctx
|
||||
cancel := func() {}
|
||||
if s.cfg.AttemptTimeout > 0 {
|
||||
attemptCtx, cancel = context.WithTimeout(ctx, s.cfg.AttemptTimeout)
|
||||
}
|
||||
err := publish(attemptCtx)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
lastErr = err
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if attempt == s.cfg.Attempts {
|
||||
break
|
||||
}
|
||||
if s.cfg.Backoff > 0 {
|
||||
timer := time.NewTimer(s.cfg.Backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
80
go/vehicle-gateway/internal/eventbus/retry_sink_test.go
Normal file
80
go/vehicle-gateway/internal/eventbus/retry_sink_test.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestRetryingSinkRetriesRawUntilSuccess(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 1}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 3})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if delegate.rawCalls != 2 {
|
||||
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryingSinkReturnsLastErrorAfterAttempts(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 3}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 2})
|
||||
|
||||
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if !errors.Is(err, errFlakyPublish) {
|
||||
t.Fatalf("PublishRaw() error = %v, want %v", err, errFlakyPublish)
|
||||
}
|
||||
if delegate.rawCalls != 2 {
|
||||
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryingSinkStopsWhenContextCanceledDuringBackoff(t *testing.T) {
|
||||
delegate := &flakySink{rawFailures: 10}
|
||||
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 5, Backoff: time.Minute})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
err := sink.PublishRaw(ctx, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("PublishRaw() error = %v, want context canceled", err)
|
||||
}
|
||||
if delegate.rawCalls != 1 {
|
||||
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
|
||||
}
|
||||
}
|
||||
|
||||
var errFlakyPublish = errors.New("publish failed")
|
||||
|
||||
type flakySink struct {
|
||||
rawFailures int
|
||||
unifiedFailures int
|
||||
rawCalls int
|
||||
unifiedCalls int
|
||||
}
|
||||
|
||||
func (s *flakySink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
s.rawCalls++
|
||||
if s.rawCalls <= s.rawFailures {
|
||||
return errFlakyPublish
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *flakySink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
s.unifiedCalls++
|
||||
if s.unifiedCalls <= s.unifiedFailures {
|
||||
return errFlakyPublish
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *flakySink) Close() error {
|
||||
return nil
|
||||
}
|
||||
218
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
218
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
@@ -0,0 +1,218 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
|
||||
)
|
||||
|
||||
type MQTTClientConfig struct {
|
||||
EndpointName string
|
||||
Broker string
|
||||
ClientID string
|
||||
Username string
|
||||
Password string
|
||||
Topics []string
|
||||
QoS byte
|
||||
CleanSession bool
|
||||
KeepAlive time.Duration
|
||||
ConnectTimeout time.Duration
|
||||
TLSCACertPath string
|
||||
TLSClientCertPath string
|
||||
TLSClientKeyPath string
|
||||
TLSHostnameVerification bool
|
||||
Sink eventbus.Sink
|
||||
Resolver identity.Resolver
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
type MQTTClient struct {
|
||||
cfg MQTTClientConfig
|
||||
client mqtt.Client
|
||||
}
|
||||
|
||||
func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
|
||||
if strings.TrimSpace(cfg.Broker) == "" {
|
||||
return nil, errors.New("mqtt broker is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ClientID) == "" {
|
||||
return nil, errors.New("mqtt client id is required")
|
||||
}
|
||||
if len(cfg.Topics) == 0 {
|
||||
return nil, errors.New("mqtt topics are required")
|
||||
}
|
||||
if cfg.Sink == nil {
|
||||
return nil, errors.New("sink is required")
|
||||
}
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
if cfg.Resolver == nil {
|
||||
cfg.Resolver = identity.NoopResolver{}
|
||||
}
|
||||
if cfg.EndpointName == "" {
|
||||
cfg.EndpointName = "yutong"
|
||||
}
|
||||
return &MQTTClient{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) Start(ctx context.Context) error {
|
||||
opts, err := c.buildOptions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.client = mqtt.NewClient(opts)
|
||||
token := c.client.Connect()
|
||||
if token.Wait() && token.Error() != nil {
|
||||
return token.Error()
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
if c.client != nil && c.client.IsConnected() {
|
||||
c.client.Disconnect(250)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) buildOptions(ctx context.Context) (*mqtt.ClientOptions, error) {
|
||||
keepAlive := c.cfg.KeepAlive
|
||||
if keepAlive <= 0 {
|
||||
keepAlive = 20 * time.Second
|
||||
}
|
||||
connectTimeout := c.cfg.ConnectTimeout
|
||||
if connectTimeout <= 0 {
|
||||
connectTimeout = 10 * time.Second
|
||||
}
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker(c.cfg.Broker).
|
||||
SetClientID(c.cfg.ClientID).
|
||||
SetUsername(c.cfg.Username).
|
||||
SetPassword(c.cfg.Password).
|
||||
SetCleanSession(c.cfg.CleanSession).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectRetry(true).
|
||||
SetConnectRetryInterval(5 * time.Second).
|
||||
SetResumeSubs(true).
|
||||
SetOrderMatters(false).
|
||||
SetKeepAlive(keepAlive).
|
||||
SetConnectTimeout(connectTimeout)
|
||||
|
||||
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
|
||||
c.handleMessage(ctx, message.Topic(), message.Payload())
|
||||
})
|
||||
opts.OnConnect = func(client mqtt.Client) {
|
||||
for _, topic := range c.cfg.Topics {
|
||||
token := client.Subscribe(topic, c.cfg.QoS, nil)
|
||||
if token.Wait() && token.Error() != nil {
|
||||
c.cfg.Logger.Error("mqtt subscribe failed", "broker", c.cfg.Broker, "topic", topic, "error", token.Error())
|
||||
continue
|
||||
}
|
||||
c.cfg.Logger.Info("mqtt subscribed", "broker", c.cfg.Broker, "topic", topic, "qos", c.cfg.QoS)
|
||||
}
|
||||
}
|
||||
opts.OnConnectionLost = func(_ mqtt.Client, err error) {
|
||||
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
|
||||
}
|
||||
|
||||
tlsConfig, err := c.buildTLSConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tlsConfig != nil {
|
||||
opts.SetTLSConfig(tlsConfig)
|
||||
}
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
|
||||
caPath := strings.TrimSpace(c.cfg.TLSCACertPath)
|
||||
certPath := strings.TrimSpace(c.cfg.TLSClientCertPath)
|
||||
keyPath := strings.TrimSpace(c.cfg.TLSClientKeyPath)
|
||||
if caPath == "" && certPath == "" && keyPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
config := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: !c.cfg.TLSHostnameVerification,
|
||||
}
|
||||
if caPath != "" {
|
||||
caPEM, err := os.ReadFile(caPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read mqtt ca certificate: %w", err)
|
||||
}
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("parse mqtt ca certificate %s", caPath)
|
||||
}
|
||||
config.RootCAs = roots
|
||||
}
|
||||
if certPath != "" || keyPath != "" {
|
||||
if certPath == "" || keyPath == "" {
|
||||
return nil, errors.New("mqtt client certificate and key must be configured together")
|
||||
}
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load mqtt client certificate: %w", err)
|
||||
}
|
||||
config.Certificates = []tls.Certificate{cert}
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := yutongmqtt.ParseMessage(c.cfg.EndpointName, topic, payload, receivedAtMS)
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
MessageID: "MQTT",
|
||||
VehicleKeyHint: "unknown",
|
||||
SourceEndpoint: "mqtt://" + c.cfg.EndpointName + topic,
|
||||
EventTimeMS: receivedAtMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawText: string(payload),
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(payload)),
|
||||
ParseStatus: envelope.ParseBadFrame,
|
||||
ParseError: err.Error(),
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
} else {
|
||||
resolved, resolveErr := c.cfg.Resolver.Resolve(ctx, env)
|
||||
if resolveErr != nil {
|
||||
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
|
||||
env.ParseStatus = envelope.ParsePartial
|
||||
} else {
|
||||
env = resolved
|
||||
}
|
||||
}
|
||||
if err := c.cfg.Sink.PublishRaw(ctx, env); err != nil {
|
||||
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return
|
||||
}
|
||||
if err := c.cfg.Sink.PublishUnified(ctx, env); err != nil {
|
||||
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||
}
|
||||
}
|
||||
183
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
183
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "tcp://127.0.0.1:1883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
|
||||
"device":"LTEST000000000001",
|
||||
"time":"20260413100000",
|
||||
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
|
||||
}`))
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
|
||||
t.Fatalf("unexpected raw envelope: %#v", sink.raw[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "tcp://127.0.0.1:1883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
client.handleMessage(context.Background(), "/ytforward/shln/bad", []byte("{bad-json"))
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "ssl://mqtt.example.test:8883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
QoS: 1,
|
||||
Sink: &recordingSink{},
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
TLSCACertPath: caPath,
|
||||
TLSClientCertPath: certPath,
|
||||
TLSClientKeyPath: keyPath,
|
||||
TLSHostnameVerification: false,
|
||||
CleanSession: true,
|
||||
KeepAlive: 20 * time.Second,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
opts, err := client.buildOptions(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("buildOptions() error = %v", err)
|
||||
}
|
||||
if opts.TLSConfig == nil {
|
||||
t.Fatal("TLSConfig is nil")
|
||||
}
|
||||
if opts.TLSConfig.RootCAs == nil {
|
||||
t.Fatal("RootCAs is nil")
|
||||
}
|
||||
if len(opts.TLSConfig.Certificates) != 1 {
|
||||
t.Fatalf("client certificates = %d, want 1", len(opts.TLSConfig.Certificates))
|
||||
}
|
||||
if !opts.TLSConfig.InsecureSkipVerify {
|
||||
t.Fatal("InsecureSkipVerify should be true when hostname verification is disabled")
|
||||
}
|
||||
if !opts.CleanSession {
|
||||
t.Fatal("CleanSession should be true")
|
||||
}
|
||||
if opts.Order {
|
||||
t.Fatal("OrderMatters should be false so MQTT network handling is not blocked by Kafka/DB work")
|
||||
}
|
||||
if !opts.ResumeSubs {
|
||||
t.Fatal("ResumeSubs should be true to avoid subscribe failures during reconnect churn")
|
||||
}
|
||||
if got := opts.KeepAlive; got != 20 {
|
||||
t.Fatalf("KeepAlive = %d, want 20", got)
|
||||
}
|
||||
if got := opts.ConnectTimeout; got != 10*time.Second {
|
||||
t.Fatalf("ConnectTimeout = %v, want 10s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestTLSMaterial(t *testing.T, dir string) (string, string, string) {
|
||||
t.Helper()
|
||||
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate ca key: %v", err)
|
||||
}
|
||||
caTemplate := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "test-ca"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
}
|
||||
caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
|
||||
if err != nil {
|
||||
t.Fatalf("create ca cert: %v", err)
|
||||
}
|
||||
clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate client key: %v", err)
|
||||
}
|
||||
clientTemplate := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(2),
|
||||
Subject: pkix.Name{CommonName: "test-client"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
||||
}
|
||||
clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey)
|
||||
if err != nil {
|
||||
t.Fatalf("create client cert: %v", err)
|
||||
}
|
||||
caPath := filepath.Join(dir, "ca.pem")
|
||||
certPath := filepath.Join(dir, "client.pem")
|
||||
keyPath := filepath.Join(dir, "client-key.pem")
|
||||
writePEM(t, caPath, "CERTIFICATE", caDER)
|
||||
writePEM(t, certPath, "CERTIFICATE", clientDER)
|
||||
keyDER := x509.MarshalPKCS1PrivateKey(clientKey)
|
||||
writePEM(t, keyPath, "RSA PRIVATE KEY", keyDER)
|
||||
return caPath, certPath, keyPath
|
||||
}
|
||||
|
||||
func writePEM(t *testing.T, path, typ string, der []byte) {
|
||||
t.Helper()
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create %s: %v", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err := pem.Encode(file, &pem.Block{Type: typ, Bytes: der}); err != nil {
|
||||
t.Fatalf("write pem %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
238
go/vehicle-gateway/internal/gateway/tcp_server.go
Normal file
238
go/vehicle-gateway/internal/gateway/tcp_server.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
)
|
||||
|
||||
type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
|
||||
|
||||
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
|
||||
|
||||
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
|
||||
|
||||
type TCPProtocol struct {
|
||||
Protocol envelope.Protocol
|
||||
Addr string
|
||||
Extract FrameExtractor
|
||||
Parse FrameParser
|
||||
Respond FrameResponder
|
||||
}
|
||||
|
||||
type TCPServer struct {
|
||||
protocol TCPProtocol
|
||||
sink eventbus.Sink
|
||||
resolver identity.Resolver
|
||||
logger *slog.Logger
|
||||
readBufferSize int
|
||||
idleTimeout time.Duration
|
||||
maxConnections int
|
||||
}
|
||||
|
||||
type TCPServerConfig struct {
|
||||
Protocol TCPProtocol
|
||||
Sink eventbus.Sink
|
||||
Resolver identity.Resolver
|
||||
Logger *slog.Logger
|
||||
ReadBufferSize int
|
||||
IdleTimeout time.Duration
|
||||
MaxConnections int
|
||||
}
|
||||
|
||||
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
|
||||
if cfg.Protocol.Protocol == "" {
|
||||
return nil, errors.New("protocol is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Protocol.Addr) == "" {
|
||||
return nil, errors.New("listen addr is required")
|
||||
}
|
||||
if cfg.Protocol.Extract == nil {
|
||||
return nil, errors.New("frame extractor is required")
|
||||
}
|
||||
if cfg.Protocol.Parse == nil {
|
||||
return nil, errors.New("frame parser is required")
|
||||
}
|
||||
if cfg.Sink == nil {
|
||||
return nil, errors.New("sink is required")
|
||||
}
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
if cfg.Resolver == nil {
|
||||
cfg.Resolver = identity.NoopResolver{}
|
||||
}
|
||||
if cfg.ReadBufferSize <= 0 {
|
||||
cfg.ReadBufferSize = 32 * 1024
|
||||
}
|
||||
if cfg.IdleTimeout <= 0 {
|
||||
cfg.IdleTimeout = 2 * time.Minute
|
||||
}
|
||||
if cfg.MaxConnections <= 0 {
|
||||
cfg.MaxConnections = 10_000
|
||||
}
|
||||
return &TCPServer{
|
||||
protocol: cfg.Protocol,
|
||||
sink: cfg.Sink,
|
||||
resolver: cfg.Resolver,
|
||||
logger: cfg.Logger,
|
||||
readBufferSize: cfg.ReadBufferSize,
|
||||
idleTimeout: cfg.IdleTimeout,
|
||||
maxConnections: cfg.MaxConnections,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *TCPServer) ListenAndServe(ctx context.Context) error {
|
||||
var lc net.ListenConfig
|
||||
listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
_ = listener.Close()
|
||||
}()
|
||||
|
||||
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
|
||||
sem := make(chan struct{}, s.maxConnections)
|
||||
var wg sync.WaitGroup
|
||||
defer wg.Wait()
|
||||
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
s.handleConnection(ctx, conn)
|
||||
}()
|
||||
default:
|
||||
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
||||
defer conn.Close()
|
||||
source := conn.RemoteAddr().String()
|
||||
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
|
||||
log.Info("tcp connection opened")
|
||||
defer log.Info("tcp connection closed")
|
||||
|
||||
readBuffer := make([]byte, s.readBufferSize)
|
||||
var pending []byte
|
||||
for {
|
||||
_ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout))
|
||||
n, err := conn.Read(readBuffer)
|
||||
if n > 0 {
|
||||
pending = append(pending, readBuffer[:n]...)
|
||||
frames, remainder, extractErr := s.protocol.Extract(pending)
|
||||
if extractErr != nil {
|
||||
log.Warn("frame extraction failed", "error", extractErr)
|
||||
return
|
||||
}
|
||||
pending = remainder
|
||||
for _, frame := range frames {
|
||||
s.handleFrame(ctx, conn, frame, source)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
log.Warn("tcp connection idle timeout")
|
||||
return
|
||||
}
|
||||
log.Warn("tcp read failed", "error", err)
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string) {
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := s.protocol.Parse(raw, receivedAtMS, source)
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: s.protocol.Protocol,
|
||||
SourceEndpoint: source,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
EventTimeMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
ParseStatus: envelope.ParseBadFrame,
|
||||
ParseError: err.Error(),
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
} else {
|
||||
resolved, resolveErr := s.resolver.Resolve(ctx, env)
|
||||
if resolveErr != nil {
|
||||
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
|
||||
env.ParseStatus = envelope.ParsePartial
|
||||
} else {
|
||||
env = resolved
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.sink.PublishRaw(ctx, env); err != nil {
|
||||
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return
|
||||
}
|
||||
if err := s.sink.PublishUnified(ctx, env); err != nil {
|
||||
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if s.protocol.Respond == nil {
|
||||
return
|
||||
}
|
||||
response, ok, err := s.protocol.Respond(raw, env)
|
||||
if err != nil {
|
||||
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if !ok || len(response) == 0 {
|
||||
return
|
||||
}
|
||||
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := conn.Write(response); err != nil {
|
||||
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (p TCPProtocol) String() string {
|
||||
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
|
||||
}
|
||||
177
go/vehicle-gateway/internal/gateway/tcp_server_test.go
Normal file
177
go/vehicle-gateway/internal/gateway/tcp_server_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||
)
|
||||
|
||||
func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
|
||||
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Addr: ":0",
|
||||
Extract: jt808.ExtractFrames,
|
||||
Parse: jt808.ParseFrame,
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(frame); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].Phone != "013307795425" {
|
||||
t.Fatalf("phone = %q", sink.raw[0].Phone)
|
||||
}
|
||||
if sink.raw[0].Fields[envelope.FieldTotalMileageKM] != 10241.2 {
|
||||
t.Fatalf("total mileage = %#v", sink.raw[0].Fields[envelope.FieldTotalMileageKM])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
|
||||
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
good[len(good)-1] ^= 0xff
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(good); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||
}
|
||||
if sink.raw[0].ParseError == "" {
|
||||
t.Fatal("parse error should be recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
|
||||
frame := buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
|
||||
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
return []byte("ACK"), true, nil
|
||||
},
|
||||
}, sink)
|
||||
|
||||
client, done := runPipe(t, server)
|
||||
if _, err := client.Write(frame); err != nil {
|
||||
t.Fatalf("client.Write() error = %v", err)
|
||||
}
|
||||
buf := make([]byte, 3)
|
||||
if _, err := io.ReadFull(client, buf); err != nil {
|
||||
t.Fatalf("read response error = %v", err)
|
||||
}
|
||||
if string(buf) != "ACK" {
|
||||
t.Fatalf("response = %q", string(buf))
|
||||
}
|
||||
_ = client.Close()
|
||||
<-done
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer {
|
||||
t.Helper()
|
||||
server, err := NewTCPServer(TCPServerConfig{
|
||||
Protocol: protocol,
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
ReadBufferSize: 1024,
|
||||
IdleTimeout: time.Second,
|
||||
MaxConnections: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewTCPServer() error = %v", err)
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
func runPipe(t *testing.T, server *TCPServer) (net.Conn, <-chan struct{}) {
|
||||
t.Helper()
|
||||
client, srv := net.Pipe()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
server.handleConnection(context.Background(), srv)
|
||||
}()
|
||||
return client, done
|
||||
}
|
||||
|
||||
type recordingSink struct {
|
||||
raw []envelope.FrameEnvelope
|
||||
unified []envelope.FrameEnvelope
|
||||
}
|
||||
|
||||
func (s *recordingSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.raw = append(s.raw, env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *recordingSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
|
||||
s.unified = append(s.unified, env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *recordingSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type testWriter struct {
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (w testWriter) Write(p []byte) (int, error) {
|
||||
w.t.Log(string(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func buildGBFrame(command byte, response byte, vin string, body []byte) []byte {
|
||||
frame := []byte{'#', '#', command, response}
|
||||
vinBytes := []byte(vin)
|
||||
if len(vinBytes) < 17 {
|
||||
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
|
||||
}
|
||||
frame = append(frame, vinBytes[:17]...)
|
||||
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
|
||||
frame = append(frame, body...)
|
||||
var bcc byte
|
||||
for _, value := range frame[2:] {
|
||||
bcc ^= value
|
||||
}
|
||||
return append(frame, bcc)
|
||||
}
|
||||
908
go/vehicle-gateway/internal/history/query.go
Normal file
908
go/vehicle-gateway/internal/history/query.go
Normal file
@@ -0,0 +1,908 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Queryer interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
type RawFrameQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
MessageID string
|
||||
OrderBy string
|
||||
IncludeTotal bool
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type RawFrameRow struct {
|
||||
TS string `json:"ts"`
|
||||
FrameID string `json:"frame_id"`
|
||||
EventID string `json:"event_id"`
|
||||
MessageID int64 `json:"message_id"`
|
||||
MessageIDHex string `json:"message_id_hex"`
|
||||
EventTime string `json:"event_time"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
RawSizeBytes int64 `json:"raw_size_bytes"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
ParsedJSON string `json:"parsed_json,omitempty"`
|
||||
FieldsJSON string `json:"fields_json,omitempty"`
|
||||
ParseStatus string `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type LocationQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type LocationRow struct {
|
||||
TS string `json:"ts"`
|
||||
EventID string `json:"event_id"`
|
||||
FrameID string `json:"frame_id"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
Longitude float64 `json:"longitude"`
|
||||
Latitude float64 `json:"latitude"`
|
||||
AltitudeM *float64 `json:"altitude_m,omitempty"`
|
||||
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
|
||||
DirectionDeg *int64 `json:"direction_deg,omitempty"`
|
||||
AlarmFlag *int64 `json:"alarm_flag,omitempty"`
|
||||
StatusFlag *int64 `json:"status_flag,omitempty"`
|
||||
TotalMileageKM *float64 `json:"total_mileage_km,omitempty"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type MileagePointQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type MileagePointRow struct {
|
||||
TS string `json:"ts"`
|
||||
EventID string `json:"event_id"`
|
||||
FrameID string `json:"frame_id"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
TotalMileageKM float64 `json:"total_mileage_km"`
|
||||
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Protocol string `json:"protocol"`
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
}
|
||||
|
||||
type RawFrameRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
func NewRawFrameRepository(db Queryer, database string) *RawFrameRepository {
|
||||
if db == nil {
|
||||
panic("raw frame query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &RawFrameRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
type LocationRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
type MileagePointRepository struct {
|
||||
db Queryer
|
||||
database string
|
||||
}
|
||||
|
||||
func NewLocationRepository(db Queryer, database string) *LocationRepository {
|
||||
if db == nil {
|
||||
panic("location query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &LocationRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
func NewMileagePointRepository(db Queryer, database string) *MileagePointRepository {
|
||||
if db == nil {
|
||||
panic("mileage point query db must not be nil")
|
||||
}
|
||||
database = strings.TrimSpace(database)
|
||||
if database != "" && !safeIdentifier(database) {
|
||||
database = ""
|
||||
}
|
||||
return &MileagePointRepository{db: db, database: database}
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) {
|
||||
query = normalizeRawFrameQuery(query)
|
||||
sqlText, args := buildRawFrameSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]RawFrameRow, 0)
|
||||
for rows.Next() {
|
||||
var row RawFrameRow
|
||||
var ts scanDateTime
|
||||
var eventTime scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.FrameID,
|
||||
&row.EventID,
|
||||
&row.MessageID,
|
||||
&eventTime,
|
||||
&receivedAt,
|
||||
&row.RawSizeBytes,
|
||||
&row.RawHex,
|
||||
&row.RawText,
|
||||
&row.ParsedJSON,
|
||||
&row.FieldsJSON,
|
||||
&row.ParseStatus,
|
||||
&row.ParseError,
|
||||
&row.SourceEndpoint,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.EventTime = eventTime.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.MessageIDHex = fmt.Sprintf("0x%04X", row.MessageID)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
|
||||
query = normalizeRawFrameQuery(query)
|
||||
sqlText, args := buildRawFrameCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]LocationRow, error) {
|
||||
query = normalizeLocationQuery(query)
|
||||
sqlText, args := buildLocationSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]LocationRow, 0)
|
||||
for rows.Next() {
|
||||
var row LocationRow
|
||||
var ts scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
var altitude sql.NullFloat64
|
||||
var speed sql.NullFloat64
|
||||
var direction sql.NullInt64
|
||||
var alarm sql.NullInt64
|
||||
var status sql.NullInt64
|
||||
var mileage sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.EventID,
|
||||
&row.FrameID,
|
||||
&receivedAt,
|
||||
&row.Longitude,
|
||||
&row.Latitude,
|
||||
&altitude,
|
||||
&speed,
|
||||
&direction,
|
||||
&alarm,
|
||||
&status,
|
||||
&mileage,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.AltitudeM = nullableFloat(altitude)
|
||||
row.SpeedKMH = nullableFloat(speed)
|
||||
row.DirectionDeg = nullableInt(direction)
|
||||
row.AlarmFlag = nullableInt(alarm)
|
||||
row.StatusFlag = nullableInt(status)
|
||||
row.TotalMileageKM = nullableFloat(mileage)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *LocationRepository) Count(ctx context.Context, query LocationQuery) (int64, error) {
|
||||
query = normalizeLocationQuery(query)
|
||||
sqlText, args := buildLocationCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) Query(ctx context.Context, query MileagePointQuery) ([]MileagePointRow, error) {
|
||||
query = normalizeMileagePointQuery(query)
|
||||
sqlText, args := buildMileagePointSQL(r.tableName(), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]MileagePointRow, 0)
|
||||
for rows.Next() {
|
||||
var row MileagePointRow
|
||||
var ts scanDateTime
|
||||
var receivedAt scanDateTime
|
||||
var speed sql.NullFloat64
|
||||
var longitude sql.NullFloat64
|
||||
var latitude sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&ts,
|
||||
&row.EventID,
|
||||
&row.FrameID,
|
||||
&receivedAt,
|
||||
&row.TotalMileageKM,
|
||||
&speed,
|
||||
&longitude,
|
||||
&latitude,
|
||||
&row.Protocol,
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&row.Phone,
|
||||
&row.DeviceID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.TS = ts.String
|
||||
row.ReceivedAt = receivedAt.String
|
||||
row.SpeedKMH = nullableFloat(speed)
|
||||
row.Longitude = nullableFloat(longitude)
|
||||
row.Latitude = nullableFloat(latitude)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) Count(ctx context.Context, query MileagePointQuery) (int64, error) {
|
||||
query = normalizeMileagePointQuery(query)
|
||||
sqlText, args := buildMileagePointCountSQL(r.tableName(), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
func countRows(ctx context.Context, db Queryer, sqlText string, args ...any) (int64, error) {
|
||||
rows, err := db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var total int64
|
||||
if rows.Next() {
|
||||
if err := rows.Scan(&total); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return total, rows.Err()
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "raw_frames"
|
||||
}
|
||||
return r.database + ".raw_frames"
|
||||
}
|
||||
|
||||
func (r *LocationRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "vehicle_locations"
|
||||
}
|
||||
return r.database + ".vehicle_locations"
|
||||
}
|
||||
|
||||
func (r *MileagePointRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "vehicle_mileage_points"
|
||||
}
|
||||
return r.database + ".vehicle_mileage_points"
|
||||
}
|
||||
|
||||
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.MessageID = strings.TrimSpace(query.MessageID)
|
||||
query.OrderBy = normalizeRawFrameOrderBy(query.OrderBy)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func normalizeLocationQuery(query LocationQuery) LocationQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func normalizeMileagePointQuery(query MileagePointQuery) MileagePointQuery {
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Phone = strings.TrimSpace(query.Phone)
|
||||
query.DeviceID = strings.TrimSpace(query.DeviceID)
|
||||
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
|
||||
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 20
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
|
||||
where := rawFrameWhere(query)
|
||||
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY " + rawFrameOrderColumn(query.OrderBy) + " DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := rawFrameWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
|
||||
where := locationWhere(query)
|
||||
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildLocationCountSQL(table string, query LocationQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := locationWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildMileagePointSQL(table string, query MileagePointQuery) (string, []any) {
|
||||
where := mileagePointWhere(query)
|
||||
sqlText := `SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func buildMileagePointCountSQL(table string, query MileagePointQuery) (string, []any) {
|
||||
sqlText := `SELECT COUNT(*) FROM ` + table
|
||||
if where := mileagePointWhere(query); len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
func rawFrameWhere(query RawFrameQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.MessageID != "" {
|
||||
if parsed, ok := parseMessageID(query.MessageID); ok {
|
||||
add("message_id = " + strconv.FormatInt(parsed, 10))
|
||||
}
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
func normalizeRawFrameOrderBy(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
switch value {
|
||||
case "receivedat", "received_at":
|
||||
return "receivedAt"
|
||||
default:
|
||||
return "ts"
|
||||
}
|
||||
}
|
||||
|
||||
func rawFrameOrderColumn(value string) string {
|
||||
if normalizeRawFrameOrderBy(value) == "receivedAt" {
|
||||
return "received_at"
|
||||
}
|
||||
return "ts"
|
||||
}
|
||||
|
||||
func locationWhere(query LocationQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
func mileagePointWhere(query MileagePointQuery) []string {
|
||||
var where []string
|
||||
add := func(clause string) {
|
||||
where = append(where, clause)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = '" + quote(query.Protocol) + "'")
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = '" + quote(query.VIN) + "'")
|
||||
}
|
||||
if query.Phone != "" {
|
||||
add("phone = '" + quote(query.Phone) + "'")
|
||||
}
|
||||
if query.DeviceID != "" {
|
||||
add("device_id = '" + quote(query.DeviceID) + "'")
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
type RawFrameHandler struct {
|
||||
repository *RawFrameRepository
|
||||
}
|
||||
|
||||
type LocationHandler struct {
|
||||
repository *LocationRepository
|
||||
}
|
||||
|
||||
type MileagePointHandler struct {
|
||||
repository *MileagePointRepository
|
||||
}
|
||||
|
||||
func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler {
|
||||
if repository == nil {
|
||||
panic("raw frame repository must not be nil")
|
||||
}
|
||||
return &RawFrameHandler{repository: repository}
|
||||
}
|
||||
|
||||
func NewLocationHandler(repository *LocationRepository) *LocationHandler {
|
||||
if repository == nil {
|
||||
panic("location repository must not be nil")
|
||||
}
|
||||
return &LocationHandler{repository: repository}
|
||||
}
|
||||
|
||||
func NewMileagePointHandler(repository *MileagePointRepository) *MileagePointHandler {
|
||||
if repository == nil {
|
||||
panic("mileage point repository must not be nil")
|
||||
}
|
||||
return &MileagePointHandler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/raw-frames" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseRawFrameQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
var total int64
|
||||
if query.IncludeTotal {
|
||||
total, err = h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !query.IncludeTotal {
|
||||
total = int64(len(rows))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/locations" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseLocationQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *MileagePointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/history/mileage-points" {
|
||||
writeHistoryError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseMileagePointQuery(r)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return RawFrameQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return RawFrameQuery{}, err
|
||||
}
|
||||
query := RawFrameQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
MessageID: values.Get("messageId"),
|
||||
OrderBy: values.Get("orderBy"),
|
||||
IncludeTotal: values.Get("includeTotal") != "false",
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
if strings.TrimSpace(query.MessageID) != "" {
|
||||
if _, ok := parseMessageID(query.MessageID); !ok {
|
||||
return RawFrameQuery{}, errors.New("messageId must be decimal or hex like 0x0200")
|
||||
}
|
||||
}
|
||||
return normalizeRawFrameQuery(query), nil
|
||||
}
|
||||
|
||||
func parseMileagePointQuery(r *http.Request) (MileagePointQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return MileagePointQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return MileagePointQuery{}, err
|
||||
}
|
||||
query := MileagePointQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return MileagePointQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
return normalizeMileagePointQuery(query), nil
|
||||
}
|
||||
|
||||
func parseLocationQuery(r *http.Request) (LocationQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
|
||||
if err != nil {
|
||||
return LocationQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return LocationQuery{}, err
|
||||
}
|
||||
query := LocationQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
}
|
||||
return normalizeLocationQuery(query), nil
|
||||
}
|
||||
|
||||
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil || value < min || value > max {
|
||||
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseMessageID(value string) (int64, bool) {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
if value == "" {
|
||||
return 0, false
|
||||
}
|
||||
if strings.HasPrefix(value, "0x") {
|
||||
parsed, err := strconv.ParseInt(strings.TrimPrefix(value, "0x"), 16, 32)
|
||||
return parsed, err == nil
|
||||
}
|
||||
parsed, err := strconv.ParseInt(value, 10, 32)
|
||||
return parsed, err == nil
|
||||
}
|
||||
|
||||
func validDateTime(value string) bool {
|
||||
value = normalizeDateTimeLiteral(value)
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
for _, layout := range []string{"2006-01-02", "2006-01-02 15:04:05", time.RFC3339} {
|
||||
if _, err := time.Parse(layout, value); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeDateTimeLiteral(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} {
|
||||
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
|
||||
return parsed.UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
}
|
||||
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
|
||||
return parsed.UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func safeIdentifier(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type scanDateTime struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDateTime) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatSQLTime(value any, layout string) string {
|
||||
switch typed := value.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case time.Time:
|
||||
return typed.Format(layout)
|
||||
case []byte:
|
||||
return strings.TrimSpace(string(typed))
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
|
||||
func nullableFloat(value sql.NullFloat64) *float64 {
|
||||
if !value.Valid {
|
||||
return nil
|
||||
}
|
||||
return &value.Float64
|
||||
}
|
||||
|
||||
func nullableInt(value sql.NullInt64) *int64 {
|
||||
if !value.Valid {
|
||||
return nil
|
||||
}
|
||||
return &value.Int64
|
||||
}
|
||||
|
||||
func writeHistoryError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
423
go/vehicle-gateway/internal/history/query_test.go
Normal file
423
go/vehicle-gateway/internal/history/query_test.go
Normal file
@@ -0,0 +1,423 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
"go_frame", "event-1", 0x0200,
|
||||
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
time.Date(2026, 7, 1, 23, 25, 37, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
64, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
|
||||
"OK", "", "222.66.200.68:29646", "JT808", "013079963379", "LKLG7C4E3NA774736", "013079963379", "9963379",
|
||||
))
|
||||
|
||||
repository := NewRawFrameRepository(db, "lingniu_vehicle_ts")
|
||||
rows, err := repository.Query(context.Background(), RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
DateFrom: "2026-07-01 00:00:00",
|
||||
DateTo: "2026-07-01 23:59:59",
|
||||
Limit: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query() error = %v", err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("row count = %d", len(rows))
|
||||
}
|
||||
if rows[0].MessageID != 512 || rows[0].MessageIDHex != "0x0200" || rows[0].ParsedJSON == "" {
|
||||
t.Fatalf("unexpected row: %#v", rows[0])
|
||||
}
|
||||
if rows[0].TS != "2026-07-01 23:25:36" {
|
||||
t.Fatalf("timestamp = %q", rows[0].TS)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(38))
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-01 22:28:25", "go_frame", "event-2", 2, "2026-07-01 22:28:25", "2026-07-01 22:28:25",
|
||||
128, "2323", "", `{"command":"REALTIME"}`, `{"total_mileage_km":53490.9}`,
|
||||
"OK", "", "8.134.95.166:53702", "GB32960", "LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"message_id_hex":"0x0002"`, `"total":38`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(11))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "go_frame", "event-3", 0x0200, "2026-07-02 00:18:22", "2026-07-02 00:22:43",
|
||||
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"total_mileage_km":8792.8}`,
|
||||
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"phone":"013307811350"`, `"total":11`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
|
||||
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
|
||||
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
|
||||
))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
|
||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}))
|
||||
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=NOT_EXISTS&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
|
||||
t.Fatalf("expected empty items array, got: %s", body)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocationHandlerReturnsLocationsByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(17))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
|
||||
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
|
||||
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
|
||||
"JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
|
||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
|
||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||
}).AddRow(
|
||||
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
|
||||
8792.8, 8.0, 121.07764, 30.585928,
|
||||
"JT808", "JT808:013307811350", "", "013307811350", "",
|
||||
))
|
||||
|
||||
handler := NewMileagePointHandler(NewMileagePointRepository(db, "lingniu_vehicle_ts"))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/mileage-points?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"total_mileage_km":8792.8`, `"speed_kmh":8`, `"total":19`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
|
||||
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) {
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?dateFrom=2026-07-01T00:00:00&dateTo=2026-07-02T00:00:00", nil)
|
||||
|
||||
query, err := parseRawFrameQuery(request)
|
||||
if err != nil {
|
||||
t.Fatalf("parseRawFrameQuery() error = %v", err)
|
||||
}
|
||||
if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" {
|
||||
t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) {
|
||||
for raw, want := range map[string]string{
|
||||
"2026-07-01T00:00:00": "2026-06-30 16:00:00",
|
||||
"2026-07-01 00:00:00": "2026-06-30 16:00:00",
|
||||
"2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00",
|
||||
"2026-06-30T16:00:00Z": "2026-06-30 16:00:00",
|
||||
} {
|
||||
if got := normalizeDateTimeLiteral(raw); got != want {
|
||||
t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
|
||||
for raw, want := range map[string]int64{
|
||||
"512": 512,
|
||||
"0x0200": 512,
|
||||
"0X0100": 256,
|
||||
} {
|
||||
got, ok := parseMessageID(raw)
|
||||
if !ok || got != want {
|
||||
t.Fatalf("parseMessageID(%q) = %d,%v want %d,true", raw, got, ok, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
DateFrom: "2026-07-02 00:00:00",
|
||||
DateTo: "2026-07-02 23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"ts >= '2026-07-01 16:00:00'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildLocationSQL("lingniu_vehicle_ts.vehicle_locations", LocationQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
DateFrom: "2026-07-02 00:00:00",
|
||||
DateTo: "2026-07-02 23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"ts >= '2026-07-01 16:00:00'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
VehicleKey: "JT808:013307811350",
|
||||
VIN: "VIN'1",
|
||||
MessageID: "0x0200",
|
||||
DateFrom: "2026-07-01T00:00:00",
|
||||
DateTo: "2026-07-01T23:59:59",
|
||||
Limit: 20,
|
||||
Offset: 5,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"protocol = 'JT808'",
|
||||
"vehicle_key = 'JT808:013307811350'",
|
||||
"vin = 'VIN''1'",
|
||||
"message_id = 512",
|
||||
"ts >= '2026-06-30 16:00:00'",
|
||||
"ts <= '2026-07-01 15:59:59'",
|
||||
"LIMIT 20 OFFSET 5",
|
||||
} {
|
||||
if !strings.Contains(sqlText, want) {
|
||||
t.Fatalf("sql missing %s: %s", want, sqlText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRawFrameSQLCanOrderByReceivedAt(t *testing.T) {
|
||||
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
|
||||
Protocol: "JT808",
|
||||
OrderBy: "receivedAt",
|
||||
Limit: 1,
|
||||
})
|
||||
if len(args) != 0 {
|
||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||
}
|
||||
if !strings.Contains(sqlText, "ORDER BY received_at DESC LIMIT 1 OFFSET 0") {
|
||||
t.Fatalf("sql should order by received_at: %s", sqlText)
|
||||
}
|
||||
}
|
||||
71
go/vehicle-gateway/internal/history/schema.go
Normal file
71
go/vehicle-gateway/internal/history/schema.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package history
|
||||
|
||||
const DefaultDatabase = "lingniu_vehicle_ts"
|
||||
|
||||
func SchemaStatements(database string) []string {
|
||||
if database == "" {
|
||||
database = DefaultDatabase
|
||||
}
|
||||
return []string{
|
||||
"CREATE DATABASE IF NOT EXISTS " + database + " KEEP 7300 DURATION 10 BUFFER 256",
|
||||
"USE " + database,
|
||||
`CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
event_id NCHAR(64),
|
||||
message_id INT,
|
||||
event_time TIMESTAMP,
|
||||
received_at TIMESTAMP,
|
||||
raw_size_bytes INT,
|
||||
raw_hex BINARY(16374),
|
||||
raw_text BINARY(16374),
|
||||
parsed_json BINARY(16374),
|
||||
fields_json BINARY(4096),
|
||||
parse_status NCHAR(16),
|
||||
parse_error BINARY(1024),
|
||||
source_endpoint NCHAR(128)
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
`CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg INT,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
total_mileage_km DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
}
|
||||
}
|
||||
339
go/vehicle-gateway/internal/history/writer.go
Normal file
339
go/vehicle-gateway/internal/history/writer.go
Normal file
@@ -0,0 +1,339 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Execer interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
exec Execer
|
||||
cache tableCache
|
||||
}
|
||||
|
||||
type tableCache struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]struct{}
|
||||
}
|
||||
|
||||
func NewWriter(exec Execer) *Writer {
|
||||
if exec == nil {
|
||||
panic("history execer must not be nil")
|
||||
}
|
||||
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
|
||||
}
|
||||
|
||||
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
||||
for _, statement := range SchemaStatements(database) {
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if err := w.AppendRawFrame(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.AppendLocation(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.AppendMileagePoint(ctx, env)
|
||||
}
|
||||
|
||||
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
table := tableName("raw", env)
|
||||
if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
||||
raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
|
||||
VALUES (%s)`, table, joinLiterals(rawValues(env))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
longitude, okLon := floatField(env, envelope.FieldLongitude)
|
||||
latitude, okLat := floatField(env, envelope.FieldLatitude)
|
||||
if !okLon || !okLat {
|
||||
return nil
|
||||
}
|
||||
table := tableName("loc", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_locations", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
||||
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
||||
VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
table := tableName("mil", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
|
||||
VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage))))
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
|
||||
key := stable + "." + table
|
||||
w.cache.mu.Lock()
|
||||
_, ok := w.cache.seen[key]
|
||||
w.cache.mu.Unlock()
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
|
||||
table, stable,
|
||||
quote(string(env.Protocol)),
|
||||
quote(env.VehicleKey()),
|
||||
quote(env.VIN),
|
||||
quote(env.Phone),
|
||||
quote(env.DeviceID))
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
w.cache.mu.Lock()
|
||||
w.cache.seen[key] = struct{}{}
|
||||
w.cache.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawValues(env envelope.FrameEnvelope) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
eventTime := millis(env.EventTimeMS)
|
||||
return []any{
|
||||
received,
|
||||
frameID(env),
|
||||
env.StableEventID(),
|
||||
messageIDInt(env.MessageID),
|
||||
eventTime,
|
||||
received,
|
||||
rawSizeBytes(env),
|
||||
env.RawHex,
|
||||
env.RawText,
|
||||
jsonString(env.Parsed),
|
||||
jsonString(env.Fields),
|
||||
string(env.ParseStatus),
|
||||
env.ParseError,
|
||||
env.SourceEndpoint,
|
||||
}
|
||||
}
|
||||
|
||||
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
longitude,
|
||||
latitude,
|
||||
floatFieldOrNil(env, "altitude_m"),
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
intFieldOrNil(env, "direction_deg"),
|
||||
intFieldOrNil(env, "alarm_flag"),
|
||||
intFieldOrNil(env, "status_flag"),
|
||||
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
|
||||
}
|
||||
}
|
||||
|
||||
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
totalMileage,
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
floatFieldOrNil(env, envelope.FieldLongitude),
|
||||
floatFieldOrNil(env, envelope.FieldLatitude),
|
||||
}
|
||||
}
|
||||
|
||||
func frameID(env envelope.FrameEnvelope) string {
|
||||
return "go_" + env.StableEventID()
|
||||
}
|
||||
|
||||
func tableName(prefix string, env envelope.FrameEnvelope) string {
|
||||
return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
|
||||
}
|
||||
|
||||
func hash16(value string) string {
|
||||
sum := sha1.Sum([]byte(value))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
func messageIDInt(value string) int64 {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
value = strings.TrimPrefix(value, "0x")
|
||||
parsed, err := strconv.ParseInt(value, 16, 32)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
parsed, _ = strconv.ParseInt(value, 10, 32)
|
||||
return parsed
|
||||
}
|
||||
|
||||
func rawSizeBytes(env envelope.FrameEnvelope) int {
|
||||
rawHex := strings.TrimSpace(env.RawHex)
|
||||
if len(rawHex)%2 != 0 {
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
if rawHex != "" {
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
return len([]byte(env.RawText))
|
||||
}
|
||||
|
||||
func jsonString(value any) string {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||
if env.Fields == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
value, ok := floatField(env, key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
if env.Fields == nil {
|
||||
return nil
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return nil
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return typed
|
||||
case int64:
|
||||
return typed
|
||||
case uint16:
|
||||
return int64(typed)
|
||||
case uint32:
|
||||
return int64(typed)
|
||||
case float64:
|
||||
return int64(typed)
|
||||
case string:
|
||||
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
|
||||
if env.EventTimeMS > 0 {
|
||||
return millis(env.EventTimeMS)
|
||||
}
|
||||
return millis(env.ReceivedAtMS)
|
||||
}
|
||||
|
||||
func millis(value int64) time.Time {
|
||||
if value <= 0 {
|
||||
return time.UnixMilli(time.Now().UnixMilli()).UTC()
|
||||
}
|
||||
return time.UnixMilli(value).UTC()
|
||||
}
|
||||
|
||||
func quote(value string) string {
|
||||
return strings.ReplaceAll(value, "'", "''")
|
||||
}
|
||||
|
||||
func joinLiterals(values []any) string {
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, literal(value))
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
func literal(value any) string {
|
||||
if value == nil {
|
||||
return "NULL"
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case time.Time:
|
||||
return "'" + typed.UTC().Format("2006-01-02 15:04:05.000") + "'"
|
||||
case string:
|
||||
return "'" + quote(typed) + "'"
|
||||
case envelope.ParseStatus:
|
||||
return "'" + quote(string(typed)) + "'"
|
||||
case int:
|
||||
return strconv.FormatInt(int64(typed), 10)
|
||||
case int64:
|
||||
return strconv.FormatInt(typed, 10)
|
||||
case uint16:
|
||||
return strconv.FormatUint(uint64(typed), 10)
|
||||
case uint32:
|
||||
return strconv.FormatUint(uint64(typed), 10)
|
||||
case float64:
|
||||
return strconv.FormatFloat(typed, 'f', -1, 64)
|
||||
case float32:
|
||||
return strconv.FormatFloat(float64(typed), 'f', -1, 64)
|
||||
default:
|
||||
return "'" + quote(fmt.Sprint(typed)) + "'"
|
||||
}
|
||||
}
|
||||
146
go/vehicle-gateway/internal/history/writer_test.go
Normal file
146
go/vehicle-gateway/internal/history/writer_test.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
|
||||
statements := strings.Join(SchemaStatements("test_ts"), "\n")
|
||||
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} {
|
||||
if !strings.Contains(statements, want) {
|
||||
t.Fatalf("schema missing %q:\n%s", want, statements)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec)
|
||||
env := sampleEnvelope()
|
||||
|
||||
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||
t.Fatalf("AppendAll() error = %v", err)
|
||||
}
|
||||
|
||||
if got := countSQL(exec.calls, "USING raw_frames"); got != 1 {
|
||||
t.Fatalf("raw child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
|
||||
t.Fatalf("location child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 {
|
||||
t.Fatalf("mileage child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||
t.Fatalf("raw insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
|
||||
t.Fatalf("location insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 {
|
||||
t.Fatalf("mileage insert count = %d", got)
|
||||
}
|
||||
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
|
||||
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {
|
||||
t.Fatalf("raw insert should quote string literals: %s", rawInsert)
|
||||
}
|
||||
if strings.Contains(rawInsert, "VALUES (?,") {
|
||||
t.Fatalf("raw insert should not use placeholders for tdengine websocket plain insert: %s", rawInsert)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec)
|
||||
env := sampleEnvelope()
|
||||
env.Fields = map[string]any{}
|
||||
|
||||
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||
t.Fatalf("AppendAll() error = %v", err)
|
||||
}
|
||||
|
||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||
t.Fatalf("raw insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
|
||||
t.Fatalf("location insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
|
||||
t.Fatalf("mileage insert count = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawSizeBytesUsesRawTextWhenHexIsEmpty(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{RawText: `{"code":"0F80","data":{"speed":12}}`}
|
||||
if got, want := rawSizeBytes(env), len([]byte(env.RawText)); got != want {
|
||||
t.Fatalf("raw text size = %d, want %d", got, want)
|
||||
}
|
||||
|
||||
env.RawHex = "7E0200"
|
||||
if got, want := rawSizeBytes(env), 3; got != want {
|
||||
t.Fatalf("raw hex size = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func sampleEnvelope() envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Sequence: 1,
|
||||
VIN: "LNBVIN00000000001",
|
||||
Phone: "013307795425",
|
||||
SourceEndpoint: "127.0.0.1:18080",
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7E0200",
|
||||
Parsed: map[string]any{"message": "location"},
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 121.069881,
|
||||
envelope.FieldLatitude: 30.590151,
|
||||
envelope.FieldSpeedKMH: 23.0,
|
||||
envelope.FieldTotalMileageKM: 10241.2,
|
||||
"direction_deg": uint16(79),
|
||||
"alarm_flag": uint32(0),
|
||||
"status_flag": uint32(72),
|
||||
},
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
}
|
||||
|
||||
func countSQL(calls []execCall, pattern string) int {
|
||||
var count int
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call.query, pattern) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func findSQL(calls []execCall, pattern string) string {
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call.query, pattern) {
|
||||
return call.query
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type execCall struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type recordingExec struct {
|
||||
calls []execCall
|
||||
}
|
||||
|
||||
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||
return nil, nil
|
||||
}
|
||||
237
go/vehicle-gateway/internal/identity/resolver.go
Normal file
237
go/vehicle-gateway/internal/identity/resolver.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Resolver interface {
|
||||
Resolve(context.Context, envelope.FrameEnvelope) (envelope.FrameEnvelope, error)
|
||||
}
|
||||
|
||||
type NoopResolver struct{}
|
||||
|
||||
func (NoopResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||
return env, nil
|
||||
}
|
||||
|
||||
type MySQLResolver struct {
|
||||
db *sql.DB
|
||||
table string
|
||||
}
|
||||
|
||||
func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver {
|
||||
if db == nil {
|
||||
panic("identity db must not be nil")
|
||||
}
|
||||
table = strings.TrimSpace(table)
|
||||
if table == "" || !safeIdentifier(table) {
|
||||
table = "vehicle_identity_binding"
|
||||
}
|
||||
return &MySQLResolver{db: db, table: table}
|
||||
}
|
||||
|
||||
func safeIdentifier(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||
if strings.TrimSpace(env.VIN) != "" {
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
candidates := CandidateKeys(env)
|
||||
for _, candidate := range candidates {
|
||||
vin, err := r.lookup(ctx, candidate.Column, candidate.Value)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
return env, err
|
||||
}
|
||||
if strings.TrimSpace(vin) != "" {
|
||||
env.VIN = strings.TrimSpace(vin)
|
||||
if env.Parsed == nil {
|
||||
env.Parsed = map[string]any{}
|
||||
}
|
||||
env.Parsed["identity"] = map[string]any{
|
||||
"resolved": true,
|
||||
"source": candidate.Column,
|
||||
"value": candidate.Value,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
}
|
||||
if err := r.trackJT808(ctx, env); err != nil {
|
||||
return env, err
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) {
|
||||
query := "SELECT vin FROM " + r.table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' ORDER BY updated_at DESC LIMIT 1"
|
||||
var vin string
|
||||
err := r.db.QueryRowContext(ctx, query, value).Scan(&vin)
|
||||
return vin, err
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) trackJT808(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if env.Protocol != envelope.ProtocolJT808 || strings.TrimSpace(env.Phone) == "" || strings.TrimSpace(env.MessageID) == "" {
|
||||
return nil
|
||||
}
|
||||
registration := mapValue(env.Parsed, "registration")
|
||||
authentication := mapValue(env.Parsed, "authentication")
|
||||
deviceID := firstNonEmpty(env.DeviceID, textValue(registration, "device_id"))
|
||||
plate := firstNonEmpty(env.Plate, textValue(registration, "plate"))
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
if vin == "" {
|
||||
vin = "unknown"
|
||||
}
|
||||
firstRegisteredAt := nilTime()
|
||||
latestRegisteredAt := nilTime()
|
||||
if env.MessageID == "0x0100" {
|
||||
firstRegisteredAt = "CURRENT_TIMESTAMP"
|
||||
latestRegisteredAt = "CURRENT_TIMESTAMP"
|
||||
}
|
||||
latestAuthenticatedAt := nilTime()
|
||||
if env.MessageID == "0x0102" {
|
||||
latestAuthenticatedAt = "CURRENT_TIMESTAMP"
|
||||
}
|
||||
|
||||
query := `INSERT INTO jt808_registration
|
||||
(phone, device_id, plate, vin, province, city, manufacturer, device_type, plate_color,
|
||||
auth_token, auth_imei, auth_software_version, source_endpoint,
|
||||
first_registered_at, latest_registered_at, latest_authenticated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ` + firstRegisteredAt + `, ` + latestRegisteredAt + `, ` + latestAuthenticatedAt + `)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
device_id = IF(VALUES(device_id) <> '', VALUES(device_id), device_id),
|
||||
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
|
||||
vin = IF(VALUES(vin) <> 'unknown', VALUES(vin), vin),
|
||||
province = IF(VALUES(province) <> '', VALUES(province), province),
|
||||
city = IF(VALUES(city) <> '', VALUES(city), city),
|
||||
manufacturer = IF(VALUES(manufacturer) <> '', VALUES(manufacturer), manufacturer),
|
||||
device_type = IF(VALUES(device_type) <> '', VALUES(device_type), device_type),
|
||||
plate_color = IF(VALUES(plate_color) <> '', VALUES(plate_color), plate_color),
|
||||
auth_token = IF(VALUES(auth_token) <> '', VALUES(auth_token), auth_token),
|
||||
auth_imei = IF(VALUES(auth_imei) <> '', VALUES(auth_imei), auth_imei),
|
||||
auth_software_version = IF(VALUES(auth_software_version) <> '', VALUES(auth_software_version), auth_software_version),
|
||||
source_endpoint = IF(VALUES(source_endpoint) <> '', VALUES(source_endpoint), source_endpoint),
|
||||
latest_seen_at = CURRENT_TIMESTAMP,
|
||||
first_registered_at = COALESCE(first_registered_at, VALUES(first_registered_at)),
|
||||
latest_registered_at = COALESCE(VALUES(latest_registered_at), latest_registered_at),
|
||||
latest_authenticated_at = COALESCE(VALUES(latest_authenticated_at), latest_authenticated_at)`
|
||||
_, err := r.db.ExecContext(ctx, query,
|
||||
env.Phone,
|
||||
deviceID,
|
||||
plate,
|
||||
vin,
|
||||
textValue(registration, "province"),
|
||||
textValue(registration, "city"),
|
||||
textValue(registration, "manufacturer"),
|
||||
textValue(registration, "device_type"),
|
||||
textValue(registration, "plate_color"),
|
||||
textValue(authentication, "token"),
|
||||
textValue(authentication, "imei"),
|
||||
textValue(authentication, "software_version"),
|
||||
env.SourceEndpoint,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if vin == "unknown" {
|
||||
return nil
|
||||
}
|
||||
_, err = r.db.ExecContext(ctx, `UPDATE IGNORE `+r.table+`
|
||||
SET
|
||||
phone = CASE WHEN (phone IS NULL OR phone = '') AND ? <> '' THEN ? ELSE phone END,
|
||||
device_id = CASE WHEN (device_id IS NULL OR device_id = '') AND ? <> '' THEN ? ELSE device_id END,
|
||||
plate = CASE WHEN (plate IS NULL OR plate = '') AND ? <> '' THEN ? ELSE plate END
|
||||
WHERE vin = ?`, env.Phone, env.Phone, deviceID, deviceID, plate, plate, vin)
|
||||
return err
|
||||
}
|
||||
|
||||
type CandidateKey struct {
|
||||
Column string
|
||||
Value string
|
||||
}
|
||||
|
||||
func CandidateKeys(env envelope.FrameEnvelope) []CandidateKey {
|
||||
var out []CandidateKey
|
||||
add := func(column string, value string) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || strings.EqualFold(value, "unknown") {
|
||||
return
|
||||
}
|
||||
for _, existing := range out {
|
||||
if existing.Column == column && existing.Value == value {
|
||||
return
|
||||
}
|
||||
}
|
||||
out = append(out, CandidateKey{Column: column, Value: value})
|
||||
}
|
||||
add("phone", env.Phone)
|
||||
if trimmed := strings.TrimLeft(strings.TrimSpace(env.Phone), "0"); trimmed != "" {
|
||||
add("phone", trimmed)
|
||||
}
|
||||
add("device_id", env.DeviceID)
|
||||
add("plate", env.Plate)
|
||||
add("vin", env.VehicleKeyHint)
|
||||
return out
|
||||
}
|
||||
|
||||
func mapValue(values map[string]any, key string) map[string]any {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
nested, _ := values[key].(map[string]any)
|
||||
return nested
|
||||
}
|
||||
|
||||
func textValue(values map[string]any, key string) string {
|
||||
if values == nil {
|
||||
return ""
|
||||
}
|
||||
value, ok := values[key]
|
||||
if !ok || value == nil {
|
||||
return ""
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func nilTime() string {
|
||||
return "NULL"
|
||||
}
|
||||
171
go/vehicle-gateway/internal/identity/resolver_test.go
Normal file
171
go/vehicle-gateway/internal/identity/resolver_test.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestCandidateKeysPrioritizesPhoneDevicePlate(t *testing.T) {
|
||||
keys := CandidateKeys(envelope.FrameEnvelope{
|
||||
Phone: "013307795425",
|
||||
DeviceID: "D1",
|
||||
Plate: "豫A12345",
|
||||
VehicleKeyHint: "D1",
|
||||
})
|
||||
got := []string{}
|
||||
for _, key := range keys {
|
||||
got = append(got, key.Column+"="+key.Value)
|
||||
}
|
||||
want := []string{"phone=013307795425", "phone=13307795425", "device_id=D1", "plate=豫A12345", "vin=D1"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("candidate count = %d got=%#v", len(got), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateKeysAddsTrimmedPhoneVariant(t *testing.T) {
|
||||
keys := CandidateKeys(envelope.FrameEnvelope{
|
||||
Phone: "013079963379",
|
||||
})
|
||||
got := []string{}
|
||||
for _, key := range keys {
|
||||
got = append(got, key.Column+"="+key.Value)
|
||||
}
|
||||
want := []string{"phone=013079963379", "phone=13079963379"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("candidate count = %d got=%#v", len(got), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001"))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNBVIN00000000001" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "phone" {
|
||||
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverFallsBackToDeviceID(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
|
||||
WithArgs("D1").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000002"))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
DeviceID: "D1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNBVIN00000000002" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverTracksJT808RegistrationAndBackfillsBinding(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("013079963379").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13079963379").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
|
||||
WithArgs("DEV0001").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("TEST123").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LKLG7C4E3NA774736"))
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE IGNORE vehicle_identity_binding").
|
||||
WithArgs("013079963379", "013079963379", "DEV0001", "DEV0001", "TEST123", "TEST123", "LKLG7C4E3NA774736").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0100",
|
||||
Phone: "013079963379",
|
||||
DeviceID: "DEV0001",
|
||||
Plate: "TEST123",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
Parsed: map[string]any{
|
||||
"registration": map[string]any{
|
||||
"province": uint16(16),
|
||||
"city": uint16(32),
|
||||
"manufacturer": "YUTNG",
|
||||
"device_type": "ZK6105CHEVNPG4",
|
||||
"device_id": "DEV0001",
|
||||
"plate_color": uint8(2),
|
||||
"plate": "TEST123",
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LKLG7C4E3NA774736" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) {
|
||||
t.Helper()
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
return db, mock
|
||||
}
|
||||
13
go/vehicle-gateway/internal/observability/logger.go
Normal file
13
go/vehicle-gateway/internal/observability/logger.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
// NewLogger returns a JSON logger with a stable service field so ECS logs from
|
||||
// different Go processes can be filtered without relying on container names.
|
||||
func NewLogger(service string) *slog.Logger {
|
||||
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||
return slog.New(handler).With("service", service)
|
||||
}
|
||||
840
go/vehicle-gateway/internal/protocol/gb32960/parser.go
Normal file
840
go/vehicle-gateway/internal/protocol/gb32960/parser.go
Normal file
@@ -0,0 +1,840 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFrameTooShort = errors.New("gb32960 frame too short")
|
||||
ErrBadStartSymbol = errors.New("gb32960 bad start symbol")
|
||||
ErrBodyLength = errors.New("gb32960 body length mismatch")
|
||||
ErrBCC = errors.New("gb32960 bcc mismatch")
|
||||
)
|
||||
|
||||
const (
|
||||
minFrameLen = 25
|
||||
headerLen = 24
|
||||
)
|
||||
|
||||
// ExtractFrames splits GB/T 32960 TCP streams using the protocol length field.
|
||||
// Returned frames include the start symbols and trailing BCC byte.
|
||||
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
|
||||
offset := 0
|
||||
for {
|
||||
start := indexStart(stream[offset:])
|
||||
if start < 0 {
|
||||
return frames, nil, nil
|
||||
}
|
||||
offset += start
|
||||
remaining := stream[offset:]
|
||||
if len(remaining) < headerLen {
|
||||
return frames, append([]byte(nil), remaining...), nil
|
||||
}
|
||||
bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
|
||||
frameLen := frameLength(remaining, bodyLen)
|
||||
if len(remaining) < frameLen {
|
||||
return frames, append([]byte(nil), remaining...), nil
|
||||
}
|
||||
frames = append(frames, append([]byte(nil), remaining[:frameLen]...))
|
||||
offset += frameLen
|
||||
if offset >= len(stream) {
|
||||
return frames, nil, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||
if len(raw) < minFrameLen {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
version, ok := version(raw[0], raw[1])
|
||||
if !ok {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
|
||||
}
|
||||
bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
|
||||
actualBodyLen := len(raw) - headerLen - 1
|
||||
if len(raw) != headerLen+bodyLen+1 && !isExtendedPlatformLogin(raw[2], bodyLen, actualBodyLen) {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
|
||||
}
|
||||
if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrBCC, got, want)
|
||||
}
|
||||
|
||||
command := raw[2]
|
||||
responseFlag := raw[3]
|
||||
vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
|
||||
body := raw[headerLen : headerLen+actualBodyLen]
|
||||
parsed := map[string]any{
|
||||
"header": map[string]any{
|
||||
"version": version,
|
||||
"command": fmt.Sprintf("0x%02X", command),
|
||||
"response_flag": fmt.Sprintf("0x%02X", responseFlag),
|
||||
"vin": vin,
|
||||
"encrypt": raw[21],
|
||||
"body_length": bodyLen,
|
||||
"actual_body_length": actualBodyLen,
|
||||
},
|
||||
}
|
||||
fields := map[string]any{}
|
||||
eventTimeMS := receivedAtMS
|
||||
|
||||
if command == 0x02 || command == 0x03 {
|
||||
eventTime, units := parseDataBody(version, body, fields)
|
||||
if !eventTime.IsZero() {
|
||||
eventTimeMS = eventTime.UnixMilli()
|
||||
fields["device_time"] = eventTime.Format(time.RFC3339)
|
||||
parsed["device_time"] = eventTime.Format(time.RFC3339)
|
||||
}
|
||||
parsed["data_units"] = units
|
||||
} else if command == 0x05 {
|
||||
login := parsePlatformLogin(body)
|
||||
parsed["platform_login"] = login
|
||||
if deviceTime, ok := login["login_time"].(string); ok && deviceTime != "" {
|
||||
fields["device_time"] = deviceTime
|
||||
parsed["device_time"] = deviceTime
|
||||
}
|
||||
if account, ok := login["username"].(string); ok && account != "" {
|
||||
fields["platform_account"] = account
|
||||
}
|
||||
}
|
||||
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
MessageID: fmt.Sprintf("0x%02X", command),
|
||||
VIN: vin,
|
||||
SourceEndpoint: sourceEndpoint,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
|
||||
if len(body) < 6 {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
eventTime := parseGBTime(body[:6])
|
||||
cursor := 6
|
||||
var units []map[string]any
|
||||
for cursor < len(body) {
|
||||
unitType := body[cursor]
|
||||
cursor++
|
||||
switch unitType {
|
||||
case 0x01:
|
||||
size := 20
|
||||
if version == "V2025" {
|
||||
size = 18
|
||||
}
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x01", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseVehicleData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x01", "name": "vehicle", "value": unit})
|
||||
fields["vehicle_status"] = unit["vehicle_status"]
|
||||
fields["charge_status"] = unit["charge_status"]
|
||||
fields["running_mode"] = unit["running_mode"]
|
||||
fields[envelope.FieldSpeedKMH] = unit["speed_kmh"]
|
||||
fields[envelope.FieldTotalMileageKM] = unit["total_mileage_km"]
|
||||
fields[envelope.FieldSOCPercent] = unit["soc_percent"]
|
||||
cursor += size
|
||||
case 0x05:
|
||||
if len(body[cursor:]) < 9 {
|
||||
units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parsePositionData(body[cursor : cursor+9])
|
||||
units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
|
||||
fields["position_status"] = unit["position_status"]
|
||||
fields[envelope.FieldLongitude] = unit["longitude"]
|
||||
fields[envelope.FieldLatitude] = unit["latitude"]
|
||||
cursor += 9
|
||||
case 0x02:
|
||||
if len(body[cursor:]) < 1 {
|
||||
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
size := 1 + int(body[cursor])*12
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseDriveMotorData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x02", "name": "drive_motor", "value": unit})
|
||||
cursor += size
|
||||
case 0x03:
|
||||
if len(body[cursor:]) < 8 {
|
||||
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
probeCount := int(binary.BigEndian.Uint16(body[cursor+6 : cursor+8]))
|
||||
size := 8 + probeCount + 10
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseFuelCellData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x03", "name": "fuel_cell", "value": unit})
|
||||
fields["fuel_cell_hydrogen_consumption_kg_per_100km"] = unit["hydrogen_consumption_kg_per_100km"]
|
||||
fields["fuel_cell_voltage_v"] = unit["fuel_cell_voltage_v"]
|
||||
fields["fuel_cell_current_a"] = unit["fuel_cell_current_a"]
|
||||
cursor += size
|
||||
case 0x04:
|
||||
if len(body[cursor:]) < 5 {
|
||||
units = append(units, map[string]any{"type": "0x04", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseEngineData(body[cursor : cursor+5])
|
||||
units = append(units, map[string]any{"type": "0x04", "name": "engine", "value": unit})
|
||||
cursor += 5
|
||||
case 0x06:
|
||||
if len(body[cursor:]) < 14 {
|
||||
units = append(units, map[string]any{"type": "0x06", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseExtremeData(body[cursor : cursor+14])
|
||||
units = append(units, map[string]any{"type": "0x06", "name": "extreme", "value": unit})
|
||||
cursor += 14
|
||||
case 0x07:
|
||||
size, ok := alarmDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x07", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseAlarmData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x07", "name": "alarm", "value": unit})
|
||||
cursor += size
|
||||
case 0x08:
|
||||
size, ok := voltageDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x08", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseVoltageData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x08", "name": "voltage", "value": unit})
|
||||
cursor += size
|
||||
case 0x09:
|
||||
size, ok := temperatureDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x09", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseTemperatureData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x09", "name": "temperature", "value": unit})
|
||||
cursor += size
|
||||
case 0x30:
|
||||
size, ok := gdFCStackDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x30", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCStackData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x30", "name": "gd_fc_stack", "value": unit})
|
||||
if summaries, ok := unit["summaries"].([]map[string]any); ok && len(summaries) > 0 {
|
||||
fields["gd_fc_stack_hydrogen_inlet_pressure_kpa"] = summaries[0]["hydrogen_inlet_pressure_kpa"]
|
||||
fields["gd_fc_stack_water_outlet_temp_c"] = summaries[0]["stack_water_outlet_temp_c"]
|
||||
fields["gd_fc_stack_cell_count"] = summaries[0]["cell_count"]
|
||||
fields["gd_fc_stack_frame_cell_count"] = summaries[0]["frame_cell_count"]
|
||||
}
|
||||
cursor += size
|
||||
case 0x31:
|
||||
size, ok := gdFCAuxiliaryDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x31", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCAuxiliaryData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x31", "name": "gd_fc_auxiliary", "value": unit})
|
||||
cursor += size
|
||||
case 0x32:
|
||||
if len(body[cursor:]) < 9 {
|
||||
units = append(units, map[string]any{"type": "0x32", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCDcDcData(body[cursor : cursor+9])
|
||||
units = append(units, map[string]any{"type": "0x32", "name": "gd_fc_dcdc", "value": unit})
|
||||
fields["gd_fc_dcdc_controller_temp_c"] = unit["controller_temp_c"]
|
||||
cursor += 9
|
||||
case 0x33:
|
||||
if len(body[cursor:]) < 5 {
|
||||
units = append(units, map[string]any{"type": "0x33", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCAirConditionerData(body[cursor : cursor+5])
|
||||
units = append(units, map[string]any{"type": "0x33", "name": "gd_fc_air_conditioner", "value": unit})
|
||||
cursor += 5
|
||||
case 0x34:
|
||||
if len(body[cursor:]) < 7 {
|
||||
units = append(units, map[string]any{"type": "0x34", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCVehicleInfoData(body[cursor : cursor+7])
|
||||
units = append(units, map[string]any{"type": "0x34", "name": "gd_fc_vehicle_info", "value": unit})
|
||||
fields["gd_fc_vehicle_hydrogen_mass_kg"] = unit["hydrogen_mass_kg"]
|
||||
cursor += 7
|
||||
case 0x80:
|
||||
if len(body[cursor:]) < 11 {
|
||||
units = append(units, map[string]any{"type": "0x80", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCDemoExtensionData(body[cursor : cursor+11])
|
||||
units = append(units, map[string]any{"type": "0x80", "name": "gd_fc_demo_extension", "value": unit})
|
||||
fields["gd_fc_demo_stack_temp_c"] = unit["stack_temp_c"]
|
||||
cursor += 11
|
||||
case 0x83:
|
||||
size, ok := gdFCVendorTLVDataSize(body[cursor:])
|
||||
if !ok {
|
||||
units = append(units, map[string]any{"type": "0x83", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parseGdFCVendorTLVData(0x83, body[cursor:cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x83", "name": "gd_fc_vendor_tlv", "value": unit})
|
||||
cursor += size
|
||||
default:
|
||||
units = append(units, map[string]any{
|
||||
"type": fmt.Sprintf("0x%02X", unitType),
|
||||
"raw_tail": strings.ToUpper(hex.EncodeToString(body[cursor:])),
|
||||
"parse": "unknown_unit",
|
||||
"byte_size": len(body) - cursor,
|
||||
})
|
||||
return eventTime, units
|
||||
}
|
||||
}
|
||||
return eventTime, units
|
||||
}
|
||||
|
||||
func frameLength(frameStart []byte, bodyLen int) int {
|
||||
standardLen := headerLen + bodyLen + 1
|
||||
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
|
||||
return standardLen
|
||||
}
|
||||
// Some field platform-login frames declare the standard 41-byte body but
|
||||
// carry a 32-byte password, making the actual body 53 bytes. Only consume
|
||||
// the extended form when its own BCC verifies so the splitter does not eat
|
||||
// bytes from the next frame.
|
||||
extendedLen := headerLen + 53 + 1
|
||||
if len(frameStart) < extendedLen {
|
||||
return standardLen
|
||||
}
|
||||
extended := frameStart[:extendedLen]
|
||||
if bcc(extended[2:len(extended)-1]) == extended[len(extended)-1] {
|
||||
return extendedLen
|
||||
}
|
||||
return standardLen
|
||||
}
|
||||
|
||||
func isExtendedPlatformLogin(command byte, declaredBodyLen int, actualBodyLen int) bool {
|
||||
return command == 0x05 && declaredBodyLen == 41 && actualBodyLen == 53
|
||||
}
|
||||
|
||||
func parsePlatformLogin(body []byte) map[string]any {
|
||||
login := map[string]any{
|
||||
"raw_body_length": len(body),
|
||||
}
|
||||
if len(body) < 9 {
|
||||
login["error"] = "truncated"
|
||||
return login
|
||||
}
|
||||
loginTime := parseGBTime(body[:6])
|
||||
if !loginTime.IsZero() {
|
||||
login["login_time"] = loginTime.Format(time.RFC3339)
|
||||
}
|
||||
login["serial_no"] = binary.BigEndian.Uint16(body[6:8])
|
||||
if len(body) >= 21 {
|
||||
login["username"] = trimASCII(body[8:20])
|
||||
}
|
||||
if len(body) > 21 {
|
||||
passwordEnd := len(body) - 1
|
||||
login["password"] = trimASCII(body[20:passwordEnd])
|
||||
login["encrypt_rule"] = body[passwordEnd]
|
||||
}
|
||||
return login
|
||||
}
|
||||
|
||||
func parseVehicleData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"vehicle_status": int(data[0]),
|
||||
"charge_status": int(data[1]),
|
||||
"running_mode": int(data[2]),
|
||||
"speed_kmh": float64(binary.BigEndian.Uint16(data[3:5])) / 10,
|
||||
"total_mileage_km": float64(binary.BigEndian.Uint32(data[5:9])) / 10,
|
||||
"total_voltage_v": float64(binary.BigEndian.Uint16(data[9:11])) / 10,
|
||||
"total_current_a": float64(binary.BigEndian.Uint16(data[11:13]))/10 - 1000,
|
||||
"soc_percent": int(data[13]),
|
||||
"dc_dc_status": int(data[14]),
|
||||
"gear": int(data[15]),
|
||||
"insulation_kohm": binary.BigEndian.Uint16(data[16:18]),
|
||||
"accelerator_pct": int(data[18]),
|
||||
"brake_pct": int(data[19]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseDriveMotorData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
motors := make([]map[string]any, 0, count)
|
||||
cursor := 1
|
||||
for i := 0; i < count; i++ {
|
||||
motors = append(motors, map[string]any{
|
||||
"serial_no": int(data[cursor]),
|
||||
"state": int(data[cursor+1]),
|
||||
"controller_temperature_c": int(data[cursor+2]) - 40,
|
||||
"speed_rpm": int(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) - 20000,
|
||||
"torque_nm": float64(int(binary.BigEndian.Uint16(data[cursor+5:cursor+7]))-20000) / 10,
|
||||
"motor_temperature_c": int(data[cursor+7]) - 40,
|
||||
"controller_voltage_v": float64(binary.BigEndian.Uint16(data[cursor+8:cursor+10])) / 10,
|
||||
"controller_current_a": float64(binary.BigEndian.Uint16(data[cursor+10:cursor+12]))/10 - 1000,
|
||||
})
|
||||
cursor += 12
|
||||
}
|
||||
return map[string]any{
|
||||
"count": count,
|
||||
"motors": motors,
|
||||
}
|
||||
}
|
||||
|
||||
func parseFuelCellData(data []byte) map[string]any {
|
||||
probeCount := int(binary.BigEndian.Uint16(data[6:8]))
|
||||
probes := make([]int, 0, probeCount)
|
||||
cursor := 8
|
||||
for i := 0; i < probeCount; i++ {
|
||||
probes = append(probes, int(data[cursor])-40)
|
||||
cursor++
|
||||
}
|
||||
out := map[string]any{
|
||||
"fuel_cell_voltage_v": float64(binary.BigEndian.Uint16(data[0:2])) / 10,
|
||||
"fuel_cell_current_a": float64(binary.BigEndian.Uint16(data[2:4])) / 10,
|
||||
"hydrogen_consumption_kg_per_100km": float64(binary.BigEndian.Uint16(data[4:6])) / 100,
|
||||
"temperature_probe_count": probeCount,
|
||||
"temperature_probe_values_c": probes,
|
||||
"max_hydrogen_temperature_c": float64(binary.BigEndian.Uint16(data[cursor:cursor+2]))/10 - 40,
|
||||
"max_hydrogen_temperature_probe_id": int(data[cursor+2]),
|
||||
"max_hydrogen_concentration_fraction": float64(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) * 0.000001,
|
||||
"max_hydrogen_concentration_probe_id": int(data[cursor+5]),
|
||||
"max_hydrogen_pressure_mpa": float64(binary.BigEndian.Uint16(data[cursor+6:cursor+8])) / 10,
|
||||
"max_hydrogen_pressure_probe_id": int(data[cursor+8]),
|
||||
"dc_dc_status": int(data[cursor+9]),
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseEngineData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"engine_status": int(data[0]),
|
||||
"crank_speed_rpm": binary.BigEndian.Uint16(data[1:3]),
|
||||
"fuel_rate": binary.BigEndian.Uint16(data[3:5]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseExtremeData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"max_voltage_subsystem_no": int(data[0]),
|
||||
"max_voltage_cell_no": int(data[1]),
|
||||
"max_voltage_v": float64(binary.BigEndian.Uint16(data[2:4])) / 1000,
|
||||
"min_voltage_subsystem_no": int(data[4]),
|
||||
"min_voltage_cell_no": int(data[5]),
|
||||
"min_voltage_v": float64(binary.BigEndian.Uint16(data[6:8])) / 1000,
|
||||
"max_temp_subsystem_no": int(data[8]),
|
||||
"max_temp_probe_no": int(data[9]),
|
||||
"max_temp_c": int(data[10]) - 40,
|
||||
"min_temp_subsystem_no": int(data[11]),
|
||||
"min_temp_probe_no": int(data[12]),
|
||||
"min_temp_c": int(data[13]) - 40,
|
||||
}
|
||||
}
|
||||
|
||||
func alarmDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 5 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 5
|
||||
for i := 0; i < 4; i++ {
|
||||
if len(data[cursor:]) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
count := int(data[cursor])
|
||||
cursor++
|
||||
cursor += count * 4
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseAlarmData(data []byte) map[string]any {
|
||||
cursor := 5
|
||||
readList := func() []string {
|
||||
count := int(data[cursor])
|
||||
cursor++
|
||||
values := make([]string, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
values = append(values, fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[cursor:cursor+4])))
|
||||
cursor += 4
|
||||
}
|
||||
return values
|
||||
}
|
||||
return map[string]any{
|
||||
"max_alarm_level": int(data[0]),
|
||||
"general_alarm_flag": fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[1:5])),
|
||||
"battery_faults": readList(),
|
||||
"motor_faults": readList(),
|
||||
"engine_faults": readList(),
|
||||
"other_faults": readList(),
|
||||
}
|
||||
}
|
||||
|
||||
func voltageDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 10 {
|
||||
return 0, false
|
||||
}
|
||||
frameCellCount := int(data[cursor+9])
|
||||
cursor += 10 + frameCellCount*2
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseVoltageData(data []byte) map[string]any {
|
||||
subCount := int(data[0])
|
||||
cursor := 1
|
||||
totalVoltage := 0.0
|
||||
maxCell := 0.0
|
||||
minCell := 0.0
|
||||
cellSeen := false
|
||||
for i := 0; i < subCount; i++ {
|
||||
totalVoltage += float64(binary.BigEndian.Uint16(data[cursor+1:cursor+3])) / 10
|
||||
cellCount := int(binary.BigEndian.Uint16(data[cursor+5 : cursor+7]))
|
||||
frameCellCount := int(data[cursor+9])
|
||||
cursor += 10
|
||||
for c := 0; c < frameCellCount; c++ {
|
||||
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
|
||||
if !cellSeen || voltage > maxCell {
|
||||
maxCell = voltage
|
||||
}
|
||||
if !cellSeen || voltage < minCell {
|
||||
minCell = voltage
|
||||
}
|
||||
cellSeen = true
|
||||
cursor += 2
|
||||
}
|
||||
_ = cellCount
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": subCount,
|
||||
"total_voltage_v": totalVoltage,
|
||||
"max_cell_v": maxCell,
|
||||
"min_cell_v": minCell,
|
||||
}
|
||||
}
|
||||
|
||||
func temperatureDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 3 {
|
||||
return 0, false
|
||||
}
|
||||
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
|
||||
cursor += 3 + probeCount
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseTemperatureData(data []byte) map[string]any {
|
||||
subCount := int(data[0])
|
||||
cursor := 1
|
||||
maxTemp := 0
|
||||
minTemp := 0
|
||||
seen := false
|
||||
for i := 0; i < subCount; i++ {
|
||||
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
|
||||
cursor += 3
|
||||
for p := 0; p < probeCount; p++ {
|
||||
temp := int(data[cursor]) - 40
|
||||
if !seen || temp > maxTemp {
|
||||
maxTemp = temp
|
||||
}
|
||||
if !seen || temp < minTemp {
|
||||
minTemp = temp
|
||||
}
|
||||
seen = true
|
||||
cursor++
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": subCount,
|
||||
"max_temp_c": maxTemp,
|
||||
"min_temp_c": minTemp,
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCStackDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
cursor := 1
|
||||
for i := 0; i < int(data[0]); i++ {
|
||||
if len(data[cursor:]) < 22 {
|
||||
return 0, false
|
||||
}
|
||||
frameCellCount := int(data[cursor+21])
|
||||
cursor += 22 + frameCellCount*2
|
||||
if cursor > len(data) {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return cursor, true
|
||||
}
|
||||
|
||||
func parseGdFCStackData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
cursor := 1
|
||||
summaries := make([]map[string]any, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
frameCellCount := int(data[cursor+21])
|
||||
summary := map[string]any{
|
||||
"engine_work_state": int(data[cursor]),
|
||||
"stack_water_outlet_temp_c": nullableTempOffset40(data[cursor+1]),
|
||||
"hydrogen_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+2:cursor+4], 0.1, -100),
|
||||
"air_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+4:cursor+6], 0.1, -100),
|
||||
"air_inlet_temp_c": nullableTempOffset40(data[cursor+6]),
|
||||
"max_cell_voltage_id": nullableU16(data[cursor+7 : cursor+9]),
|
||||
"min_cell_voltage_id": nullableU16(data[cursor+9 : cursor+11]),
|
||||
"max_cell_voltage_v": nullableScaledU16(data[cursor+11:cursor+13], 0.001),
|
||||
"min_cell_voltage_v": nullableScaledU16(data[cursor+13:cursor+15], 0.001),
|
||||
"avg_cell_voltage_v": nullableScaledU16(data[cursor+15:cursor+17], 0.001),
|
||||
"cell_count": nullableU16(data[cursor+17 : cursor+19]),
|
||||
"frame_cell_start": nullableU16(data[cursor+19 : cursor+21]),
|
||||
"frame_cell_count": frameCellCount,
|
||||
}
|
||||
cursor += 22
|
||||
maxCell := 0.0
|
||||
minCell := 0.0
|
||||
seen := false
|
||||
for c := 0; c < frameCellCount; c++ {
|
||||
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
|
||||
if !seen || voltage > maxCell {
|
||||
maxCell = voltage
|
||||
}
|
||||
if !seen || voltage < minCell {
|
||||
minCell = voltage
|
||||
}
|
||||
seen = true
|
||||
cursor += 2
|
||||
}
|
||||
if seen {
|
||||
summary["frame_max_cell_voltage_v"] = maxCell
|
||||
summary["frame_min_cell_voltage_v"] = minCell
|
||||
}
|
||||
summaries = append(summaries, summary)
|
||||
}
|
||||
return map[string]any{
|
||||
"stack_count": count,
|
||||
"summaries": summaries,
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCAuxiliaryDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 1 {
|
||||
return 0, false
|
||||
}
|
||||
size := 1 + int(data[0])*16
|
||||
return size, len(data) >= size
|
||||
}
|
||||
|
||||
func parseGdFCAuxiliaryData(data []byte) map[string]any {
|
||||
count := int(data[0])
|
||||
cursor := 1
|
||||
subsystems := make([]map[string]any, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
subsystems = append(subsystems, map[string]any{
|
||||
"air_compressor_motor_voltage_v": nullableScaledU16(data[cursor:cursor+2], 0.1),
|
||||
"air_compressor_power_kw": nullableU16WithOffset(data[cursor+2:cursor+4], -5),
|
||||
"hydrogen_pump_motor_voltage_v": nullableScaledU16(data[cursor+4:cursor+6], 0.2),
|
||||
"hydrogen_pump_power_kw": nullableU16WithOffset(data[cursor+6:cursor+8], -5),
|
||||
"water_pump_voltage_v": nullableScaledU16(data[cursor+8:cursor+10], 0.1),
|
||||
"ptc_voltage_v": nullableScaledU16(data[cursor+10:cursor+12], 0.1),
|
||||
"ptc_power_kw": nullableU16WithOffset(data[cursor+12:cursor+14], -5),
|
||||
"low_voltage_battery_voltage_v": nullableScaledU16(data[cursor+14:cursor+16], 0.1),
|
||||
})
|
||||
cursor += 16
|
||||
}
|
||||
return map[string]any{
|
||||
"subsystem_count": count,
|
||||
"subsystems": subsystems,
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCDcDcData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"input_voltage_v": nullableScaledU16(data[0:2], 0.1),
|
||||
"input_current_a": nullableScaledU16(data[2:4], 0.1),
|
||||
"output_voltage_v": nullableScaledU16(data[4:6], 0.1),
|
||||
"output_current_a": nullableScaledU16(data[6:8], 0.1),
|
||||
"controller_temp_c": nullableTempOffset40(data[8]),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCAirConditionerData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"status": int(data[0]),
|
||||
"power_kw": nullableU16WithOffset(data[1:3], -5),
|
||||
"compressor_input_voltage_v": nullableScaledU16(data[3:5], 0.1),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCVehicleInfoData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"collision_alarm": int(data[0]),
|
||||
"ambient_temp_c": nullableU16WithOffset(data[1:3], -40),
|
||||
"ambient_pressure_kpa": nullableScaledU16WithOffset(data[3:5], 0.1, -100),
|
||||
"hydrogen_mass_kg": nullableScaledU16(data[5:7], 0.1),
|
||||
}
|
||||
}
|
||||
|
||||
func parseGdFCDemoExtensionData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"declared_length": int(binary.BigEndian.Uint16(data[0:2])),
|
||||
"stack_temp_c": nullableTempOffset40(data[2]),
|
||||
"air_compressor_voltage_v": nullableScaledU16(data[3:5], 0.1),
|
||||
"air_compressor_current_a": nullableScaledU16WithOffset(data[5:7], 0.1, -1000),
|
||||
"hydrogen_pump_voltage_v": nullableScaledU16(data[7:9], 0.1),
|
||||
"hydrogen_pump_current_a": nullableScaledU16WithOffset(data[9:11], 0.1, -1000),
|
||||
}
|
||||
}
|
||||
|
||||
func gdFCVendorTLVDataSize(data []byte) (int, bool) {
|
||||
if len(data) < 2 {
|
||||
return 0, false
|
||||
}
|
||||
size := 2 + int(binary.BigEndian.Uint16(data[0:2]))
|
||||
return size, len(data) >= size
|
||||
}
|
||||
|
||||
func parseGdFCVendorTLVData(typeCode byte, data []byte) map[string]any {
|
||||
length := int(binary.BigEndian.Uint16(data[0:2]))
|
||||
return map[string]any{
|
||||
"type": fmt.Sprintf("0x%02X", typeCode),
|
||||
"declared_length": length,
|
||||
"payload_hex": strings.ToUpper(hex.EncodeToString(data[2:])),
|
||||
}
|
||||
}
|
||||
|
||||
func nullableU16(data []byte) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return int(value)
|
||||
}
|
||||
|
||||
func nullableU16WithOffset(data []byte, offset int) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return int(value) + offset
|
||||
}
|
||||
|
||||
func nullableScaledU16(data []byte, scale float64) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return float64(value) * scale
|
||||
}
|
||||
|
||||
func nullableScaledU16WithOffset(data []byte, scale float64, offset float64) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
return nil
|
||||
}
|
||||
return float64(value)*scale + offset
|
||||
}
|
||||
|
||||
func nullableTempOffset40(value byte) any {
|
||||
if value == 0xfe || value == 0xff {
|
||||
return nil
|
||||
}
|
||||
return int(value) - 40
|
||||
}
|
||||
|
||||
func parsePositionData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"position_status": int(data[0]),
|
||||
"longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
|
||||
"latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
func indexStart(data []byte) int {
|
||||
for i := 0; i+1 < len(data); i++ {
|
||||
if _, ok := version(data[i], data[i+1]); ok {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func version(a byte, b byte) (string, bool) {
|
||||
switch {
|
||||
case a == '#' && b == '#':
|
||||
return "V2016", true
|
||||
case a == '$' && b == '$':
|
||||
return "V2025", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func parseGBTime(data []byte) time.Time {
|
||||
if len(data) != 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
year := 2000 + int(data[0])
|
||||
month := time.Month(data[1])
|
||||
day := int(data[2])
|
||||
hour := int(data[3])
|
||||
minute := int(data[4])
|
||||
second := int(data[5])
|
||||
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func trimASCII(data []byte) string {
|
||||
return strings.TrimRight(string(data), "\x00 ")
|
||||
}
|
||||
|
||||
func bcc(data []byte) byte {
|
||||
var out byte
|
||||
for _, value := range data {
|
||||
out ^= value
|
||||
}
|
||||
return out
|
||||
}
|
||||
258
go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
Normal file
258
go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
Normal file
@@ -0,0 +1,258 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
|
||||
|
||||
func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
|
||||
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39})
|
||||
second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
|
||||
stream := append([]byte{0x00, 0x01}, first...)
|
||||
stream = append(stream, second[:len(second)-3]...)
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 {
|
||||
t.Fatalf("expected one complete frame, got %d", len(frames))
|
||||
}
|
||||
if string(remainder) != string(second[:len(second)-3]) {
|
||||
t.Fatalf("expected partial second frame as remainder, got %s", hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x02" {
|
||||
t.Fatalf("unexpected envelope header: %#v", env)
|
||||
}
|
||||
if env.VIN != "LNBSCB3D4R1234567" {
|
||||
t.Fatalf("unexpected vin: %q", env.VIN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameRejectsBadBCC(t *testing.T) {
|
||||
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
frame[len(frame)-1] ^= 0xff
|
||||
|
||||
if _, err := ParseFrame(frame, 1782745114999, ""); err == nil {
|
||||
t.Fatal("expected bad bcc error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFramesSupportsExtendedPlatformLogin(t *testing.T) {
|
||||
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x1f, 0x00, 0x01}
|
||||
body = append(body, fixedASCII("Hyundai", 12)...)
|
||||
body = append(body, fixedASCII("f2e3445d7cda409fb4f278f6fb890734", 32)...)
|
||||
body = append(body, 0x01)
|
||||
frame := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
|
||||
|
||||
frames, remainder, err := ExtractFrames(append(frame, []byte{0x23, 0x23, 0x05}...))
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 3 {
|
||||
t.Fatalf("unexpected split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
if len(frames[0]) != headerLen+53+1 {
|
||||
t.Fatalf("extended frame len = %d", len(frames[0]))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "8.134.95.166:39376")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x05" || env.ParseStatus != envelope.ParseOK {
|
||||
t.Fatalf("unexpected env: %#v", env)
|
||||
}
|
||||
header := env.Parsed["header"].(map[string]any)
|
||||
if header["body_length"] != 41 || header["actual_body_length"] != 53 {
|
||||
t.Fatalf("unexpected length header: %#v", header)
|
||||
}
|
||||
login := env.Parsed["platform_login"].(map[string]any)
|
||||
if login["username"] != "Hyundai" || login["password"] != "f2e3445d7cda409fb4f278f6fb890734" {
|
||||
t.Fatalf("unexpected platform login: %#v", login)
|
||||
}
|
||||
if login["login_time"] != "2026-07-02T00:38:31+08:00" {
|
||||
t.Fatalf("unexpected login time: %#v", login)
|
||||
}
|
||||
if env.Fields["platform_account"] != "Hyundai" {
|
||||
t.Fatalf("platform account not exposed: %#v", env.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponseEchoesRawVINAndOriginalTime(t *testing.T) {
|
||||
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x0f, 0x00, 0x01}
|
||||
body = append(body, fixedASCII("Hyundai", 12)...)
|
||||
body = append(body, fixedASCII("ack-test-password", 32)...)
|
||||
body = append(body, 0x01)
|
||||
request := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
|
||||
env, err := ParseFrame(request, 1782914969584, "8.134.95.166:39376")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := AutoResponse(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("AutoResponse() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
if response[2] != 0x05 || response[3] != 0x01 {
|
||||
t.Fatalf("unexpected response header: %x", response[:4])
|
||||
}
|
||||
if hex.EncodeToString(response[4:21]) != "0000000000000000000000000000000000" {
|
||||
t.Fatalf("raw vin not echoed: %x", response[4:21])
|
||||
}
|
||||
if hex.EncodeToString(response[24:30]) != "1a070200260f" {
|
||||
t.Fatalf("timestamp body not preserved: %x", response[24:30])
|
||||
}
|
||||
if got, want := bcc(response[2:len(response)-1]), response[len(response)-1]; got != want {
|
||||
t.Fatalf("response bcc got=0x%02x want=0x%02x", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
|
||||
body := []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39}
|
||||
body = append(body, 0x01)
|
||||
body = append(body,
|
||||
0x01, // vehicle status
|
||||
0x03, // charge status
|
||||
0x02, // running mode
|
||||
0x01, 0x2c, // speed: 30.0km/h
|
||||
0x00, 0x01, 0x86, 0xa0, // total mileage: 10000.0km
|
||||
0x15, 0xe5, // total voltage: 560.5V
|
||||
0x27, 0x10, // total current: 0A after -1000 offset
|
||||
85, 0x01, 0x00,
|
||||
0x27, 0x10,
|
||||
0x00, 0x00)
|
||||
body = append(body, 0x05)
|
||||
body = append(body,
|
||||
0x00,
|
||||
0x07, 0x36, 0x50, 0x40, // longitude: 121.000000
|
||||
0x01, 0xd2, 0x4f, 0x00) // latitude: 30.560000
|
||||
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", body)
|
||||
|
||||
env, err := ParseFrame(frame, 1782745114999, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 30.0)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10000.0)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121.0)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.56)
|
||||
if env.Fields[envelope.FieldSOCPercent] != 85 {
|
||||
t.Fatalf("unexpected soc: %#v", env.Fields[envelope.FieldSOCPercent])
|
||||
}
|
||||
if env.EventTimeMS == 1782745114999 {
|
||||
t.Fatal("event time should come from GB32960 device timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testing.T) {
|
||||
frame, err := hex.DecodeString(realFuelCellFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frame, 1782914969584, "115.29.187.205:32960")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LB9A32A21R0LS1707" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 53490.9)
|
||||
assertFloatField(t, env, "fuel_cell_hydrogen_consumption_kg_per_100km", 1.8)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 120.800326)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 31.634907)
|
||||
assertFloatField(t, env, "gd_fc_stack_hydrogen_inlet_pressure_kpa", 97.0)
|
||||
assertFloatField(t, env, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
|
||||
assertIntField(t, env, "gd_fc_demo_stack_temp_c", 65)
|
||||
|
||||
units, ok := env.Parsed["data_units"].([]map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("data_units missing: %#v", env.Parsed["data_units"])
|
||||
}
|
||||
gotTypes := make([]string, 0, len(units))
|
||||
for _, unit := range units {
|
||||
gotTypes = append(gotTypes, fmt.Sprint(unit["type"]))
|
||||
}
|
||||
wantTypes := []string{"0x01", "0x02", "0x03", "0x04", "0x05", "0x06", "0x07", "0x08", "0x09", "0x30", "0x31", "0x32", "0x33", "0x34", "0x80", "0x83"}
|
||||
if fmt.Sprint(gotTypes) != fmt.Sprint(wantTypes) {
|
||||
t.Fatalf("unit types = %v, want %v", gotTypes, wantTypes)
|
||||
}
|
||||
if units[1]["name"] != "drive_motor" {
|
||||
t.Fatalf("unexpected motor unit: %#v", units[1])
|
||||
}
|
||||
if units[2]["name"] != "fuel_cell" {
|
||||
t.Fatalf("unexpected fuel cell unit: %#v", units[2])
|
||||
}
|
||||
if units[9]["name"] != "gd_fc_stack" {
|
||||
t.Fatalf("unexpected gd stack unit: %#v", units[9])
|
||||
}
|
||||
if units[10]["name"] != "gd_fc_auxiliary" {
|
||||
t.Fatalf("unexpected gd auxiliary unit: %#v", units[10])
|
||||
}
|
||||
if units[15]["name"] != "gd_fc_vendor_tlv" {
|
||||
t.Fatalf("unexpected gd vendor tlv unit: %#v", units[15])
|
||||
}
|
||||
tlv := units[15]["value"].(map[string]any)
|
||||
if tlv["declared_length"] != 36 {
|
||||
t.Fatalf("unexpected vendor tlv: %#v", tlv)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if math.Abs(got-want) > 0.000001 {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntField(t *testing.T, env envelope.FrameEnvelope, key string, want int) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(int)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not int: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func buildFrame(command byte, response byte, vin string, body []byte) []byte {
|
||||
return buildFrameWithDeclaredLength(command, response, vin, body, len(body))
|
||||
}
|
||||
|
||||
func buildFrameWithDeclaredLength(command byte, response byte, vin string, body []byte, declaredBodyLen int) []byte {
|
||||
frame := []byte{'#', '#', command, response}
|
||||
vinBytes := []byte(vin)
|
||||
if len(vinBytes) < 17 {
|
||||
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
|
||||
}
|
||||
frame = append(frame, vinBytes[:17]...)
|
||||
frame = append(frame, 0x01, byte(declaredBodyLen>>8), byte(declaredBodyLen))
|
||||
frame = append(frame, body...)
|
||||
return append(frame, bcc(frame[2:]))
|
||||
}
|
||||
|
||||
func fixedASCII(value string, size int) []byte {
|
||||
out := make([]byte, size)
|
||||
copy(out, []byte(value))
|
||||
return out
|
||||
}
|
||||
80
go/vehicle-gateway/internal/protocol/gb32960/response.go
Normal file
80
go/vehicle-gateway/internal/protocol/gb32960/response.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrResponseFrameTooShort = errors.New("gb32960 response frame too short")
|
||||
|
||||
const (
|
||||
responseSuccess = byte(0x01)
|
||||
responseCommand = byte(0xfe)
|
||||
encryptNone = byte(0x01)
|
||||
)
|
||||
|
||||
// AutoResponse builds the protocol ACK for accepted upstream GB/T 32960 frames.
|
||||
// The ACK is written only after the caller has durably published the frame.
|
||||
func AutoResponse(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return nil, false, nil
|
||||
}
|
||||
if len(raw) < headerLen+1 {
|
||||
return nil, false, ErrResponseFrameTooShort
|
||||
}
|
||||
if raw[3] != responseCommand {
|
||||
return nil, false, nil
|
||||
}
|
||||
command := raw[2]
|
||||
if !shouldRespond(command) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
body := []byte(nil)
|
||||
if timestamp := responseTime(raw, command); !timestamp.IsZero() {
|
||||
body = encodeGBTime(timestamp)
|
||||
}
|
||||
return buildResponse(raw[0], command, responseSuccess, raw[4:21], body), true, nil
|
||||
}
|
||||
|
||||
func shouldRespond(command byte) bool {
|
||||
switch command {
|
||||
case 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func responseTime(raw []byte, command byte) time.Time {
|
||||
bodyLen := len(raw) - headerLen - 1
|
||||
if bodyLen >= 6 {
|
||||
return parseGBTime(raw[headerLen : headerLen+6])
|
||||
}
|
||||
return time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func encodeGBTime(t time.Time) []byte {
|
||||
local := t.In(time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
return []byte{
|
||||
byte(local.Year() - 2000),
|
||||
byte(local.Month()),
|
||||
byte(local.Day()),
|
||||
byte(local.Hour()),
|
||||
byte(local.Minute()),
|
||||
byte(local.Second()),
|
||||
}
|
||||
}
|
||||
|
||||
func buildResponse(start byte, command byte, responseFlag byte, vin17 []byte, body []byte) []byte {
|
||||
out := make([]byte, 0, headerLen+len(body)+1)
|
||||
out = append(out, start, start, command, responseFlag)
|
||||
out = append(out, vin17[:17]...)
|
||||
out = append(out, encryptNone, 0, 0)
|
||||
binary.BigEndian.PutUint16(out[22:24], uint16(len(body)))
|
||||
out = append(out, body...)
|
||||
return append(out, bcc(out[2:]))
|
||||
}
|
||||
600
go/vehicle-gateway/internal/protocol/jt808/parser.go
Normal file
600
go/vehicle-gateway/internal/protocol/jt808/parser.go
Normal file
@@ -0,0 +1,600 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFrameTooShort = errors.New("jt808 frame too short")
|
||||
ErrChecksum = errors.New("jt808 checksum mismatch")
|
||||
ErrEscape = errors.New("jt808 invalid escape sequence")
|
||||
)
|
||||
|
||||
type Location struct {
|
||||
AlarmFlag uint32
|
||||
StatusFlag uint32
|
||||
Latitude float64
|
||||
Longitude float64
|
||||
AltitudeM uint16
|
||||
SpeedKMH float64
|
||||
DirectionDeg uint16
|
||||
Time time.Time
|
||||
TotalMileageKM *float64
|
||||
Additional []AdditionalItem
|
||||
}
|
||||
|
||||
type AdditionalItem struct {
|
||||
ID byte
|
||||
Length int
|
||||
ValueHex string
|
||||
Parsed any
|
||||
}
|
||||
|
||||
// ExtractFrames splits JT/T 808 TCP streams by 0x7e delimiters and unescapes
|
||||
// complete frame payloads. Returned frames exclude the leading and trailing 0x7e.
|
||||
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
|
||||
start := -1
|
||||
for i, value := range stream {
|
||||
if value != 0x7e {
|
||||
continue
|
||||
}
|
||||
if start < 0 {
|
||||
start = i
|
||||
continue
|
||||
}
|
||||
if i == start+1 {
|
||||
start = i
|
||||
continue
|
||||
}
|
||||
frame, err := unescape(stream[start+1 : i])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
frames = append(frames, frame)
|
||||
start = i
|
||||
}
|
||||
if start >= 0 && start < len(stream)-1 {
|
||||
return frames, append([]byte(nil), stream[start:]...), nil
|
||||
}
|
||||
return frames, nil, nil
|
||||
}
|
||||
|
||||
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||
if len(raw) < 13 {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
if got, want := checksum(raw[:len(raw)-1]), raw[len(raw)-1]; got != want {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrChecksum, got, want)
|
||||
}
|
||||
|
||||
messageID := binary.BigEndian.Uint16(raw[0:2])
|
||||
props := binary.BigEndian.Uint16(raw[2:4])
|
||||
bodySize := props & 0x03ff
|
||||
versioned := props&0x4000 != 0
|
||||
headerLen := 12
|
||||
phoneStart := 4
|
||||
phoneEnd := 10
|
||||
serialStart := 10
|
||||
packageInfo := map[string]any(nil)
|
||||
protocolVersion := byte(0)
|
||||
if versioned {
|
||||
headerLen = 17
|
||||
if len(raw) < headerLen {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: versioned header len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
protocolVersion = raw[4]
|
||||
phoneStart = 5
|
||||
phoneEnd = 15
|
||||
serialStart = 15
|
||||
}
|
||||
subpackage := props&0x2000 != 0
|
||||
if subpackage {
|
||||
headerLen += 4
|
||||
}
|
||||
if len(raw) < headerLen+int(bodySize)+1 {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
|
||||
}
|
||||
if subpackage {
|
||||
packageInfo = map[string]any{
|
||||
"total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]),
|
||||
"index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]),
|
||||
}
|
||||
}
|
||||
|
||||
phoneBCD := raw[phoneStart:phoneEnd]
|
||||
phone := bcdString(phoneBCD)
|
||||
phoneTrimZero := strings.TrimLeft(phone, "0")
|
||||
if versioned && phoneTrimZero != "" {
|
||||
phone = phoneTrimZero
|
||||
}
|
||||
body := raw[headerLen : headerLen+int(bodySize)]
|
||||
parsed := map[string]any{
|
||||
"header": map[string]any{
|
||||
"message_id": fmt.Sprintf("0x%04X", messageID),
|
||||
"protocol_version": protocolVersion,
|
||||
"versioned": versioned,
|
||||
"properties": props,
|
||||
"body_size": bodySize,
|
||||
"phone_bcd_hex": strings.ToUpper(hex.EncodeToString(phoneBCD)),
|
||||
"phone": phone,
|
||||
"phone_trim_zero": phoneTrimZero,
|
||||
"sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
|
||||
"subpackage": subpackage,
|
||||
"package_info": packageInfo,
|
||||
"encryption_flags": (props >> 10) & 0x07,
|
||||
},
|
||||
}
|
||||
fields := map[string]any{}
|
||||
eventTimeMS := receivedAtMS
|
||||
|
||||
if messageID == 0x0200 {
|
||||
location, err := parseLocation(body)
|
||||
if err != nil {
|
||||
return envelope.FrameEnvelope{}, err
|
||||
}
|
||||
parsed["location"] = locationToMap(location)
|
||||
fields[envelope.FieldLatitude] = location.Latitude
|
||||
fields[envelope.FieldLongitude] = location.Longitude
|
||||
fields[envelope.FieldSpeedKMH] = location.SpeedKMH
|
||||
fields["altitude_m"] = location.AltitudeM
|
||||
fields["direction_deg"] = location.DirectionDeg
|
||||
fields["alarm_flag"] = location.AlarmFlag
|
||||
fields["status_flag"] = location.StatusFlag
|
||||
if !location.Time.IsZero() {
|
||||
eventTimeMS = location.Time.UnixMilli()
|
||||
fields["device_time"] = location.Time.Format(time.RFC3339)
|
||||
}
|
||||
if location.TotalMileageKM != nil {
|
||||
fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
|
||||
}
|
||||
} else if messageID == 0x0100 {
|
||||
registration, err := parseRegistration(body, versioned)
|
||||
if err != nil {
|
||||
return envelope.FrameEnvelope{}, err
|
||||
}
|
||||
parsed["registration"] = registration
|
||||
if deviceID, ok := registration["device_id"].(string); ok {
|
||||
fields["device_id"] = deviceID
|
||||
}
|
||||
if plate, ok := registration["plate"].(string); ok {
|
||||
fields["plate"] = plate
|
||||
}
|
||||
} else if messageID == 0x0102 {
|
||||
authentication := parseAuthentication(body, versioned)
|
||||
parsed["authentication"] = authentication
|
||||
if token, ok := authentication["token"].(string); ok {
|
||||
fields["auth_token"] = token
|
||||
}
|
||||
}
|
||||
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: fmt.Sprintf("0x%04X", messageID),
|
||||
Sequence: binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
|
||||
Phone: phone,
|
||||
SourceEndpoint: sourceEndpoint,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
if registration, ok := parsed["registration"].(map[string]any); ok {
|
||||
env.DeviceID, _ = registration["device_id"].(string)
|
||||
env.Plate, _ = registration["plate"].(string)
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func parseRegistration(body []byte, versioned bool) (map[string]any, error) {
|
||||
schemaVersion := "2011_2013"
|
||||
manufacturerSize := 5
|
||||
deviceTypeSize := 20
|
||||
deviceIDSize := 7
|
||||
if versioned {
|
||||
schemaVersion = "2019"
|
||||
manufacturerSize = 11
|
||||
deviceTypeSize = 30
|
||||
deviceIDSize = 30
|
||||
}
|
||||
minLen := 2 + 2 + manufacturerSize + deviceTypeSize + deviceIDSize + 1
|
||||
if len(body) < minLen {
|
||||
return nil, fmt.Errorf("%w: registration body len=%d", ErrFrameTooShort, len(body))
|
||||
}
|
||||
manufacturerStart := 4
|
||||
deviceTypeStart := manufacturerStart + manufacturerSize
|
||||
deviceIDStart := deviceTypeStart + deviceTypeSize
|
||||
plateColorStart := deviceIDStart + deviceIDSize
|
||||
registration := map[string]any{
|
||||
"schema_version": schemaVersion,
|
||||
"province": binary.BigEndian.Uint16(body[0:2]),
|
||||
"city": binary.BigEndian.Uint16(body[2:4]),
|
||||
"manufacturer": fixedText(body[manufacturerStart:deviceTypeStart]),
|
||||
"device_type": fixedText(body[deviceTypeStart:deviceIDStart]),
|
||||
"device_id": fixedText(body[deviceIDStart:plateColorStart]),
|
||||
"plate_color": body[plateColorStart],
|
||||
"plate": freeText(body[plateColorStart+1:]),
|
||||
}
|
||||
return registration, nil
|
||||
}
|
||||
|
||||
func parseAuthentication(body []byte, versioned bool) map[string]any {
|
||||
if versioned {
|
||||
if out, ok := parseVersionedAuthentication(body); ok {
|
||||
return out
|
||||
}
|
||||
if out, ok := parseDelimitedVersionedAuthentication(body); ok {
|
||||
return out
|
||||
}
|
||||
}
|
||||
return map[string]any{"token": freeText(body)}
|
||||
}
|
||||
|
||||
func parseVersionedAuthentication(body []byte) (map[string]any, bool) {
|
||||
if len(body) < 17 {
|
||||
return nil, false
|
||||
}
|
||||
tokenLen := int(body[0])
|
||||
imeiStart := 1 + tokenLen
|
||||
softwareLenStart := imeiStart + 15
|
||||
if tokenLen <= 0 || len(body) < softwareLenStart+1 {
|
||||
return nil, false
|
||||
}
|
||||
softwareLen := int(body[softwareLenStart])
|
||||
softwareStart := softwareLenStart + 1
|
||||
if len(body) < softwareStart+softwareLen {
|
||||
return nil, false
|
||||
}
|
||||
return map[string]any{
|
||||
"schema_version": "2019",
|
||||
"token": freeText(body[1:imeiStart]),
|
||||
"imei": fixedText(body[imeiStart:softwareLenStart]),
|
||||
"software_version": freeText(body[softwareStart : softwareStart+softwareLen]),
|
||||
}, true
|
||||
}
|
||||
|
||||
func parseDelimitedVersionedAuthentication(body []byte) (map[string]any, bool) {
|
||||
parts := bytes.Split(body, []byte{0xff, 0xff})
|
||||
if len(parts) < 2 {
|
||||
return nil, false
|
||||
}
|
||||
out := map[string]any{
|
||||
"schema_version": "2019_delimited",
|
||||
"token": freeText(parts[0]),
|
||||
"imei": freeText(parts[1]),
|
||||
}
|
||||
if len(parts) > 2 {
|
||||
out["software_version"] = freeText(parts[2])
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
func parseLocation(body []byte) (Location, error) {
|
||||
if len(body) < 28 {
|
||||
return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
|
||||
}
|
||||
status := binary.BigEndian.Uint32(body[4:8])
|
||||
latitude := float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000
|
||||
longitude := float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000
|
||||
if status&(1<<2) != 0 {
|
||||
latitude = -latitude
|
||||
}
|
||||
if status&(1<<3) != 0 {
|
||||
longitude = -longitude
|
||||
}
|
||||
location := Location{
|
||||
AlarmFlag: binary.BigEndian.Uint32(body[0:4]),
|
||||
StatusFlag: status,
|
||||
Latitude: latitude,
|
||||
Longitude: longitude,
|
||||
AltitudeM: binary.BigEndian.Uint16(body[16:18]),
|
||||
SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10,
|
||||
DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
|
||||
Time: parseBCDTime(body[22:28]),
|
||||
}
|
||||
location.Additional = parseAdditional(body[28:], &location)
|
||||
return location, nil
|
||||
}
|
||||
|
||||
func parseAdditional(data []byte, location *Location) []AdditionalItem {
|
||||
var out []AdditionalItem
|
||||
for len(data) >= 2 {
|
||||
id := data[0]
|
||||
size := int(data[1])
|
||||
data = data[2:]
|
||||
if len(data) < size {
|
||||
out = append(out, AdditionalItem{
|
||||
ID: id,
|
||||
Length: size,
|
||||
ValueHex: strings.ToUpper(hex.EncodeToString(data)),
|
||||
Parsed: "truncated",
|
||||
})
|
||||
return out
|
||||
}
|
||||
value := data[:size]
|
||||
item := AdditionalItem{
|
||||
ID: id,
|
||||
Length: size,
|
||||
ValueHex: strings.ToUpper(hex.EncodeToString(value)),
|
||||
}
|
||||
switch {
|
||||
case id == 0x01 && size == 4:
|
||||
rawMileage := binary.BigEndian.Uint32(value)
|
||||
mileage := float64(rawMileage) / 10
|
||||
location.TotalMileageKM = &mileage
|
||||
item.Parsed = map[string]any{
|
||||
"name": envelope.FieldTotalMileageKM,
|
||||
"raw_value_tenth_km": rawMileage,
|
||||
"value": mileage,
|
||||
"unit": "km",
|
||||
}
|
||||
case id == 0x02 && size == 2:
|
||||
rawFuel := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "fuel_l",
|
||||
"raw_value_tenth_l": rawFuel,
|
||||
"value": float64(rawFuel) / 10,
|
||||
"unit": "L",
|
||||
}
|
||||
case id == 0x03 && size == 2:
|
||||
rawSpeed := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "recorder_speed_kmh",
|
||||
"raw_value_tenth_kmh": rawSpeed,
|
||||
"value": float64(rawSpeed) / 10,
|
||||
"unit": "km/h",
|
||||
}
|
||||
case id == 0x04 && size == 2:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "manual_alarm_event_id",
|
||||
"value": binary.BigEndian.Uint16(value),
|
||||
}
|
||||
case id == 0x05 && size == 30:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "tire_pressure",
|
||||
"values": parseTirePressure(value),
|
||||
}
|
||||
case id == 0x06 && size == 2:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "carriage_temperature_c",
|
||||
"value": int16(binary.BigEndian.Uint16(value)),
|
||||
"unit": "C",
|
||||
}
|
||||
case id == 0x11 && (size == 1 || size == 5):
|
||||
parsed := map[string]any{
|
||||
"name": "overspeed_alarm",
|
||||
"location_type": value[0],
|
||||
}
|
||||
if size == 5 {
|
||||
parsed["area_id"] = binary.BigEndian.Uint32(value[1:5])
|
||||
}
|
||||
item.Parsed = parsed
|
||||
case id == 0x12 && size == 6:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "area_route_alarm",
|
||||
"location_type": value[0],
|
||||
"area_id": binary.BigEndian.Uint32(value[1:5]),
|
||||
"direction": value[5],
|
||||
}
|
||||
case id == 0x13 && size == 7:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "road_section_driving_time_alarm",
|
||||
"road_section_id": binary.BigEndian.Uint32(value[0:4]),
|
||||
"driving_time_seconds": binary.BigEndian.Uint16(value[4:6]),
|
||||
"result": value[6],
|
||||
"result_description": roadSectionDrivingTimeResult(value[6]),
|
||||
"driving_time_result_source": "jt808_0x0200_additional_0x13",
|
||||
}
|
||||
case id == 0x25 && size == 4:
|
||||
rawStatus := binary.BigEndian.Uint32(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "extended_vehicle_signal_status",
|
||||
"value": rawStatus,
|
||||
"bits": parseExtendedVehicleSignal(rawStatus),
|
||||
}
|
||||
case id == 0x2a && size == 2:
|
||||
rawStatus := binary.BigEndian.Uint16(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "io_status",
|
||||
"value": rawStatus,
|
||||
"bits": parseIOStatus(rawStatus),
|
||||
}
|
||||
case id == 0x2b && size == 4:
|
||||
analog := binary.BigEndian.Uint32(value)
|
||||
item.Parsed = map[string]any{
|
||||
"name": "analog",
|
||||
"ad0": analog & 0xffff,
|
||||
"ad1": analog >> 16,
|
||||
"value": analog,
|
||||
}
|
||||
case id == 0x30 && size == 1:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "network_signal_strength",
|
||||
"value": value[0],
|
||||
}
|
||||
case id == 0x31 && size == 1:
|
||||
item.Parsed = map[string]any{
|
||||
"name": "gnss_satellite_count",
|
||||
"value": value[0],
|
||||
}
|
||||
}
|
||||
out = append(out, item)
|
||||
data = data[size:]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func locationToMap(location Location) map[string]any {
|
||||
out := map[string]any{
|
||||
"alarm_flag": location.AlarmFlag,
|
||||
"status_flag": location.StatusFlag,
|
||||
"latitude": location.Latitude,
|
||||
"longitude": location.Longitude,
|
||||
"altitude_m": location.AltitudeM,
|
||||
"speed_kmh": location.SpeedKMH,
|
||||
"direction_deg": location.DirectionDeg,
|
||||
"additional": additionalToMaps(location.Additional),
|
||||
}
|
||||
if !location.Time.IsZero() {
|
||||
out["device_time"] = location.Time.Format(time.RFC3339)
|
||||
}
|
||||
if location.TotalMileageKM != nil {
|
||||
out[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func additionalToMaps(items []AdditionalItem) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(items))
|
||||
for _, item := range items {
|
||||
value := map[string]any{
|
||||
"id": fmt.Sprintf("0x%02X", item.ID),
|
||||
"length": item.Length,
|
||||
"value_hex": item.ValueHex,
|
||||
}
|
||||
if item.Parsed != nil {
|
||||
value["parsed"] = item.Parsed
|
||||
}
|
||||
out = append(out, value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseTirePressure(data []byte) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(data))
|
||||
for index, value := range data {
|
||||
out = append(out, map[string]any{
|
||||
"position": index,
|
||||
"value": value,
|
||||
"valid": value != 0xff,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseExtendedVehicleSignal(value uint32) map[string]bool {
|
||||
return map[string]bool{
|
||||
"low_beam": value&0x0001 != 0,
|
||||
"high_beam": value&0x0002 != 0,
|
||||
"right_turn": value&0x0004 != 0,
|
||||
"left_turn": value&0x0008 != 0,
|
||||
"brake": value&0x0010 != 0,
|
||||
"reverse_gear": value&0x0020 != 0,
|
||||
"fog_light": value&0x0040 != 0,
|
||||
"clearance_light": value&0x0080 != 0,
|
||||
"horn": value&0x0100 != 0,
|
||||
"air_conditioner": value&0x0200 != 0,
|
||||
"neutral": value&0x0400 != 0,
|
||||
"retarder": value&0x0800 != 0,
|
||||
"abs": value&0x1000 != 0,
|
||||
"heater": value&0x2000 != 0,
|
||||
"clutch": value&0x4000 != 0,
|
||||
}
|
||||
}
|
||||
|
||||
func parseIOStatus(value uint16) map[string]bool {
|
||||
return map[string]bool{
|
||||
"deep_sleep": value&0x0001 != 0,
|
||||
"sleep": value&0x0002 != 0,
|
||||
}
|
||||
}
|
||||
|
||||
func roadSectionDrivingTimeResult(value byte) string {
|
||||
if value == 0 {
|
||||
return "insufficient"
|
||||
}
|
||||
if value == 1 {
|
||||
return "too_long"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func bcdString(data []byte) string {
|
||||
out := make([]byte, 0, len(data)*2)
|
||||
for _, value := range data {
|
||||
out = append(out, '0'+((value>>4)&0x0f), '0'+(value&0x0f))
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func fixedText(data []byte) string {
|
||||
return strings.Trim(freeText(data), "\x00 ")
|
||||
}
|
||||
|
||||
func freeText(data []byte) string {
|
||||
data = bytes.Trim(data, "\x00 ")
|
||||
if len(data) == 0 {
|
||||
return ""
|
||||
}
|
||||
if utf8.Valid(data) {
|
||||
return string(data)
|
||||
}
|
||||
decoded, err := simplifiedchinese.GBK.NewDecoder().Bytes(data)
|
||||
if err == nil && utf8.Valid(decoded) {
|
||||
return string(decoded)
|
||||
}
|
||||
return strings.ToValidUTF8(string(data), "")
|
||||
}
|
||||
|
||||
func parseBCDTime(data []byte) time.Time {
|
||||
if len(data) != 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
year := 2000 + bcdByte(data[0])
|
||||
month := time.Month(bcdByte(data[1]))
|
||||
day := bcdByte(data[2])
|
||||
hour := bcdByte(data[3])
|
||||
minute := bcdByte(data[4])
|
||||
second := bcdByte(data[5])
|
||||
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
}
|
||||
|
||||
func bcdByte(value byte) int {
|
||||
return int(value>>4)*10 + int(value&0x0f)
|
||||
}
|
||||
|
||||
func unescape(data []byte) ([]byte, error) {
|
||||
out := make([]byte, 0, len(data))
|
||||
for i := 0; i < len(data); i++ {
|
||||
if data[i] != 0x7d {
|
||||
out = append(out, data[i])
|
||||
continue
|
||||
}
|
||||
if i+1 >= len(data) {
|
||||
return nil, ErrEscape
|
||||
}
|
||||
i++
|
||||
switch data[i] {
|
||||
case 0x01:
|
||||
out = append(out, 0x7d)
|
||||
case 0x02:
|
||||
out = append(out, 0x7e)
|
||||
default:
|
||||
return nil, ErrEscape
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func checksum(data []byte) byte {
|
||||
var out byte
|
||||
for _, value := range data {
|
||||
out ^= value
|
||||
}
|
||||
return out
|
||||
}
|
||||
503
go/vehicle-gateway/internal/protocol/jt808/parser_test.go
Normal file
503
go/vehicle-gateway/internal/protocol/jt808/parser_test.go
Normal file
@@ -0,0 +1,503 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
|
||||
const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E"
|
||||
const fullAdditionalLocationFrame = "7e020000800123456789017fff000004000000080006eeb6ad02633df701380003006320070719235901040000000b02020016030200210402002c051e3737370000000000000000000000000000000000000000000000000000001105420000004212064d0000004d4d1307000000580058582504000000632a02000a2b040000001430011e31012806020001927e"
|
||||
|
||||
func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
|
||||
stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 {
|
||||
t.Fatalf("expected one complete frame, got %d", len(frames))
|
||||
}
|
||||
if hex.EncodeToString(remainder) != "7e02000000" {
|
||||
t.Fatalf("expected partial remainder, got %s", hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782745114999, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolJT808 {
|
||||
t.Fatalf("protocol = %q", env.Protocol)
|
||||
}
|
||||
if env.MessageID != "0x0200" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "013307795425" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.Sequence != 1 {
|
||||
t.Fatalf("sequence = %d", env.Sequence)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.590151)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121.069881)
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 23.0)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10241.2)
|
||||
if env.Fields["direction_deg"] != uint16(79) {
|
||||
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
|
||||
}
|
||||
|
||||
location, ok := env.Parsed["location"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("parsed location missing: %#v", env.Parsed)
|
||||
}
|
||||
additional, ok := location["additional"].([]map[string]any)
|
||||
if !ok || len(additional) == 0 {
|
||||
t.Fatalf("additional fields missing: %#v", location["additional"])
|
||||
}
|
||||
if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
|
||||
t.Fatalf("unexpected first additional item: %#v", additional[0])
|
||||
}
|
||||
parsed, ok := additional[0]["parsed"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("mileage parsed field missing: %#v", additional[0])
|
||||
}
|
||||
if parsed["raw_value_tenth_km"] != uint32(102412) || parsed["value"] != 10241.2 {
|
||||
t.Fatalf("unexpected mileage parsed value: %#v", parsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
|
||||
stream, err := hex.DecodeString(real2019LocationFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782914967713, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0200" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "14894135060" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.Sequence != 1995 {
|
||||
t.Fatalf("sequence = %d", env.Sequence)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.570155)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 117.178187)
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 33.4)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 7868.9)
|
||||
if env.Fields["altitude_m"] != uint16(13) {
|
||||
t.Fatalf("altitude = %#v", env.Fields["altitude_m"])
|
||||
}
|
||||
if env.Fields["direction_deg"] != uint16(232) {
|
||||
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
|
||||
}
|
||||
|
||||
header, ok := env.Parsed["header"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("parsed header missing: %#v", env.Parsed)
|
||||
}
|
||||
if header["versioned"] != true || header["protocol_version"] != byte(1) {
|
||||
t.Fatalf("unexpected versioned header: %#v", header)
|
||||
}
|
||||
if header["phone_bcd_hex"] != "00000000014894135060" {
|
||||
t.Fatalf("phone bcd = %#v", header["phone_bcd_hex"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesCommonLocationAdditionalItems(t *testing.T) {
|
||||
stream, err := hex.DecodeString(fullAdditionalLocationFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
|
||||
env, err := ParseFrame(frames[0], 1782914967713, "127.0.0.1:808")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Phone != "012345678901" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.1)
|
||||
|
||||
location := env.Parsed["location"].(map[string]any)
|
||||
additional := location["additional"].([]map[string]any)
|
||||
byID := map[string]map[string]any{}
|
||||
for _, item := range additional {
|
||||
byID[item["id"].(string)] = item["parsed"].(map[string]any)
|
||||
}
|
||||
if byID["0x01"]["raw_value_tenth_km"] != uint32(11) || byID["0x01"]["value"] != 1.1 {
|
||||
t.Fatalf("mileage parsed = %#v", byID["0x01"])
|
||||
}
|
||||
if byID["0x05"]["name"] != "tire_pressure" {
|
||||
t.Fatalf("tire pressure missing: %#v", byID["0x05"])
|
||||
}
|
||||
tireValues := byID["0x05"]["values"].([]map[string]any)
|
||||
if tireValues[0]["value"] != byte(0x37) || tireValues[0]["valid"] != true {
|
||||
t.Fatalf("tire pressure first value = %#v", tireValues[0])
|
||||
}
|
||||
if byID["0x11"]["location_type"] != byte(0x42) || byID["0x11"]["area_id"] != uint32(0x42) {
|
||||
t.Fatalf("overspeed parsed = %#v", byID["0x11"])
|
||||
}
|
||||
if byID["0x12"]["location_type"] != byte(0x4d) || byID["0x12"]["area_id"] != uint32(0x4d) || byID["0x12"]["direction"] != byte(0x4d) {
|
||||
t.Fatalf("area route parsed = %#v", byID["0x12"])
|
||||
}
|
||||
if byID["0x13"]["road_section_id"] != uint32(0x58) || byID["0x13"]["driving_time_seconds"] != uint16(0x58) {
|
||||
t.Fatalf("road section parsed = %#v", byID["0x13"])
|
||||
}
|
||||
ioBits := byID["0x2A"]["bits"].(map[string]bool)
|
||||
if ioBits["deep_sleep"] || !ioBits["sleep"] {
|
||||
t.Fatalf("io bits = %#v", ioBits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameRejectsTruncatedSubpackageHeader(t *testing.T) {
|
||||
payload := []byte{0x02, 0x00, 0x20, 0x00}
|
||||
payload = append(payload, encodeBCD("013307795425", 6)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, 1)
|
||||
payload = append(payload, checksum(payload))
|
||||
|
||||
_, err := ParseFrame(payload, 1782914967713, "115.231.168.135:43625")
|
||||
if err == nil {
|
||||
t.Fatal("expected truncated subpackage header to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLocationAppliesSouthWestStatusBits(t *testing.T) {
|
||||
body := make([]byte, 28)
|
||||
binary.BigEndian.PutUint32(body[4:8], (1<<2)|(1<<3))
|
||||
binary.BigEndian.PutUint32(body[8:12], 12345678)
|
||||
binary.BigEndian.PutUint32(body[12:16], 98765432)
|
||||
|
||||
location, err := parseLocation(body)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLocation() error = %v", err)
|
||||
}
|
||||
if location.Latitude != -12.345678 {
|
||||
t.Fatalf("latitude = %v", location.Latitude)
|
||||
}
|
||||
if location.Longitude != -98.765432 {
|
||||
t.Fatalf("longitude = %v", location.Longitude)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0001", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST123")...)
|
||||
|
||||
env, err := ParseFrame(buildFrame(0x0100, "013079963379", 7, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0100" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
if env.Phone != "013079963379" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.DeviceID != "DEV0001" {
|
||||
t.Fatalf("device id = %q", env.DeviceID)
|
||||
}
|
||||
if env.Plate != "TEST123" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
registration, ok := env.Parsed["registration"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("registration missing: %#v", env.Parsed)
|
||||
}
|
||||
if registration["manufacturer"] != "YUTNG" || registration["device_type"] != "ZK6105CHEVNPG4" {
|
||||
t.Fatalf("registration fields = %#v", registration)
|
||||
}
|
||||
if registration["plate_color"] != uint8(2) {
|
||||
t.Fatalf("plate color = %#v", registration["plate_color"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesJT8082019RegistrationIdentity(t *testing.T) {
|
||||
body := make([]byte, 0, 84)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTONG00001", 11)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4-2019", 30)
|
||||
body = appendFixedASCII(body, "DEV201900000000000000000000001", 30)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST2019")...)
|
||||
|
||||
env, err := ParseFrame(buildVersionedFrame(0x0100, "14894135060", 11, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Phone != "14894135060" {
|
||||
t.Fatalf("phone = %q", env.Phone)
|
||||
}
|
||||
if env.DeviceID != "DEV201900000000000000000000001" {
|
||||
t.Fatalf("device id = %q", env.DeviceID)
|
||||
}
|
||||
if env.Plate != "TEST2019" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
registration, ok := env.Parsed["registration"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("registration missing: %#v", env.Parsed)
|
||||
}
|
||||
if registration["schema_version"] != "2019" || registration["manufacturer"] != "YUTONG00001" {
|
||||
t.Fatalf("registration fields = %#v", registration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0002", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte{0xBB, 0xA6, 'A', '5', '3', '3', '0', '1'}...)
|
||||
|
||||
env, err := ParseFrame(buildFrame(0x0100, "013079963380", 9, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.Plate != "沪A53301" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesJT8082019Authentication(t *testing.T) {
|
||||
body := []byte{5}
|
||||
body = append(body, []byte("TOKEN")...)
|
||||
body = appendFixedASCII(body, "123456789012345", 15)
|
||||
body = append(body, 4)
|
||||
body = append(body, []byte("v1.2")...)
|
||||
|
||||
env, err := ParseFrame(buildVersionedFrame(0x0102, "14894135060", 12, body), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
auth, ok := env.Parsed["authentication"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("authentication missing: %#v", env.Parsed)
|
||||
}
|
||||
if auth["token"] != "TOKEN" || auth["imei"] != "123456789012345" || auth["software_version"] != "v1.2" {
|
||||
t.Fatalf("authentication = %#v", auth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFrameParsesAuthenticationToken(t *testing.T) {
|
||||
env, err := ParseFrame(buildFrame(0x0102, "064646848757", 8, []byte("g7gps")), 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFrame() error = %v", err)
|
||||
}
|
||||
if env.MessageID != "0x0102" {
|
||||
t.Fatalf("message id = %q", env.MessageID)
|
||||
}
|
||||
auth, ok := env.Parsed["authentication"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("authentication missing: %#v", env.Parsed)
|
||||
}
|
||||
if auth["token"] != "g7gps" {
|
||||
t.Fatalf("token = %#v", auth["token"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponderBuildsRegisterAck(t *testing.T) {
|
||||
body := make([]byte, 0, 46)
|
||||
body = binary.BigEndian.AppendUint16(body, 16)
|
||||
body = binary.BigEndian.AppendUint16(body, 32)
|
||||
body = appendFixedASCII(body, "YUTNG", 5)
|
||||
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
|
||||
body = appendFixedASCII(body, "DEV0001", 7)
|
||||
body = append(body, 2)
|
||||
body = append(body, []byte("TEST123")...)
|
||||
request := buildFrame(0x0100, "013079963379", 9, body)
|
||||
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := NewAutoResponder("g7gps").Respond(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("Respond() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(response)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
payload := frames[0]
|
||||
if binary.BigEndian.Uint16(payload[0:2]) != 0x8100 {
|
||||
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
|
||||
}
|
||||
if hex.EncodeToString(payload[4:10]) != "013079963379" {
|
||||
t.Fatalf("phone bcd = %x", payload[4:10])
|
||||
}
|
||||
if binary.BigEndian.Uint16(payload[12:14]) != 9 || payload[14] != 0 || string(payload[15:20]) != "g7gps" {
|
||||
t.Fatalf("unexpected register ack body: %x", payload[12:len(payload)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoResponderBuildsVersionedGeneralAck(t *testing.T) {
|
||||
request := buildVersionedFrame(0x0200, "14894135060", 12, make([]byte, 28))
|
||||
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response, ok, err := NewAutoResponder("").Respond(request, env)
|
||||
if err != nil {
|
||||
t.Fatalf("Respond() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected response")
|
||||
}
|
||||
frames, remainder, err := ExtractFrames(response)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
payload := frames[0]
|
||||
if binary.BigEndian.Uint16(payload[0:2]) != 0x8001 {
|
||||
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
|
||||
}
|
||||
if binary.BigEndian.Uint16(payload[2:4])&0x4000 == 0 || payload[4] != 1 {
|
||||
t.Fatalf("versioned header not preserved: %x", payload[:5])
|
||||
}
|
||||
if hex.EncodeToString(payload[5:15]) != "00000000014894135060" {
|
||||
t.Fatalf("phone bcd = %x", payload[5:15])
|
||||
}
|
||||
bodyStart := 17
|
||||
if binary.BigEndian.Uint16(payload[bodyStart:bodyStart+2]) != 12 ||
|
||||
binary.BigEndian.Uint16(payload[bodyStart+2:bodyStart+4]) != 0x0200 ||
|
||||
payload[bodyStart+4] != 0 {
|
||||
t.Fatalf("unexpected general ack body: %x", payload[bodyStart:len(payload)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
|
||||
payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
|
||||
frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
|
||||
frame = append(frame, 0x7e)
|
||||
|
||||
frames, remainder, err := ExtractFrames(frame)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractFrames() error = %v", err)
|
||||
}
|
||||
if len(frames) != 1 || len(remainder) != 0 {
|
||||
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
|
||||
}
|
||||
if hex.EncodeToString(frames[0][:len(payload)]) != hex.EncodeToString(payload) {
|
||||
t.Fatalf("frame was not unescaped: %s", hex.EncodeToString(frames[0]))
|
||||
}
|
||||
}
|
||||
|
||||
func buildFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
|
||||
payload := make([]byte, 0, 12+len(body)+1)
|
||||
payload = binary.BigEndian.AppendUint16(payload, messageID)
|
||||
payload = binary.BigEndian.AppendUint16(payload, uint16(len(body)))
|
||||
payload = append(payload, encodeBCD(phone, 6)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return payload
|
||||
}
|
||||
|
||||
func buildVersionedFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
|
||||
payload := make([]byte, 0, 17+len(body)+1)
|
||||
payload = binary.BigEndian.AppendUint16(payload, messageID)
|
||||
payload = binary.BigEndian.AppendUint16(payload, 0x4000|uint16(len(body)))
|
||||
payload = append(payload, 1)
|
||||
payload = append(payload, encodeBCD(phone, 10)...)
|
||||
payload = binary.BigEndian.AppendUint16(payload, sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return payload
|
||||
}
|
||||
|
||||
func appendFixedASCII(out []byte, value string, size int) []byte {
|
||||
start := len(out)
|
||||
out = append(out, make([]byte, size)...)
|
||||
copy(out[start:], []byte(value))
|
||||
return out
|
||||
}
|
||||
|
||||
func encodeBCD(value string, size int) []byte {
|
||||
out := make([]byte, size)
|
||||
if len(value)%2 == 1 {
|
||||
value = "0" + value
|
||||
}
|
||||
if len(value) > size*2 {
|
||||
value = value[len(value)-size*2:]
|
||||
}
|
||||
for len(value) < size*2 {
|
||||
value = "0" + value
|
||||
}
|
||||
for i := 0; i < size; i++ {
|
||||
out[i] = (value[i*2]-'0')<<4 | (value[i*2+1] - '0')
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func escape(payload []byte) []byte {
|
||||
var out []byte
|
||||
for _, value := range payload {
|
||||
switch value {
|
||||
case 0x7e:
|
||||
out = append(out, 0x7d, 0x02)
|
||||
case 0x7d:
|
||||
out = append(out, 0x7d, 0x01)
|
||||
default:
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
130
go/vehicle-gateway/internal/protocol/jt808/response.go
Normal file
130
go/vehicle-gateway/internal/protocol/jt808/response.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package jt808
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrResponseHeaderTooShort = errors.New("jt808 response header too short")
|
||||
|
||||
const (
|
||||
msgTerminalGeneralResponse = uint16(0x0001)
|
||||
msgTerminalRegister = uint16(0x0100)
|
||||
msgPlatformGeneralResponse = uint16(0x8001)
|
||||
msgPlatformRegisterAck = uint16(0x8100)
|
||||
)
|
||||
|
||||
type AutoResponder struct {
|
||||
authCode string
|
||||
}
|
||||
|
||||
func NewAutoResponder(authCode string) AutoResponder {
|
||||
if authCode == "" {
|
||||
authCode = "g7gps"
|
||||
}
|
||||
return AutoResponder{authCode: authCode}
|
||||
}
|
||||
|
||||
func (r AutoResponder) Respond(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return nil, false, nil
|
||||
}
|
||||
header, err := parseResponseHeader(raw)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if header.messageID == msgTerminalGeneralResponse {
|
||||
return nil, false, nil
|
||||
}
|
||||
if header.messageID == msgTerminalRegister {
|
||||
body := make([]byte, 3, 3+len(r.authCode))
|
||||
binary.BigEndian.PutUint16(body[0:2], header.sequence)
|
||||
body[2] = 0
|
||||
body = append(body, []byte(r.authCode)...)
|
||||
return encodeResponse(msgPlatformRegisterAck, header, body), true, nil
|
||||
}
|
||||
body := make([]byte, 5)
|
||||
binary.BigEndian.PutUint16(body[0:2], header.sequence)
|
||||
binary.BigEndian.PutUint16(body[2:4], header.messageID)
|
||||
body[4] = 0
|
||||
return encodeResponse(msgPlatformGeneralResponse, header, body), true, nil
|
||||
}
|
||||
|
||||
type responseHeader struct {
|
||||
messageID uint16
|
||||
versioned bool
|
||||
protocolVersion byte
|
||||
phoneBCD []byte
|
||||
sequence uint16
|
||||
}
|
||||
|
||||
func parseResponseHeader(raw []byte) (responseHeader, error) {
|
||||
if len(raw) < 12 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
props := binary.BigEndian.Uint16(raw[2:4])
|
||||
versioned := props&0x4000 != 0
|
||||
phoneStart := 4
|
||||
phoneLen := 6
|
||||
protocolVersion := byte(0)
|
||||
if versioned {
|
||||
if len(raw) < 17 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
protocolVersion = raw[4]
|
||||
phoneStart = 5
|
||||
phoneLen = 10
|
||||
}
|
||||
sequenceStart := phoneStart + phoneLen
|
||||
if len(raw) < sequenceStart+2 {
|
||||
return responseHeader{}, ErrResponseHeaderTooShort
|
||||
}
|
||||
phone := append([]byte(nil), raw[phoneStart:sequenceStart]...)
|
||||
return responseHeader{
|
||||
messageID: binary.BigEndian.Uint16(raw[0:2]),
|
||||
versioned: versioned,
|
||||
protocolVersion: protocolVersion,
|
||||
phoneBCD: phone,
|
||||
sequence: binary.BigEndian.Uint16(raw[sequenceStart : sequenceStart+2]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func encodeResponse(messageID uint16, header responseHeader, body []byte) []byte {
|
||||
props := uint16(len(body)) & 0x03ff
|
||||
if header.versioned {
|
||||
props |= 0x4000
|
||||
}
|
||||
payload := make([]byte, 0, 4+1+len(header.phoneBCD)+2+len(body)+1)
|
||||
payload = appendU16(payload, messageID)
|
||||
payload = appendU16(payload, props)
|
||||
if header.versioned {
|
||||
payload = append(payload, header.protocolVersion)
|
||||
}
|
||||
payload = append(payload, header.phoneBCD...)
|
||||
payload = appendU16(payload, header.sequence)
|
||||
payload = append(payload, body...)
|
||||
payload = append(payload, checksum(payload))
|
||||
return frame(payload)
|
||||
}
|
||||
|
||||
func appendU16(out []byte, value uint16) []byte {
|
||||
return append(out, byte(value>>8), byte(value))
|
||||
}
|
||||
|
||||
func frame(payload []byte) []byte {
|
||||
out := []byte{0x7e}
|
||||
for _, value := range payload {
|
||||
switch value {
|
||||
case 0x7e:
|
||||
out = append(out, 0x7d, 0x02)
|
||||
case 0x7d:
|
||||
out = append(out, 0x7d, 0x01)
|
||||
default:
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
out = append(out, 0x7e)
|
||||
return out
|
||||
}
|
||||
201
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
201
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package yutongmqtt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrInvalidJSON = errors.New("yutong mqtt invalid json")
|
||||
|
||||
func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS int64) (envelope.FrameEnvelope, error) {
|
||||
root := map[string]any{}
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&root); err != nil {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %v", ErrInvalidJSON, err)
|
||||
}
|
||||
|
||||
data := objectValue(root, "data")
|
||||
if data == nil {
|
||||
data = root
|
||||
}
|
||||
deviceID := firstText(root, "device", "deviceId", "terminalId", "terminalID", "imei", "IMEI", "mqttDeviceId")
|
||||
externalVIN := firstText(root, "vin", "VIN", "externalVin", "vehicleVin")
|
||||
vin := externalVIN
|
||||
if vin == "" && looksLikeVIN(deviceID) {
|
||||
vin = deviceID
|
||||
}
|
||||
plate := firstText(root, "plateNo", "carNo", "plate", "vehicleNo")
|
||||
phone := firstText(root, "sim", "phone", "mobile")
|
||||
eventTimeMS := parseTimeMS(firstText(root, "time", "deviceTime", "eventTime", "gpsTime"), receivedAtMS)
|
||||
fields := fieldsFromData(data)
|
||||
|
||||
parsed := map[string]any{
|
||||
"endpoint": endpoint,
|
||||
"topic": topic,
|
||||
"root": root,
|
||||
"data": data,
|
||||
}
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
MessageID: "MQTT",
|
||||
VIN: vin,
|
||||
VehicleKeyHint: deviceID,
|
||||
Phone: phone,
|
||||
DeviceID: deviceID,
|
||||
Plate: plate,
|
||||
SourceEndpoint: "mqtt://" + endpoint + topic,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawText: string(payload),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func fieldsFromData(data map[string]any) map[string]any {
|
||||
fields := map[string]any{}
|
||||
if speed, ok := firstFloat(data, "METER_SPEED", "speed", "speed_kmh"); ok {
|
||||
fields[envelope.FieldSpeedKMH] = speed
|
||||
}
|
||||
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
|
||||
fields[envelope.FieldTotalMileageKM] = normalizeTotalMileageKM(mileage)
|
||||
}
|
||||
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
|
||||
fields[envelope.FieldSOCPercent] = soc
|
||||
}
|
||||
if longitude, ok := firstCoord(data, "LONGITUDE", "longitude", "lng"); ok {
|
||||
fields[envelope.FieldLongitude] = longitude
|
||||
}
|
||||
if latitude, ok := firstCoord(data, "LATITUDE", "latitude", "lat"); ok {
|
||||
fields[envelope.FieldLatitude] = latitude
|
||||
}
|
||||
if direction, ok := firstFloat(data, "GPSDirection", "direction", "direction_deg"); ok {
|
||||
fields["direction_deg"] = direction
|
||||
}
|
||||
if gear, ok := firstFloat(data, "ON_GEAR", "gear"); ok {
|
||||
fields["gear"] = gear
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func objectValue(root map[string]any, key string) map[string]any {
|
||||
value, ok := root[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
typed, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return typed
|
||||
}
|
||||
|
||||
func firstText(root map[string]any, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value, ok := root[key]; ok && value != nil {
|
||||
text := strings.TrimSpace(fmt.Sprint(value))
|
||||
if text != "" && text != "<nil>" {
|
||||
return text
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstFloat(root map[string]any, keys ...string) (float64, bool) {
|
||||
for _, key := range keys {
|
||||
value, ok := root[key]
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
if parsed, ok := toFloat(value); ok {
|
||||
return parsed, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func firstCoord(root map[string]any, keys ...string) (float64, bool) {
|
||||
value, ok := firstFloat(root, keys...)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if value == 0 || int64(value) == 999999 {
|
||||
return 0, false
|
||||
}
|
||||
if value > 1000 || value < -1000 {
|
||||
value = value / 1_000_000
|
||||
}
|
||||
return value, true
|
||||
}
|
||||
|
||||
func toFloat(value any) (float64, bool) {
|
||||
switch typed := value.(type) {
|
||||
case json.Number:
|
||||
parsed, err := typed.Float64()
|
||||
return parsed, err == nil
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func parseTimeMS(raw string, fallback int64) int64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
for _, layout := range []string{
|
||||
"2006-01-02 15:04:05.000",
|
||||
"2006-01-02 15:04:05",
|
||||
} {
|
||||
if parsed, err := time.ParseInLocation(layout, raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
}
|
||||
if len(raw) == 14 {
|
||||
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
}
|
||||
if parsed, err := time.Parse(time.RFC3339, raw); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
if epoch, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
||||
if epoch > 10_000_000_000 {
|
||||
return epoch
|
||||
}
|
||||
return epoch * 1000
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func looksLikeVIN(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
return len(value) == 17
|
||||
}
|
||||
|
||||
func normalizeTotalMileageKM(value float64) float64 {
|
||||
return value / 1000
|
||||
}
|
||||
112
go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go
Normal file
112
go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package yutongmqtt
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device": "LTEST000000000001",
|
||||
"time": "20260413100000",
|
||||
"plateNo": "豫A12345",
|
||||
"data": {
|
||||
"METER_SPEED": 52.3,
|
||||
"TOTAL_MILEAGE": 123456.7,
|
||||
"BATTERY_CAPACITY_SOC": 70,
|
||||
"LONGITUDE": 116397128,
|
||||
"LATITUDE": 39916527,
|
||||
"GPSDirection": 88
|
||||
}
|
||||
}`)
|
||||
|
||||
env, err := ParseMessage("endpoint-a", "/ytforward/shln/dev1", payload, 1782745114999)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolYutongMQTT {
|
||||
t.Fatalf("protocol = %q", env.Protocol)
|
||||
}
|
||||
if env.VIN != "LTEST000000000001" || env.DeviceID != "LTEST000000000001" {
|
||||
t.Fatalf("identity fields = vin:%q device:%q", env.VIN, env.DeviceID)
|
||||
}
|
||||
if env.Plate != "豫A12345" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123.4567)
|
||||
assertFloatField(t, env, envelope.FieldSOCPercent, 70)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
|
||||
assertFloatField(t, env, "direction_deg", 88)
|
||||
if env.EventTimeMS == 1782745114999 {
|
||||
t.Fatal("event time should come from device time")
|
||||
}
|
||||
if env.RawText == "" || env.Parsed["data"] == nil {
|
||||
t.Fatalf("raw/parsed missing: %#v", env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
|
||||
payload := []byte(`{"device":"YT-DEVICE-001","time":"20260413100000","data":{"METER_SPEED":"12.3"}}`)
|
||||
env, err := ParseMessage("endpoint-a", "/ytforward/shln/YT-DEVICE-001", payload, 1782745114999)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if env.VIN != "" {
|
||||
t.Fatalf("non-vin device should not be treated as vin: %q", env.VIN)
|
||||
}
|
||||
if env.VehicleKey() != "YT-DEVICE-001" {
|
||||
t.Fatalf("vehicle key = %q", env.VehicleKey())
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
|
||||
}
|
||||
|
||||
func TestParseMessageNormalizesProductionMileageMeters(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device":"LMRKH9AC3R1004101",
|
||||
"time":"2026-07-02 01:02:04.085",
|
||||
"data":{"TOTAL_MILEAGE":56905000,"METER_SPEED":19}
|
||||
}`)
|
||||
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 56905)
|
||||
if env.EventTimeMS != 1782925324085 {
|
||||
t.Fatalf("event time = %d, want 1782925324085", env.EventTimeMS)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageTreatsSmallYutongMileageAsMeters(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device":"LMRKH9AC3R1004101",
|
||||
"time":"2026-07-02 01:02:04.085",
|
||||
"data":{"TOTAL_MILEAGE":1024,"METER_SPEED":19}
|
||||
}`)
|
||||
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.024)
|
||||
}
|
||||
|
||||
func TestParseMessageRejectsMalformedJSON(t *testing.T) {
|
||||
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
|
||||
if !errors.Is(err, ErrInvalidJSON) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
repository *Repository
|
||||
}
|
||||
|
||||
func NewHandler(repository *Repository) *Handler {
|
||||
if repository == nil {
|
||||
panic("realtime repository must not be nil")
|
||||
}
|
||||
return &Handler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
path := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/realtime/vehicles/"), "/")
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" {
|
||||
writeError(w, http.StatusNotFound, "vehicle not found")
|
||||
return
|
||||
}
|
||||
vin := parts[0]
|
||||
switch {
|
||||
case len(parts) == 1:
|
||||
snapshot, err := h.repository.GetMerged(r.Context(), vin)
|
||||
writeResult(w, snapshot, err)
|
||||
case len(parts) == 2 && parts[1] == "online":
|
||||
status, err := h.repository.IsOnline(r.Context(), vin)
|
||||
writeResult(w, status, err)
|
||||
case len(parts) == 3 && parts[1] == "protocols":
|
||||
snapshot, err := h.repository.GetProtocol(r.Context(), vin, envelope.Protocol(strings.ToUpper(parts[2])))
|
||||
writeResult(w, snapshot, err)
|
||||
default:
|
||||
writeError(w, http.StatusNotFound, "route not found")
|
||||
}
|
||||
}
|
||||
|
||||
func writeResult(w http.ResponseWriter, value any, err error) {
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
41
go/vehicle-gateway/internal/realtime/model.go
Normal file
41
go/vehicle-gateway/internal/realtime/model.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Snapshot struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Protocol envelope.Protocol `json:"protocol,omitempty"`
|
||||
Protocols []envelope.Protocol `json:"protocols,omitempty"`
|
||||
EventID string `json:"event_id,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
|
||||
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||
}
|
||||
|
||||
type OnlineStatus struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
Online bool `json:"online"`
|
||||
LastSeenMS int64 `json:"last_seen_ms"`
|
||||
Protocols []envelope.Protocol `json:"protocols,omitempty"`
|
||||
TTLSeconds int64 `json:"ttl_seconds"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
OnlineTTL time.Duration
|
||||
}
|
||||
|
||||
func (c Config) ttl() time.Duration {
|
||||
if c.OnlineTTL <= 0 {
|
||||
return 10 * time.Minute
|
||||
}
|
||||
return c.OnlineTTL
|
||||
}
|
||||
232
go/vehicle-gateway/internal/realtime/repository.go
Normal file
232
go/vehicle-gateway/internal/realtime/repository.go
Normal file
@@ -0,0 +1,232 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
client *redis.Client
|
||||
cfg Config
|
||||
}
|
||||
|
||||
func NewRepository(client *redis.Client, cfg Config) *Repository {
|
||||
if client == nil {
|
||||
panic("redis client must not be nil")
|
||||
}
|
||||
return &Repository{client: client, cfg: cfg}
|
||||
}
|
||||
|
||||
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
vehicleKey := strings.TrimSpace(env.VehicleKey())
|
||||
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
|
||||
return nil
|
||||
}
|
||||
nowMS := time.Now().UnixMilli()
|
||||
eventMS := env.EventTimeMS
|
||||
if eventMS <= 0 {
|
||||
eventMS = env.ReceivedAtMS
|
||||
}
|
||||
protocolSnapshot := Snapshot{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
EventID: env.StableEventID(),
|
||||
EventTimeMS: eventMS,
|
||||
ReceivedAtMS: env.ReceivedAtMS,
|
||||
SourceEndpoint: env.SourceEndpoint,
|
||||
Fields: cloneFields(env.Fields),
|
||||
FieldTimesMS: fieldTimes(env.Fields, eventMS),
|
||||
UpdatedAtMS: nowMS,
|
||||
}
|
||||
if err := r.setJSON(ctx, protocolKey(vehicleKey, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
protocols, err := r.addProtocol(ctx, vehicleKey, env.Protocol)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
merged, err := r.GetMerged(ctx, vehicleKey)
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
return err
|
||||
}
|
||||
if merged.VehicleKey == "" {
|
||||
merged = Snapshot{VehicleKey: vehicleKey, VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
|
||||
} else if merged.VIN == "" && vin != "" {
|
||||
merged.VIN = vin
|
||||
}
|
||||
mergeFields(&merged, env.Fields, eventMS)
|
||||
if eventMS >= merged.EventTimeMS {
|
||||
merged.EventTimeMS = eventMS
|
||||
merged.EventID = env.StableEventID()
|
||||
merged.ReceivedAtMS = env.ReceivedAtMS
|
||||
merged.SourceEndpoint = env.SourceEndpoint
|
||||
}
|
||||
merged.Protocols = protocols
|
||||
merged.UpdatedAtMS = nowMS
|
||||
if err := r.setJSON(ctx, mergedKey(vehicleKey), merged, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
online := OnlineStatus{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Online: true,
|
||||
LastSeenMS: env.ReceivedAtMS,
|
||||
Protocols: protocols,
|
||||
TTLSeconds: int64(r.cfg.ttl().Seconds()),
|
||||
}
|
||||
if err := r.setJSON(ctx, onlineKey(vehicleKey), online, r.cfg.ttl()); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vehicleKey}).Err()
|
||||
}
|
||||
|
||||
func (r *Repository) GetMerged(ctx context.Context, vehicleKey string) (Snapshot, error) {
|
||||
return r.getSnapshot(ctx, mergedKey(vehicleKey))
|
||||
}
|
||||
|
||||
func (r *Repository) GetProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) (Snapshot, error) {
|
||||
return r.getSnapshot(ctx, protocolKey(vehicleKey, protocol))
|
||||
}
|
||||
|
||||
func (r *Repository) IsOnline(ctx context.Context, vehicleKey string) (OnlineStatus, error) {
|
||||
vehicleKey = strings.TrimSpace(vehicleKey)
|
||||
var status OnlineStatus
|
||||
payload, err := r.client.Get(ctx, onlineKey(vehicleKey)).Bytes()
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}, nil
|
||||
}
|
||||
return status, err
|
||||
}
|
||||
if err := json.Unmarshal(payload, &status); err != nil {
|
||||
return status, err
|
||||
}
|
||||
ttl := r.client.TTL(ctx, onlineKey(vehicleKey)).Val()
|
||||
status.Online = ttl > 0
|
||||
status.TTLSeconds = int64(ttl.Seconds())
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func (r *Repository) getSnapshot(ctx context.Context, key string) (Snapshot, error) {
|
||||
var snapshot Snapshot
|
||||
payload, err := r.client.Get(ctx, key).Bytes()
|
||||
if err != nil {
|
||||
return snapshot, err
|
||||
}
|
||||
return snapshot, json.Unmarshal(payload, &snapshot)
|
||||
}
|
||||
|
||||
func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl time.Duration) error {
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return r.client.Set(ctx, key, payload, ttl).Err()
|
||||
}
|
||||
|
||||
func (r *Repository) addProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
|
||||
key := protocolsKey(vehicleKey)
|
||||
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = r.client.Expire(ctx, key, r.cfg.ttl()).Err()
|
||||
values, err := r.client.SMembers(ctx, key).Result()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(values)
|
||||
out := make([]envelope.Protocol, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, envelope.Protocol(value))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func mergeFields(snapshot *Snapshot, fields map[string]any, eventMS int64) {
|
||||
if snapshot.Fields == nil {
|
||||
snapshot.Fields = map[string]any{}
|
||||
}
|
||||
if snapshot.FieldTimesMS == nil {
|
||||
snapshot.FieldTimesMS = map[string]int64{}
|
||||
}
|
||||
for key, value := range fields {
|
||||
if key == envelope.FieldTotalMileageKM && !positiveNumber(value) {
|
||||
continue
|
||||
}
|
||||
if eventMS >= snapshot.FieldTimesMS[key] {
|
||||
snapshot.Fields[key] = value
|
||||
snapshot.FieldTimesMS[key] = eventMS
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func positiveNumber(value any) bool {
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed > 0
|
||||
case float32:
|
||||
return typed > 0
|
||||
case int:
|
||||
return typed > 0
|
||||
case int64:
|
||||
return typed > 0
|
||||
case uint16:
|
||||
return typed > 0
|
||||
case uint32:
|
||||
return typed > 0
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return err == nil && parsed > 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func cloneFields(fields map[string]any) map[string]any {
|
||||
if len(fields) == 0 {
|
||||
return map[string]any{}
|
||||
}
|
||||
out := make(map[string]any, len(fields))
|
||||
for key, value := range fields {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
|
||||
out := make(map[string]int64, len(fields))
|
||||
for key := range fields {
|
||||
out[key] = eventMS
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergedKey(vehicleKey string) string {
|
||||
return "vehicle:latest:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
|
||||
func protocolKey(vehicleKey string, protocol envelope.Protocol) string {
|
||||
return "vehicle:latest:" + strings.TrimSpace(vehicleKey) + ":" + string(protocol)
|
||||
}
|
||||
|
||||
func onlineKey(vehicleKey string) string {
|
||||
return "vehicle:online:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
|
||||
func protocolsKey(vehicleKey string) string {
|
||||
return "vehicle:protocols:" + strings.TrimSpace(vehicleKey)
|
||||
}
|
||||
250
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
250
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
@@ -0,0 +1,250 @@
|
||||
package realtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 121.1,
|
||||
envelope.FieldTotalMileageKM: 10.5,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 900,
|
||||
ReceivedAtMS: 1200,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 120.0,
|
||||
envelope.FieldLatitude: 30.5,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
merged, err := repo.GetMerged(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.Fields[envelope.FieldLongitude] != 121.1 {
|
||||
t.Fatalf("older longitude overwrote newer value: %#v", merged.Fields)
|
||||
}
|
||||
if merged.Fields[envelope.FieldLatitude] != 30.5 {
|
||||
t.Fatalf("new latitude missing: %#v", merged.Fields)
|
||||
}
|
||||
if got := protocolNames(merged.Protocols); strings.Join(got, ",") != "GB32960,JT808" {
|
||||
t.Fatalf("merged protocols = %#v", got)
|
||||
}
|
||||
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProtocol() error = %v", err)
|
||||
}
|
||||
if protocol.Fields[envelope.FieldTotalMileageKM] != 10.5 {
|
||||
t.Fatalf("protocol snapshot = %#v", protocol)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryOnlineStatus(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
status, err := repo.IsOnline(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if status.Online {
|
||||
t.Fatal("empty vin should be offline")
|
||||
}
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
status, err = repo.IsOnline(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if !status.Online || status.LastSeenMS != 1100 {
|
||||
t.Fatalf("online status = %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryUpdatesPhoneOnlyVehicleKey(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811170",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldSpeedKMH: 23.0,
|
||||
envelope.FieldTotalMileageKM: 10003.7,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
vehicleKey := "JT808:013307811170"
|
||||
merged, err := repo.GetMerged(ctx, vehicleKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.VehicleKey != vehicleKey || merged.VIN != "" {
|
||||
t.Fatalf("unexpected identity: %#v", merged)
|
||||
}
|
||||
if merged.Fields[envelope.FieldTotalMileageKM] != 10003.7 {
|
||||
t.Fatalf("merged fields = %#v", merged.Fields)
|
||||
}
|
||||
status, err := repo.IsOnline(ctx, vehicleKey)
|
||||
if err != nil {
|
||||
t.Fatalf("IsOnline() error = %v", err)
|
||||
}
|
||||
if !status.Online || status.VehicleKey != vehicleKey {
|
||||
t.Fatalf("online status = %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryDoesNotOverwritePositiveMileageWithZero(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 12345.6,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 2000,
|
||||
ReceivedAtMS: 2100,
|
||||
Fields: map[string]any{
|
||||
envelope.FieldSpeedKMH: 22.0,
|
||||
envelope.FieldTotalMileageKM: 0,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
merged, err := repo.GetMerged(ctx, "VIN001")
|
||||
if err != nil {
|
||||
t.Fatalf("GetMerged() error = %v", err)
|
||||
}
|
||||
if merged.Fields[envelope.FieldTotalMileageKM] != 12345.6 {
|
||||
t.Fatalf("zero mileage overwrote positive value: %#v", merged.Fields)
|
||||
}
|
||||
if merged.Fields[envelope.FieldSpeedKMH] != 22.0 {
|
||||
t.Fatalf("new speed should still merge: %#v", merged.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReturnsMergedSnapshot(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/VIN001", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
NewHandler(repo).ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !stringsContains(rec.Body.String(), `"vin":"VIN001"`) {
|
||||
t.Fatalf("unexpected body: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReturnsPhoneOnlyVehicleKeySnapshot(t *testing.T) {
|
||||
repo, closeFn := newTestRepository(t)
|
||||
defer closeFn()
|
||||
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811170",
|
||||
EventTimeMS: 1000,
|
||||
ReceivedAtMS: 1100,
|
||||
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||
}); err != nil {
|
||||
t.Fatalf("Update() error = %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/JT808:013307811170", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
NewHandler(repo).ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !stringsContains(rec.Body.String(), `"vehicle_key":"JT808:013307811170"`) {
|
||||
t.Fatalf("unexpected body: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRepository(t *testing.T) (*Repository, func()) {
|
||||
t.Helper()
|
||||
server, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("miniredis.Run() error = %v", err)
|
||||
}
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
return NewRepository(client, Config{OnlineTTL: time.Minute}), func() {
|
||||
_ = client.Close()
|
||||
server.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func stringsContains(value string, pattern string) bool {
|
||||
return strings.Contains(value, pattern)
|
||||
}
|
||||
|
||||
func protocolNames(protocols []envelope.Protocol) []string {
|
||||
names := make([]string, 0, len(protocols))
|
||||
for _, protocol := range protocols {
|
||||
names = append(names, string(protocol))
|
||||
}
|
||||
return names
|
||||
}
|
||||
275
go/vehicle-gateway/internal/stats/daily_metric.go
Normal file
275
go/vehicle-gateway/internal/stats/daily_metric.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const (
|
||||
MetricDailyMileageKM = "daily_mileage_km"
|
||||
MetricDailyTotalMileageKM = "daily_total_mileage_km"
|
||||
)
|
||||
|
||||
type Execer interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
exec Execer
|
||||
loc *time.Location
|
||||
}
|
||||
|
||||
type MetricSample struct {
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Protocol envelope.Protocol
|
||||
StatDate string
|
||||
MetricKey string
|
||||
MetricValue float64
|
||||
TotalMileageKM float64
|
||||
}
|
||||
|
||||
func NewWriter(exec Execer, loc *time.Location) *Writer {
|
||||
if exec == nil {
|
||||
panic("stats execer must not be nil")
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
return &Writer{exec: exec, loc: loc}
|
||||
}
|
||||
|
||||
func (w *Writer) EnsureSchema(ctx context.Context) error {
|
||||
if _, err := w.exec.ExecContext(ctx, DailyMetricTableSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
if db, ok := w.exec.(metadataDB); ok {
|
||||
return migrateDailyMetricTable(ctx, db)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
samples, err := SamplesFromEnvelope(env, w.loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, sample := range samples {
|
||||
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL,
|
||||
sample.VehicleKey,
|
||||
sample.VIN,
|
||||
sample.StatDate,
|
||||
string(sample.Protocol),
|
||||
sample.MetricKey,
|
||||
sample.MetricValue,
|
||||
sample.TotalMileageKM,
|
||||
sample.TotalMileageKM); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
vehicleKey := strings.TrimSpace(env.VehicleKey())
|
||||
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
|
||||
return nil, nil
|
||||
}
|
||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
if totalMileage <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
eventMS := env.EventTimeMS
|
||||
if eventMS <= 0 {
|
||||
eventMS = env.ReceivedAtMS
|
||||
}
|
||||
if eventMS <= 0 {
|
||||
return nil, errors.New("event or received time is required")
|
||||
}
|
||||
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
|
||||
return []MetricSample{
|
||||
{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
StatDate: statDate,
|
||||
MetricKey: MetricDailyMileageKM,
|
||||
MetricValue: 0,
|
||||
TotalMileageKM: totalMileage,
|
||||
},
|
||||
{
|
||||
VehicleKey: vehicleKey,
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
StatDate: statDate,
|
||||
MetricKey: MetricDailyTotalMileageKM,
|
||||
MetricValue: totalMileage,
|
||||
TotalMileageKM: totalMileage,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
const upsertDailyMetricSQL = `
|
||||
INSERT INTO vehicle_daily_metric
|
||||
(vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
|
||||
first_total_mileage_km = CASE
|
||||
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
|
||||
THEN VALUES(first_total_mileage_km)
|
||||
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
END,
|
||||
latest_total_mileage_km = CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
END,
|
||||
metric_value = CASE
|
||||
WHEN metric_key = 'daily_mileage_km'
|
||||
THEN GREATEST(
|
||||
CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE latest_total_mileage_km
|
||||
END,
|
||||
VALUES(latest_total_mileage_km)
|
||||
)
|
||||
- CASE
|
||||
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
|
||||
THEN VALUES(first_total_mileage_km)
|
||||
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
END
|
||||
WHEN metric_key = 'daily_total_mileage_km'
|
||||
THEN GREATEST(
|
||||
CASE
|
||||
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
|
||||
THEN VALUES(latest_total_mileage_km)
|
||||
ELSE latest_total_mileage_km
|
||||
END,
|
||||
VALUES(latest_total_mileage_km)
|
||||
)
|
||||
ELSE VALUES(metric_value)
|
||||
END,
|
||||
sample_count = sample_count + 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`
|
||||
|
||||
type metadataDB interface {
|
||||
Execer
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}
|
||||
|
||||
func migrateDailyMetricTable(ctx context.Context, db metadataDB) error {
|
||||
columnExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND column_name = 'vehicle_key'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !columnExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD COLUMN vehicle_key VARCHAR(96) NOT NULL DEFAULT '' AFTER id`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `UPDATE vehicle_daily_metric SET vehicle_key = vin WHERE vehicle_key = ''`); err != nil {
|
||||
return err
|
||||
}
|
||||
vehicleIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'uk_daily_metric_vehicle'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !vehicleIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key)`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
oldIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'uk_daily_metric'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if oldIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric DROP INDEX uk_daily_metric`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
vinIndexExists, err := informationSchemaExists(ctx, db, `
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'vehicle_daily_metric'
|
||||
AND index_name = 'idx_vin'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !vinIndexExists {
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD KEY idx_vin (vin)`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func informationSchemaExists(ctx context.Context, db metadataDB, query string) (bool, error) {
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, query).Scan(&count); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||
if env.Fields == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
162
go/vehicle-gateway/internal/stats/daily_metric_test.go
Normal file
162
go/vehicle-gateway/internal/stats/daily_metric_test.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "LNBVIN00000000001",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 8, 30, 0, 0, loc).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 10241.2,
|
||||
},
|
||||
}
|
||||
|
||||
samples, err := SamplesFromEnvelope(env, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 2 {
|
||||
t.Fatalf("sample count = %d", len(samples))
|
||||
}
|
||||
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
|
||||
t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
|
||||
}
|
||||
if samples[0].VehicleKey != "LNBVIN00000000001" {
|
||||
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
|
||||
}
|
||||
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 {
|
||||
t.Fatalf("unexpected daily total sample: %#v", samples[1])
|
||||
}
|
||||
if samples[0].StatDate != "2026-07-01" {
|
||||
t.Fatalf("stat date = %q", samples[0].StatDate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307811254",
|
||||
EventTimeMS: time.Date(2026, 7, 2, 0, 3, 0, 0, loc).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: 10985.7,
|
||||
},
|
||||
}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 2 {
|
||||
t.Fatalf("sample count = %d", len(samples))
|
||||
}
|
||||
if samples[0].VehicleKey != "JT808:013307811254" {
|
||||
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
|
||||
}
|
||||
if samples[0].VIN != "" {
|
||||
t.Fatalf("vin should stay empty for unresolved JT808 identity, got %q", samples[0].VIN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples without vehicle key, got %#v", samples)
|
||||
}
|
||||
|
||||
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "LNBVIN00000000002",
|
||||
Fields: map[string]any{},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope() error = %v", err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples without mileage, got %#v", samples)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeSkipsNonPositiveMileage(t *testing.T) {
|
||||
for _, value := range []any{0, 0.0, -1.0, "0"} {
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "LNBVIN00000000001",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: value,
|
||||
},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SamplesFromEnvelope(%#v) error = %v", value, err)
|
||||
}
|
||||
if len(samples) != 0 {
|
||||
t.Fatalf("expected no samples for non-positive mileage %#v, got %#v", value, samples)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
if err := writer.EnsureSchema(context.Background()); err != nil {
|
||||
t.Fatalf("EnsureSchema() error = %v", err)
|
||||
}
|
||||
if err := writer.Append(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "LNBVIN00000000002",
|
||||
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
|
||||
Fields: map[string]any{
|
||||
envelope.FieldTotalMileageKM: "10000.0",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") {
|
||||
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
|
||||
}
|
||||
if !strings.Contains(exec.calls[0].query, "vehicle_key") {
|
||||
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
|
||||
}
|
||||
if len(exec.calls) != 3 {
|
||||
t.Fatalf("exec calls = %d", len(exec.calls))
|
||||
}
|
||||
if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") {
|
||||
t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query)
|
||||
}
|
||||
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
|
||||
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
|
||||
}
|
||||
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
|
||||
t.Fatalf("first upsert arg should be vehicle_key, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type execCall struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type recordingExec struct {
|
||||
calls []execCall
|
||||
}
|
||||
|
||||
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||
return nil, nil
|
||||
}
|
||||
326
go/vehicle-gateway/internal/stats/query.go
Normal file
326
go/vehicle-gateway/internal/stats/query.go
Normal file
@@ -0,0 +1,326 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Queryer interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
type MetricQuery struct {
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Protocol string
|
||||
MetricKey string
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type MetricRow struct {
|
||||
VehicleKey string `json:"vehicle_key"`
|
||||
VIN string `json:"vin"`
|
||||
StatDate string `json:"stat_date"`
|
||||
Protocol string `json:"protocol"`
|
||||
MetricKey string `json:"metric_key"`
|
||||
MetricValue float64 `json:"metric_value"`
|
||||
MetricUnit string `json:"metric_unit"`
|
||||
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
|
||||
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
|
||||
SampleCount int64 `json:"sample_count"`
|
||||
CalculationMethod string `json:"calculation_method"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type MetricRepository struct {
|
||||
db Queryer
|
||||
}
|
||||
|
||||
func NewMetricRepository(db Queryer) *MetricRepository {
|
||||
if db == nil {
|
||||
panic("metric query db must not be nil")
|
||||
}
|
||||
return &MetricRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]MetricRow, error) {
|
||||
query = normalizeMetricQuery(query)
|
||||
sqlText, args := buildMetricSQL(query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]MetricRow, 0)
|
||||
for rows.Next() {
|
||||
var row MetricRow
|
||||
var statDate scanDate
|
||||
var createdAt scanDateTime
|
||||
var updatedAt scanDateTime
|
||||
var first sql.NullFloat64
|
||||
var latest sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
&row.VehicleKey,
|
||||
&row.VIN,
|
||||
&statDate,
|
||||
&row.Protocol,
|
||||
&row.MetricKey,
|
||||
&row.MetricValue,
|
||||
&row.MetricUnit,
|
||||
&first,
|
||||
&latest,
|
||||
&row.SampleCount,
|
||||
&row.CalculationMethod,
|
||||
&createdAt,
|
||||
&updatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.StatDate = statDate.String
|
||||
row.CreatedAt = createdAt.String
|
||||
row.UpdatedAt = updatedAt.String
|
||||
if first.Valid {
|
||||
row.FirstTotalMileageKM = &first.Float64
|
||||
}
|
||||
if latest.Valid {
|
||||
row.LatestTotalMileageKM = &latest.Float64
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, error) {
|
||||
query = normalizeMetricQuery(query)
|
||||
sqlText, args := buildMetricCountSQL(query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var total int64
|
||||
if rows.Next() {
|
||||
if err := rows.Scan(&total); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return total, rows.Err()
|
||||
}
|
||||
|
||||
func normalizeMetricQuery(query MetricQuery) MetricQuery {
|
||||
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
|
||||
query.VIN = strings.TrimSpace(query.VIN)
|
||||
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
query.MetricKey = strings.TrimSpace(query.MetricKey)
|
||||
query.DateFrom = strings.TrimSpace(query.DateFrom)
|
||||
query.DateTo = strings.TrimSpace(query.DateTo)
|
||||
if query.Limit <= 0 {
|
||||
query.Limit = 50
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func buildMetricSQL(query MetricQuery) (string, []any) {
|
||||
where, args := buildMetricWhere(query)
|
||||
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric`
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
|
||||
args = append(args, query.Limit, query.Offset)
|
||||
return sqlText, args
|
||||
}
|
||||
|
||||
func buildMetricCountSQL(query MetricQuery) (string, []any) {
|
||||
where, args := buildMetricWhere(query)
|
||||
sqlText := `SELECT COUNT(*) FROM vehicle_daily_metric`
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
return sqlText, args
|
||||
}
|
||||
|
||||
func buildMetricWhere(query MetricQuery) ([]string, []any) {
|
||||
var where []string
|
||||
var args []any
|
||||
add := func(clause string, value any) {
|
||||
where = append(where, clause)
|
||||
args = append(args, value)
|
||||
}
|
||||
if query.VIN != "" {
|
||||
add("vin = ?", query.VIN)
|
||||
}
|
||||
if query.VehicleKey != "" {
|
||||
add("vehicle_key = ?", query.VehicleKey)
|
||||
}
|
||||
if query.Protocol != "" {
|
||||
add("protocol = ?", query.Protocol)
|
||||
}
|
||||
if query.MetricKey != "" {
|
||||
add("metric_key = ?", query.MetricKey)
|
||||
}
|
||||
if query.DateFrom != "" {
|
||||
add("stat_date >= ?", query.DateFrom)
|
||||
}
|
||||
if query.DateTo != "" {
|
||||
add("stat_date <= ?", query.DateTo)
|
||||
}
|
||||
return where, args
|
||||
}
|
||||
|
||||
type MetricHandler struct {
|
||||
repository *MetricRepository
|
||||
}
|
||||
|
||||
func NewMetricHandler(repository *MetricRepository) *MetricHandler {
|
||||
if repository == nil {
|
||||
panic("metric repository must not be nil")
|
||||
}
|
||||
return &MetricHandler{repository: repository}
|
||||
}
|
||||
|
||||
func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
writeMetricError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if strings.Trim(r.URL.Path, "/") != "api/stats/daily-metrics" {
|
||||
writeMetricError(w, http.StatusNotFound, "route not found")
|
||||
return
|
||||
}
|
||||
query, err := parseMetricQuery(r)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
total, err := h.repository.Count(r.Context(), query)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
rows, err := h.repository.Query(r.Context(), query)
|
||||
if err != nil {
|
||||
writeMetricError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"limit": query.Limit,
|
||||
"offset": query.Offset,
|
||||
})
|
||||
}
|
||||
|
||||
func parseMetricQuery(r *http.Request) (MetricQuery, error) {
|
||||
values := r.URL.Query()
|
||||
limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit")
|
||||
if err != nil {
|
||||
return MetricQuery{}, err
|
||||
}
|
||||
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
|
||||
if err != nil {
|
||||
return MetricQuery{}, err
|
||||
}
|
||||
query := MetricQuery{
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Protocol: values.Get("protocol"),
|
||||
MetricKey: values.Get("metricKey"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDate(query.DateFrom) || !validDate(query.DateTo) {
|
||||
return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD")
|
||||
}
|
||||
return normalizeMetricQuery(query), nil
|
||||
}
|
||||
|
||||
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil || value < min || value > max {
|
||||
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func validDate(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
if len(value) != len("2006-01-02") {
|
||||
return false
|
||||
}
|
||||
for i, r := range value {
|
||||
switch i {
|
||||
case 4, 7:
|
||||
if r != '-' {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func writeMetricError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||
}
|
||||
|
||||
type scanDate struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDate) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02")
|
||||
return nil
|
||||
}
|
||||
|
||||
type scanDateTime struct {
|
||||
String string
|
||||
}
|
||||
|
||||
func (s *scanDateTime) Scan(value any) error {
|
||||
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatSQLTime(value any, layout string) string {
|
||||
switch typed := value.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case time.Time:
|
||||
return typed.Format(layout)
|
||||
case []byte:
|
||||
return strings.TrimSpace(string(typed))
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(typed))
|
||||
}
|
||||
}
|
||||
185
go/vehicle-gateway/internal/stats/query_test.go
Normal file
185
go/vehicle-gateway/internal/stats/query_test.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km",
|
||||
12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
))
|
||||
|
||||
repository := NewMetricRepository(db)
|
||||
rows, err := repository.Query(context.Background(), MetricQuery{
|
||||
VIN: "LKLG7C4E3NA774736",
|
||||
Protocol: "JT808",
|
||||
DateFrom: "2026-07-01",
|
||||
DateTo: "2026-07-01",
|
||||
Limit: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query() error = %v", err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("row count = %d", len(rows))
|
||||
}
|
||||
if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 {
|
||||
t.Fatalf("unexpected row: %#v", rows[0])
|
||||
}
|
||||
if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
|
||||
t.Fatalf("vehicle key = %q", rows[0].VehicleKey)
|
||||
}
|
||||
if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" {
|
||||
t.Fatalf("unexpected time formatting: %#v", rows[0])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km",
|
||||
53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25",
|
||||
))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":42`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("JT808:013307811254", "JT808").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("JT808:013307811254", "JT808", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}).AddRow(
|
||||
"JT808:013307811254", "", "2026-07-02", "JT808", MetricDailyTotalMileageKM, 10985.7, "km",
|
||||
10985.7, 10985.7, 1, "TOTAL_MILEAGE_DIFF", "2026-07-02 00:03:00", "2026-07-02 00:03:00",
|
||||
))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vehicleKey=JT808:013307811254&protocol=JT808", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"metric_value":10985.7`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("response missing %s: %s", want, body)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
|
||||
WithArgs("YUTONG_MQTT").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
|
||||
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
|
||||
WithArgs("YUTONG_MQTT", 50, 0).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
|
||||
"created_at", "updated_at",
|
||||
}))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?protocol=YUTONG_MQTT", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
|
||||
t.Fatalf("expected empty items array, got: %s", body)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricHandlerRejectsInvalidPagination(t *testing.T) {
|
||||
handler := NewMetricHandler(NewMetricRepository(&sql.DB{}))
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?limit=2001", nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
22
go/vehicle-gateway/internal/stats/schema.go
Normal file
22
go/vehicle-gateway/internal/stats/schema.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package stats
|
||||
|
||||
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vehicle_key VARCHAR(96) NOT NULL,
|
||||
vin VARCHAR(32) NOT NULL DEFAULT '',
|
||||
stat_date DATE NOT NULL,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,3) NOT NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key),
|
||||
KEY idx_vin (vin),
|
||||
KEY idx_stat_date (stat_date),
|
||||
KEY idx_protocol_metric (protocol, metric_key)
|
||||
)`
|
||||
162
tools/go_kafka_prod_smoke.py
Executable file
162
tools/go_kafka_prod_smoke.py
Executable file
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Kafka topic and consumer lag smoke checks for the Go native production stack."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
|
||||
DEFAULT_HOST = "114.55.58.251"
|
||||
DEFAULT_USER = "root"
|
||||
DEFAULT_BOOTSTRAP = "127.0.0.1:9092"
|
||||
DEFAULT_KAFKA_BIN = "/opt/kafka/current/bin"
|
||||
DEFAULT_TOPICS = [
|
||||
"vehicle.raw.go.gb32960.v1",
|
||||
"vehicle.raw.go.jt808.v1",
|
||||
"vehicle.raw.go.yutong-mqtt.v1",
|
||||
"vehicle.event.go.unified.v1",
|
||||
]
|
||||
DEFAULT_GROUPS = [
|
||||
"go-history-writer",
|
||||
"go-stat-writer",
|
||||
"go-realtime-api",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Check:
|
||||
name: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConsumerLag:
|
||||
group: str
|
||||
topic: str
|
||||
partition: int
|
||||
current_offset: int | None
|
||||
log_end_offset: int
|
||||
lag: int | None
|
||||
|
||||
|
||||
def parse_topic_list(output: str) -> set[str]:
|
||||
return {line.strip() for line in output.splitlines() if line.strip()}
|
||||
|
||||
|
||||
def topic_checks(existing: set[str], required: list[str]) -> list[Check]:
|
||||
checks: list[Check] = []
|
||||
for topic in required:
|
||||
if topic in existing:
|
||||
checks.append(Check("topic." + topic, "pass", "present"))
|
||||
else:
|
||||
checks.append(Check("topic." + topic, "fail", "missing"))
|
||||
return checks
|
||||
|
||||
|
||||
def parse_consumer_group_describe(output: str) -> list[ConsumerLag]:
|
||||
rows: list[ConsumerLag] = []
|
||||
for line in output.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 6 or parts[0] == "GROUP":
|
||||
continue
|
||||
group, topic = parts[0], parts[1]
|
||||
try:
|
||||
partition = int(parts[2])
|
||||
current = parse_optional_int(parts[3])
|
||||
log_end = int(parts[4])
|
||||
lag = parse_optional_int(parts[5])
|
||||
except ValueError:
|
||||
continue
|
||||
rows.append(ConsumerLag(group, topic, partition, current, log_end, lag))
|
||||
return rows
|
||||
|
||||
|
||||
def parse_optional_int(value: str) -> int | None:
|
||||
value = value.strip()
|
||||
if value == "-":
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
|
||||
def group_lag_check(group: str, rows: list[ConsumerLag], max_lag: int) -> Check:
|
||||
group_rows = [row for row in rows if row.group == group]
|
||||
if not group_rows:
|
||||
return Check("group." + group, "fail", "no rows")
|
||||
concrete_lags = [row.lag for row in group_rows if row.lag is not None]
|
||||
if not concrete_lags:
|
||||
return Check("group." + group, "fail", "no concrete lag rows")
|
||||
max_seen = max(concrete_lags)
|
||||
topics = sorted({row.topic for row in group_rows})
|
||||
status = "pass" if max_seen <= max_lag else "fail"
|
||||
return Check(
|
||||
"group." + group,
|
||||
status,
|
||||
"max_lag=" + str(max_seen) + "; threshold=" + str(max_lag) + "; topics=" + ",".join(topics),
|
||||
)
|
||||
|
||||
|
||||
def ssh(host: str, user: str, command: str, timeout: float) -> str:
|
||||
target = user + "@" + host if user else host
|
||||
completed = subprocess.run(
|
||||
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", target, command],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def remote_topic_command(kafka_bin: str, bootstrap: str) -> str:
|
||||
return kafka_bin.rstrip("/") + "/kafka-topics.sh --bootstrap-server " + bootstrap + " --list"
|
||||
|
||||
|
||||
def remote_group_command(kafka_bin: str, bootstrap: str, group: str) -> str:
|
||||
return kafka_bin.rstrip("/") + "/kafka-consumer-groups.sh --bootstrap-server " + bootstrap + " --describe --group " + group
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
|
||||
topic_output = ssh(args.host, args.user, remote_topic_command(args.kafka_bin, args.bootstrap), args.timeout)
|
||||
checks = topic_checks(parse_topic_list(topic_output), DEFAULT_TOPICS)
|
||||
for group in DEFAULT_GROUPS:
|
||||
group_output = ssh(args.host, args.user, remote_group_command(args.kafka_bin, args.bootstrap, group), args.timeout)
|
||||
checks.append(group_lag_check(group, parse_consumer_group_describe(group_output), args.max_lag))
|
||||
status = "fail" if any(check.status == "fail" for check in checks) else "pass"
|
||||
return status, checks
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
||||
parser.add_argument("--user", default=DEFAULT_USER)
|
||||
parser.add_argument("--bootstrap", default=DEFAULT_BOOTSTRAP)
|
||||
parser.add_argument("--kafka-bin", default=DEFAULT_KAFKA_BIN)
|
||||
parser.add_argument("--max-lag", type=int, default=100)
|
||||
parser.add_argument("--timeout", type=float, default=20.0)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
try:
|
||||
status, checks = run(args)
|
||||
except Exception as exc:
|
||||
status = "fail"
|
||||
checks = [Check("kafka.ssh", "fail", str(exc))]
|
||||
print(json.dumps({
|
||||
"status": status,
|
||||
"host": args.host,
|
||||
"bootstrap": args.bootstrap,
|
||||
"checks": [asdict(check) for check in checks],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
return 0 if status == "pass" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
253
tools/go_native_deploy.py
Normal file
253
tools/go_native_deploy.py
Normal file
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and deploy the Go native vehicle gateway release to ECS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
DEFAULT_APP_HOST = "115.29.187.205"
|
||||
DEFAULT_KAFKA_HOST = "114.55.58.251"
|
||||
DEFAULT_USER = "root"
|
||||
DEFAULT_RELEASE_ROOT = "/opt/lingniu-go-native"
|
||||
DEFAULT_GO_DIR = "go/vehicle-gateway"
|
||||
BINARIES = [
|
||||
("gateway", "./cmd/gateway"),
|
||||
("history-writer", "./cmd/history-writer"),
|
||||
("stat-writer", "./cmd/stat-writer"),
|
||||
("realtime-api", "./cmd/realtime-api"),
|
||||
]
|
||||
SERVICES = [
|
||||
"lingniu-go-gateway",
|
||||
"lingniu-go-history-writer",
|
||||
"lingniu-go-stat-writer",
|
||||
"lingniu-go-realtime-api",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuildCommand:
|
||||
output_name: str
|
||||
argv: list[str]
|
||||
env: list[str]
|
||||
cwd: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpoolStatus:
|
||||
files: int
|
||||
recent: int
|
||||
|
||||
|
||||
def git_short_sha(cwd: str) -> str:
|
||||
return run_capture(["git", "rev-parse", "--short=7", "HEAD"], cwd=cwd).strip()
|
||||
|
||||
|
||||
def build_commands(go_dir: str, output_dir: str) -> list[BuildCommand]:
|
||||
commands: list[BuildCommand] = []
|
||||
env = ["CGO_ENABLED=0", "GOOS=linux", "GOARCH=amd64"]
|
||||
for output_name, package in BINARIES:
|
||||
commands.append(BuildCommand(
|
||||
output_name=output_name,
|
||||
argv=[
|
||||
"go",
|
||||
"build",
|
||||
"-trimpath",
|
||||
"-ldflags=-s -w",
|
||||
"-o",
|
||||
str(pathlib.Path(output_dir) / output_name),
|
||||
package,
|
||||
],
|
||||
env=env,
|
||||
cwd=go_dir,
|
||||
))
|
||||
return commands
|
||||
|
||||
|
||||
def build_release(go_dir: str, output_dir: str) -> None:
|
||||
for command in build_commands(go_dir, output_dir):
|
||||
env = os.environ.copy()
|
||||
for item in command.env:
|
||||
key, value = item.split("=", 1)
|
||||
env[key] = value
|
||||
run(command.argv, cwd=command.cwd, env=env)
|
||||
|
||||
|
||||
def create_archive(source_dir: str, archive_path: str) -> None:
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
for binary, _ in BINARIES:
|
||||
archive.add(pathlib.Path(source_dir) / binary, arcname=binary)
|
||||
|
||||
|
||||
def remote_switch_command(release: str, release_root: str, remote_archive: str) -> str:
|
||||
if not safe_release_name(release):
|
||||
raise ValueError("release must contain only letters, numbers, dot, underscore, or dash")
|
||||
release_dir = release_root.rstrip("/") + "/releases/" + release
|
||||
root = release_root.rstrip("/")
|
||||
commands = [
|
||||
"set -euo pipefail",
|
||||
"mkdir -p " + shell_quote(release_dir),
|
||||
"tar -xzf " + shell_quote(remote_archive) + " -C " + shell_quote(release_dir),
|
||||
"chmod +x " + shell_quote(release_dir) + "/*",
|
||||
"ln -sfn " + shell_quote(release_dir) + " " + shell_quote(root + "/current"),
|
||||
"systemctl daemon-reload",
|
||||
]
|
||||
commands.extend("systemctl restart " + service for service in SERVICES)
|
||||
commands.append("rm -f " + shell_quote(remote_archive))
|
||||
return " && ".join(commands)
|
||||
|
||||
|
||||
def acceptance_command(app_host: str, kafka_host: str, date: str, timeout: float) -> list[str]:
|
||||
command = [
|
||||
"python3",
|
||||
"tools/go_prod_acceptance.py",
|
||||
"--app-host",
|
||||
app_host,
|
||||
"--kafka-host",
|
||||
kafka_host,
|
||||
"--timeout",
|
||||
str(int(timeout) if timeout == int(timeout) else timeout),
|
||||
]
|
||||
if date:
|
||||
command.extend(["--date", date])
|
||||
return command
|
||||
|
||||
|
||||
def deploy(args: argparse.Namespace) -> None:
|
||||
repo = pathlib.Path(args.repo).resolve()
|
||||
go_dir = str(repo / args.go_dir)
|
||||
release = args.release or git_short_sha(str(repo))
|
||||
with tempfile.TemporaryDirectory(prefix="lingniu-go-native-") as tmp:
|
||||
output_dir = pathlib.Path(tmp) / "out"
|
||||
output_dir.mkdir()
|
||||
archive_path = pathlib.Path(tmp) / ("lingniu-go-native-" + release + ".tar.gz")
|
||||
build_release(go_dir, str(output_dir))
|
||||
create_archive(str(output_dir), str(archive_path))
|
||||
remote_archive = "/tmp/" + archive_path.name
|
||||
target = ssh_target(args.user, args.app_host)
|
||||
run(["scp", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", str(archive_path), target + ":" + remote_archive])
|
||||
run([
|
||||
"ssh",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/dev/null",
|
||||
target,
|
||||
remote_switch_command(release, args.release_root, remote_archive),
|
||||
])
|
||||
wait_for_spool_drain(target, args.release_root, args.spool_drain_timeout, args.spool_poll_interval)
|
||||
if not args.skip_acceptance:
|
||||
run(acceptance_command(args.app_host, args.kafka_host, args.date, args.timeout), cwd=str(repo))
|
||||
|
||||
|
||||
def wait_for_spool_drain(target: str, release_root: str, timeout: float, poll_interval: float) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
last = SpoolStatus(files=-1, recent=-1)
|
||||
while time.monotonic() <= deadline:
|
||||
output = run_capture([
|
||||
"ssh",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/dev/null",
|
||||
target,
|
||||
spool_status_command(release_root),
|
||||
])
|
||||
last = parse_spool_status(output)
|
||||
print(f"spool files={last.files} recent={last.recent}", flush=True)
|
||||
if last.files == 0:
|
||||
return
|
||||
time.sleep(max(1.0, poll_interval))
|
||||
raise RuntimeError(f"gateway spool did not drain: files={last.files} recent={last.recent}")
|
||||
|
||||
|
||||
def spool_status_command(release_root: str) -> str:
|
||||
spool_dir = release_root.rstrip("/") + "/spool/gateway"
|
||||
return (
|
||||
"spool=" + shell_quote(spool_dir) + "; "
|
||||
"files=$(find \"$spool\" -type f 2>/dev/null | wc -l); "
|
||||
"recent=$(find \"$spool\" -type f -mmin -5 2>/dev/null | wc -l); "
|
||||
"printf 'files=%s recent=%s\\n' \"$files\" \"$recent\""
|
||||
)
|
||||
|
||||
|
||||
def parse_spool_status(output: str) -> SpoolStatus:
|
||||
values: dict[str, int] = {}
|
||||
for part in output.split():
|
||||
if "=" not in part:
|
||||
continue
|
||||
key, value = part.split("=", 1)
|
||||
try:
|
||||
values[key] = int(value)
|
||||
except ValueError:
|
||||
continue
|
||||
return SpoolStatus(files=values.get("files", -1), recent=values.get("recent", -1))
|
||||
|
||||
|
||||
def ssh_target(user: str, host: str) -> str:
|
||||
return user + "@" + host if user else host
|
||||
|
||||
|
||||
def shell_quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "'\"'\"'") + "'"
|
||||
|
||||
|
||||
def safe_release_name(value: str) -> bool:
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return False
|
||||
return all(r.isalnum() or r in "._-" for r in value)
|
||||
|
||||
|
||||
def run(argv: list[str], cwd: str | None = None, env: dict[str, str] | None = None) -> None:
|
||||
subprocess.run(argv, cwd=cwd, env=env, check=True)
|
||||
|
||||
|
||||
def run_capture(argv: list[str], cwd: str | None = None) -> str:
|
||||
return subprocess.run(argv, cwd=cwd, check=True, stdout=subprocess.PIPE, text=True).stdout
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--repo", default=".")
|
||||
parser.add_argument("--go-dir", default=DEFAULT_GO_DIR)
|
||||
parser.add_argument("--app-host", default=DEFAULT_APP_HOST)
|
||||
parser.add_argument("--kafka-host", default=DEFAULT_KAFKA_HOST)
|
||||
parser.add_argument("--user", default=DEFAULT_USER)
|
||||
parser.add_argument("--release-root", default=DEFAULT_RELEASE_ROOT)
|
||||
parser.add_argument("--release", default="")
|
||||
parser.add_argument("--date", default="")
|
||||
parser.add_argument("--timeout", type=float, default=20.0)
|
||||
parser.add_argument("--spool-drain-timeout", type=float, default=900.0)
|
||||
parser.add_argument("--spool-poll-interval", type=float, default=10.0)
|
||||
parser.add_argument("--skip-acceptance", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
try:
|
||||
deploy(args)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
print("command failed: " + " ".join(exc.cmd), file=sys.stderr)
|
||||
return exc.returncode or 1
|
||||
except (OSError, shutil.Error) as exc:
|
||||
print("deploy failed: " + str(exc), file=sys.stderr)
|
||||
return 1
|
||||
except RuntimeError as exc:
|
||||
print("deploy failed: " + str(exc), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
544
tools/go_native_prod_smoke.py
Executable file
544
tools/go_native_prod_smoke.py
Executable file
@@ -0,0 +1,544 @@
|
||||
#!/usr/bin/env python3
|
||||
"""HTTP smoke checks for the Go native production vehicle ingest stack."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
SHANGHAI = dt.timezone(dt.timedelta(hours=8))
|
||||
DEFAULT_BASE_URL = "http://115.29.187.205:20210"
|
||||
DEFAULT_REALTIME_MAX_AGE_MINUTES = 15.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CheckSpec:
|
||||
name: str
|
||||
path: str
|
||||
params: dict[str, Any]
|
||||
minimum: int
|
||||
kind: str = "total"
|
||||
max_age_minutes: float | None = None
|
||||
require_parsed_json: bool = False
|
||||
require_frame_id: bool = False
|
||||
require_metric_formula: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Check:
|
||||
name: str
|
||||
status: str
|
||||
count: int
|
||||
minimum: int
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CheckWindow:
|
||||
date_from: str
|
||||
date_to: str
|
||||
stat_date: str
|
||||
max_raw_age_minutes: float | None
|
||||
|
||||
|
||||
def shanghai_day_window(now: dt.datetime | None = None) -> tuple[str, str]:
|
||||
now = now or dt.datetime.now(tz=SHANGHAI)
|
||||
local = now.astimezone(SHANGHAI)
|
||||
start = dt.datetime(local.year, local.month, local.day, tzinfo=SHANGHAI)
|
||||
end = start + dt.timedelta(days=1)
|
||||
return start.isoformat(timespec="seconds"), end.isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def resolve_check_window(
|
||||
raw_date: str | None,
|
||||
max_raw_age_minutes: float,
|
||||
now: dt.datetime | None = None,
|
||||
) -> CheckWindow:
|
||||
current = (now or dt.datetime.now(tz=SHANGHAI)).astimezone(SHANGHAI)
|
||||
today = current.date()
|
||||
if raw_date:
|
||||
day = dt.date.fromisoformat(raw_date)
|
||||
start = dt.datetime(day.year, day.month, day.day, tzinfo=SHANGHAI)
|
||||
date_from, date_to = shanghai_day_window(start)
|
||||
freshness = max_raw_age_minutes if day == today else None
|
||||
return CheckWindow(date_from, date_to, raw_date, freshness)
|
||||
date_from, date_to = shanghai_day_window(current)
|
||||
return CheckWindow(date_from, date_to, date_from[:10], max_raw_age_minutes)
|
||||
|
||||
|
||||
def api_url(base_url: str, path: str, params: dict[str, Any]) -> str:
|
||||
url = base_url.rstrip("/") + path
|
||||
if not params:
|
||||
return url
|
||||
return url + "?" + urllib.parse.urlencode(params)
|
||||
|
||||
|
||||
def query_json(base_url: str, path: str, params: dict[str, Any], timeout: float) -> dict[str, Any]:
|
||||
request = urllib.request.Request(
|
||||
api_url(base_url, path, params),
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
|
||||
def check_total(
|
||||
name: str,
|
||||
payload: dict[str, Any],
|
||||
minimum: int,
|
||||
max_age_minutes: float | None = None,
|
||||
now: dt.datetime | None = None,
|
||||
require_parsed_json: bool = False,
|
||||
require_frame_id: bool = False,
|
||||
require_metric_formula: bool = False,
|
||||
) -> Check:
|
||||
count = int(payload.get("total") or 0)
|
||||
items = payload.get("items") or []
|
||||
sample = sample_summary(items)
|
||||
age_message = latest_age_message(items, now)
|
||||
if count >= minimum:
|
||||
parsed_message = ""
|
||||
if max_age_minutes is not None:
|
||||
latest_age = latest_age_minutes(items, now)
|
||||
if latest_age is None:
|
||||
return Check(name, "fail", count, minimum, f"count={count}; missing latest ts; sample={sample}")
|
||||
if latest_age > max_age_minutes:
|
||||
return Check(
|
||||
name,
|
||||
"fail",
|
||||
count,
|
||||
minimum,
|
||||
f"count={count}; latest_age_minutes={latest_age:.1f}; "
|
||||
f"expected <= {max_age_minutes:.1f}; sample={sample}",
|
||||
)
|
||||
if require_parsed_json:
|
||||
parsed_message = parsed_json_message(items)
|
||||
if not parsed_message.startswith("parsed_json=ok"):
|
||||
return Check(name, "fail", count, minimum, f"count={count}; {parsed_message}; sample={sample}")
|
||||
frame_message = ""
|
||||
if require_frame_id:
|
||||
frame_message = frame_id_message(items)
|
||||
if not frame_message.startswith("frame_id=ok"):
|
||||
return Check(name, "fail", count, minimum, f"count={count}; {frame_message}; sample={sample}")
|
||||
metric_message = ""
|
||||
if require_metric_formula:
|
||||
metric_message = metric_formula_message(items)
|
||||
if not metric_message.startswith("metric_formula=ok"):
|
||||
return Check(name, "fail", count, minimum, f"count={count}; {metric_message}; sample={sample}")
|
||||
details = [f"count={count}", age_message]
|
||||
if parsed_message:
|
||||
details.append(parsed_message)
|
||||
if frame_message:
|
||||
details.append(frame_message)
|
||||
if metric_message:
|
||||
details.append(metric_message)
|
||||
return Check(name, "pass", count, minimum, "; ".join(details) + f"; sample={sample}")
|
||||
return Check(name, "fail", count, minimum, f"count={count}; expected >= {minimum}; {age_message}; sample={sample}")
|
||||
|
||||
|
||||
def parse_tdengine_utc_timestamp(value: str) -> dt.datetime:
|
||||
value = str(value or "").strip()
|
||||
for layout in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S"):
|
||||
try:
|
||||
return dt.datetime.strptime(value[:19], layout).replace(tzinfo=dt.timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
raise ValueError(f"unsupported timestamp: {value}")
|
||||
|
||||
|
||||
def latest_age_minutes(items: list[Any], now: dt.datetime | None = None) -> float | None:
|
||||
if not items or not isinstance(items[0], dict):
|
||||
return None
|
||||
timestamp = items[0].get("received_at") or items[0].get("ts")
|
||||
if not timestamp:
|
||||
return None
|
||||
current = now or dt.datetime.now(tz=dt.timezone.utc)
|
||||
if current.tzinfo is None:
|
||||
current = current.replace(tzinfo=dt.timezone.utc)
|
||||
latest = parse_tdengine_utc_timestamp(str(timestamp))
|
||||
return max(0.0, (current.astimezone(dt.timezone.utc) - latest).total_seconds() / 60)
|
||||
|
||||
|
||||
def latest_age_message(items: list[Any], now: dt.datetime | None = None) -> str:
|
||||
latest_age = latest_age_minutes(items, now)
|
||||
if latest_age is None:
|
||||
return "latest_age_minutes=unknown"
|
||||
return f"latest_age_minutes={latest_age:.1f}"
|
||||
|
||||
|
||||
def parsed_json_message(items: list[Any]) -> str:
|
||||
if not items or not isinstance(items[0], dict):
|
||||
return "missing parsed_json"
|
||||
raw = items[0].get("parsed_json")
|
||||
if isinstance(raw, dict) and raw:
|
||||
return "parsed_json=ok"
|
||||
if not isinstance(raw, str) or not raw.strip() or raw.strip().lower() == "null":
|
||||
return "missing parsed_json"
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return "invalid parsed_json"
|
||||
if isinstance(parsed, dict) and parsed:
|
||||
return "parsed_json=ok"
|
||||
return "invalid parsed_json"
|
||||
|
||||
|
||||
def frame_id_message(items: list[Any]) -> str:
|
||||
if not items or not isinstance(items[0], dict):
|
||||
return "missing frame_id"
|
||||
frame_id = str(items[0].get("frame_id") or "").strip()
|
||||
if frame_id:
|
||||
return "frame_id=ok"
|
||||
return "missing frame_id"
|
||||
|
||||
|
||||
def metric_formula_message(items: list[Any]) -> str:
|
||||
if not items or not isinstance(items[0], dict):
|
||||
return "missing metric row"
|
||||
row = items[0]
|
||||
try:
|
||||
metric_value = float(row.get("metric_value"))
|
||||
first_total = float(row.get("first_total_mileage_km"))
|
||||
latest_total = float(row.get("latest_total_mileage_km"))
|
||||
except (TypeError, ValueError):
|
||||
return "missing metric formula fields"
|
||||
metric_key = str(row.get("metric_key") or "")
|
||||
if metric_key == "daily_mileage_km":
|
||||
expected = max(0.0, latest_total - first_total)
|
||||
elif metric_key == "daily_total_mileage_km":
|
||||
expected = latest_total
|
||||
else:
|
||||
return "unsupported metric_key=" + metric_key
|
||||
if abs(metric_value - expected) <= 0.001:
|
||||
return "metric_formula=ok"
|
||||
return f"metric_formula mismatch value={metric_value:.3f} expected={expected:.3f}"
|
||||
|
||||
|
||||
def check_realtime(
|
||||
name: str,
|
||||
payload: dict[str, Any],
|
||||
max_age_minutes: float | None = None,
|
||||
now: dt.datetime | None = None,
|
||||
) -> Check:
|
||||
if payload.get("online") is False:
|
||||
return Check(name, "fail", 0, 1, "online=false")
|
||||
updated_message = ""
|
||||
if max_age_minutes is not None:
|
||||
updated_age = realtime_updated_age_minutes(payload, now)
|
||||
if updated_age is None:
|
||||
return Check(name, "fail", 0, 1, "missing updated_at_ms")
|
||||
updated_message = f"updated_age_minutes={updated_age:.1f}"
|
||||
if updated_age > max_age_minutes:
|
||||
return Check(
|
||||
name,
|
||||
"fail",
|
||||
0,
|
||||
1,
|
||||
updated_message + f"; expected <= {max_age_minutes:.1f}",
|
||||
)
|
||||
identifier = payload.get("vin") or payload.get("vehicle_key") or ""
|
||||
protocols = payload.get("protocols") or []
|
||||
protocol = payload.get("protocol") or ""
|
||||
fields = payload.get("fields") or {}
|
||||
message = json.dumps(
|
||||
{
|
||||
key: value
|
||||
for key, value in {
|
||||
"identifier": identifier,
|
||||
"protocol": protocol,
|
||||
"protocols": protocols,
|
||||
"field_count": len(fields) if isinstance(fields, dict) else 0,
|
||||
"online": payload.get("online"),
|
||||
}.items()
|
||||
if value not in (None, "", [])
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
if updated_message:
|
||||
message = updated_message + "; " + message
|
||||
return Check(name, "pass", 1, 1, message)
|
||||
|
||||
|
||||
def realtime_updated_age_minutes(payload: dict[str, Any], now: dt.datetime | None = None) -> float | None:
|
||||
try:
|
||||
updated_ms = int(payload.get("updated_at_ms"))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
current = now or dt.datetime.now(tz=dt.timezone.utc)
|
||||
if current.tzinfo is None:
|
||||
current = current.replace(tzinfo=dt.timezone.utc)
|
||||
updated = dt.datetime.fromtimestamp(updated_ms / 1000, tz=dt.timezone.utc)
|
||||
return max(0.0, (current.astimezone(dt.timezone.utc) - updated).total_seconds() / 60)
|
||||
|
||||
|
||||
def sample_summary(items: list[Any]) -> str:
|
||||
if not items:
|
||||
return "{}"
|
||||
first = items[0]
|
||||
if not isinstance(first, dict):
|
||||
return json.dumps(first, ensure_ascii=False, sort_keys=True)
|
||||
keep = {
|
||||
key: first.get(key)
|
||||
for key in (
|
||||
"ts",
|
||||
"stat_date",
|
||||
"protocol",
|
||||
"vehicle_key",
|
||||
"vin",
|
||||
"phone",
|
||||
"message_id_hex",
|
||||
"metric_key",
|
||||
"metric_value",
|
||||
"total_mileage_km",
|
||||
"latest_total_mileage_km",
|
||||
)
|
||||
if first.get(key) not in (None, "")
|
||||
}
|
||||
return json.dumps(keep, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def overall_status(checks: list[Check]) -> str:
|
||||
return "fail" if any(check.status == "fail" for check in checks) else "pass"
|
||||
|
||||
|
||||
def build_check_specs(
|
||||
*,
|
||||
date_from: str,
|
||||
date_to: str,
|
||||
stat_date: str,
|
||||
min_raw: int,
|
||||
min_history: int,
|
||||
min_stat: int,
|
||||
max_raw_age_minutes: float | None = None,
|
||||
) -> list[CheckSpec]:
|
||||
raw_params = {"limit": 1, "orderBy": "receivedAt", "includeTotal": "false"}
|
||||
gb32960_history_params = {"protocol": "GB32960", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
|
||||
jt808_history_params = {"protocol": "JT808", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
|
||||
yutong_mqtt_history_params = {"protocol": "YUTONG_MQTT", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
|
||||
return [
|
||||
CheckSpec(
|
||||
"gb32960.raw",
|
||||
"/api/history/raw-frames",
|
||||
{**raw_params, "protocol": "GB32960"},
|
||||
min_raw,
|
||||
max_age_minutes=max_raw_age_minutes,
|
||||
require_parsed_json=True,
|
||||
),
|
||||
CheckSpec(
|
||||
"jt808.raw",
|
||||
"/api/history/raw-frames",
|
||||
{**raw_params, "protocol": "JT808"},
|
||||
min_raw,
|
||||
max_age_minutes=max_raw_age_minutes,
|
||||
require_parsed_json=True,
|
||||
),
|
||||
CheckSpec(
|
||||
"yutong_mqtt.raw",
|
||||
"/api/history/raw-frames",
|
||||
{**raw_params, "protocol": "YUTONG_MQTT"},
|
||||
min_raw,
|
||||
max_age_minutes=max_raw_age_minutes,
|
||||
require_parsed_json=True,
|
||||
),
|
||||
CheckSpec("gb32960.locations", "/api/history/locations", gb32960_history_params, min_history, require_frame_id=True),
|
||||
CheckSpec("gb32960.mileage_points", "/api/history/mileage-points", gb32960_history_params, min_history, require_frame_id=True),
|
||||
CheckSpec("jt808.locations", "/api/history/locations", jt808_history_params, min_history, require_frame_id=True),
|
||||
CheckSpec("jt808.mileage_points", "/api/history/mileage-points", jt808_history_params, min_history, require_frame_id=True),
|
||||
CheckSpec("yutong_mqtt.locations", "/api/history/locations", yutong_mqtt_history_params, min_history, require_frame_id=True),
|
||||
CheckSpec("yutong_mqtt.mileage_points", "/api/history/mileage-points", yutong_mqtt_history_params, min_history, require_frame_id=True),
|
||||
CheckSpec(
|
||||
"gb32960.daily_mileage",
|
||||
"/api/stats/daily-metrics",
|
||||
{
|
||||
"protocol": "GB32960",
|
||||
"metricKey": "daily_mileage_km",
|
||||
"dateFrom": stat_date,
|
||||
"dateTo": stat_date,
|
||||
"limit": 1,
|
||||
},
|
||||
min_stat,
|
||||
require_metric_formula=True,
|
||||
),
|
||||
CheckSpec(
|
||||
"gb32960.daily_total_mileage",
|
||||
"/api/stats/daily-metrics",
|
||||
{
|
||||
"protocol": "GB32960",
|
||||
"metricKey": "daily_total_mileage_km",
|
||||
"dateFrom": stat_date,
|
||||
"dateTo": stat_date,
|
||||
"limit": 1,
|
||||
},
|
||||
min_stat,
|
||||
require_metric_formula=True,
|
||||
),
|
||||
CheckSpec(
|
||||
"jt808.daily_mileage",
|
||||
"/api/stats/daily-metrics",
|
||||
{
|
||||
"protocol": "JT808",
|
||||
"metricKey": "daily_mileage_km",
|
||||
"dateFrom": stat_date,
|
||||
"dateTo": stat_date,
|
||||
"limit": 1,
|
||||
},
|
||||
min_stat,
|
||||
require_metric_formula=True,
|
||||
),
|
||||
CheckSpec(
|
||||
"jt808.daily_total_mileage",
|
||||
"/api/stats/daily-metrics",
|
||||
{
|
||||
"protocol": "JT808",
|
||||
"metricKey": "daily_total_mileage_km",
|
||||
"dateFrom": stat_date,
|
||||
"dateTo": stat_date,
|
||||
"limit": 1,
|
||||
},
|
||||
min_stat,
|
||||
require_metric_formula=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def vehicle_identifier(item: dict[str, Any]) -> str:
|
||||
for key in ("vin", "vehicle_key"):
|
||||
value = str(item.get(key) or "").strip()
|
||||
if value and value.lower() != "unknown":
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def build_realtime_specs(payloads: dict[str, dict[str, Any]]) -> list[CheckSpec]:
|
||||
configs = [
|
||||
("gb32960", "gb32960.raw", "GB32960"),
|
||||
("jt808", "jt808.raw", "JT808"),
|
||||
("yutong_mqtt", "yutong_mqtt.raw", "YUTONG_MQTT"),
|
||||
]
|
||||
specs: list[CheckSpec] = []
|
||||
for prefix, source_name, protocol in configs:
|
||||
items = payloads.get(source_name, {}).get("items") or []
|
||||
if not items or not isinstance(items[0], dict):
|
||||
continue
|
||||
identifier = vehicle_identifier(items[0])
|
||||
if not identifier:
|
||||
continue
|
||||
encoded = urllib.parse.quote(identifier, safe="")
|
||||
base_path = "/api/realtime/vehicles/" + encoded
|
||||
specs.extend([
|
||||
CheckSpec(
|
||||
prefix + ".realtime",
|
||||
base_path,
|
||||
{},
|
||||
1,
|
||||
"realtime",
|
||||
max_age_minutes=DEFAULT_REALTIME_MAX_AGE_MINUTES,
|
||||
),
|
||||
CheckSpec(prefix + ".realtime_online", base_path + "/online", {}, 1, "realtime"),
|
||||
CheckSpec(
|
||||
prefix + ".realtime_protocol",
|
||||
base_path + "/protocols/" + protocol,
|
||||
{},
|
||||
1,
|
||||
"realtime",
|
||||
max_age_minutes=DEFAULT_REALTIME_MAX_AGE_MINUTES,
|
||||
),
|
||||
])
|
||||
return specs
|
||||
|
||||
|
||||
def run_checks(base_url: str, specs: list[CheckSpec], timeout: float) -> list[Check]:
|
||||
checks: list[Check] = []
|
||||
for spec in specs:
|
||||
try:
|
||||
payload = query_json(base_url, spec.path, spec.params, timeout)
|
||||
if spec.kind == "realtime":
|
||||
checks.append(check_realtime(spec.name, payload, spec.max_age_minutes))
|
||||
else:
|
||||
checks.append(check_total(
|
||||
spec.name,
|
||||
payload,
|
||||
spec.minimum,
|
||||
spec.max_age_minutes,
|
||||
require_parsed_json=spec.require_parsed_json,
|
||||
require_frame_id=spec.require_frame_id,
|
||||
require_metric_formula=spec.require_metric_formula,
|
||||
))
|
||||
except Exception as exc: # pragma: no cover - exercised by live failures.
|
||||
checks.append(Check(spec.name, "fail", 0, spec.minimum, f"request failed: {exc}"))
|
||||
return checks
|
||||
|
||||
|
||||
def query_payloads(base_url: str, specs: list[CheckSpec], timeout: float) -> tuple[list[Check], dict[str, dict[str, Any]]]:
|
||||
checks: list[Check] = []
|
||||
payloads: dict[str, dict[str, Any]] = {}
|
||||
for spec in specs:
|
||||
try:
|
||||
payload = query_json(base_url, spec.path, spec.params, timeout)
|
||||
payloads[spec.name] = payload
|
||||
checks.append(check_total(
|
||||
spec.name,
|
||||
payload,
|
||||
spec.minimum,
|
||||
spec.max_age_minutes,
|
||||
require_parsed_json=spec.require_parsed_json,
|
||||
require_frame_id=spec.require_frame_id,
|
||||
require_metric_formula=spec.require_metric_formula,
|
||||
))
|
||||
except Exception as exc:
|
||||
checks.append(Check(spec.name, "fail", 0, spec.minimum, f"request failed: {exc}"))
|
||||
return checks, payloads
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
||||
parser.add_argument("--date", help="Shanghai local date, YYYY-MM-DD. Defaults to today.")
|
||||
parser.add_argument("--timeout", type=float, default=5.0)
|
||||
parser.add_argument("--min-raw", type=int, default=1)
|
||||
parser.add_argument("--min-history", type=int, default=1)
|
||||
parser.add_argument("--min-stat", type=int, default=1)
|
||||
parser.add_argument(
|
||||
"--max-raw-age-minutes",
|
||||
type=float,
|
||||
default=15.0,
|
||||
help="Require latest RAW sample age to be at most this many minutes when checking today's date.",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
window = resolve_check_window(args.date, args.max_raw_age_minutes)
|
||||
specs = build_check_specs(
|
||||
date_from=window.date_from,
|
||||
date_to=window.date_to,
|
||||
stat_date=window.stat_date,
|
||||
min_raw=args.min_raw,
|
||||
min_history=args.min_history,
|
||||
min_stat=args.min_stat,
|
||||
max_raw_age_minutes=window.max_raw_age_minutes,
|
||||
)
|
||||
checks, payloads = query_payloads(args.base_url, specs, args.timeout)
|
||||
checks.extend(run_checks(args.base_url, build_realtime_specs(payloads), args.timeout))
|
||||
status = overall_status(checks)
|
||||
print(json.dumps({
|
||||
"status": status,
|
||||
"baseUrl": args.base_url,
|
||||
"dateFrom": window.date_from,
|
||||
"dateTo": window.date_to,
|
||||
"checks": [asdict(check) for check in checks],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
return 0 if status == "pass" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
136
tools/go_prod_acceptance.py
Executable file
136
tools/go_prod_acceptance.py
Executable file
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the Go native production acceptance smoke suite."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_APP_HOST = "115.29.187.205"
|
||||
DEFAULT_KAFKA_HOST = "114.55.58.251"
|
||||
DEFAULT_USER = "root"
|
||||
DEFAULT_BASE_URL = "http://115.29.187.205:20210"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChildCommand:
|
||||
name: str
|
||||
argv: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChildResult:
|
||||
name: str
|
||||
status: str
|
||||
exit_code: int
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
def build_commands(args: argparse.Namespace) -> list[ChildCommand]:
|
||||
timeout = str(args.timeout)
|
||||
return [
|
||||
ChildCommand("systemd", [
|
||||
sys.executable,
|
||||
"tools/go_systemd_prod_smoke.py",
|
||||
"--host",
|
||||
args.app_host,
|
||||
"--user",
|
||||
args.ssh_user,
|
||||
"--timeout",
|
||||
timeout,
|
||||
]),
|
||||
ChildCommand("kafka", [
|
||||
sys.executable,
|
||||
"tools/go_kafka_prod_smoke.py",
|
||||
"--host",
|
||||
args.kafka_host,
|
||||
"--user",
|
||||
args.ssh_user,
|
||||
"--max-lag",
|
||||
str(args.max_lag),
|
||||
"--timeout",
|
||||
timeout,
|
||||
]),
|
||||
ChildCommand("http", [
|
||||
sys.executable,
|
||||
"tools/go_native_prod_smoke.py",
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
"--timeout",
|
||||
timeout,
|
||||
] + (["--date", args.date] if args.date else [])),
|
||||
]
|
||||
|
||||
|
||||
def run_child(command: ChildCommand, timeout: float) -> ChildResult:
|
||||
completed = subprocess.run(
|
||||
command.argv,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
payload = parse_json_output(completed.stdout)
|
||||
if completed.stderr.strip():
|
||||
payload.setdefault("stderr", completed.stderr.strip())
|
||||
status = payload.get("status")
|
||||
if status not in {"pass", "fail"}:
|
||||
status = "pass" if completed.returncode == 0 else "fail"
|
||||
return ChildResult(command.name, status, completed.returncode, payload)
|
||||
|
||||
|
||||
def parse_json_output(output: str) -> dict[str, Any]:
|
||||
output = output.strip()
|
||||
if not output:
|
||||
return {}
|
||||
start = output.find("{")
|
||||
end = output.rfind("}")
|
||||
if start < 0 or end < start:
|
||||
return {"rawOutput": output}
|
||||
try:
|
||||
return json.loads(output[start:end + 1])
|
||||
except json.JSONDecodeError:
|
||||
return {"rawOutput": output}
|
||||
|
||||
|
||||
def overall_status(results: list[ChildResult]) -> str:
|
||||
if not results:
|
||||
return "fail"
|
||||
return "fail" if any(result.status != "pass" or result.exit_code != 0 for result in results) else "pass"
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--app-host", default=DEFAULT_APP_HOST)
|
||||
parser.add_argument("--kafka-host", default=DEFAULT_KAFKA_HOST)
|
||||
parser.add_argument("--ssh-user", default=DEFAULT_USER)
|
||||
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
||||
parser.add_argument("--date", help="Shanghai local date, YYYY-MM-DD. Defaults to today in go_native_prod_smoke.py.")
|
||||
parser.add_argument("--timeout", type=float, default=20.0)
|
||||
parser.add_argument("--max-lag", type=int, default=100)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
results: list[ChildResult] = []
|
||||
for command in build_commands(args):
|
||||
try:
|
||||
results.append(run_child(command, args.timeout + 10))
|
||||
except Exception as exc:
|
||||
results.append(ChildResult(command.name, "fail", 1, {"status": "fail", "error": str(exc)}))
|
||||
status = overall_status(results)
|
||||
print(json.dumps({
|
||||
"status": status,
|
||||
"checks": [asdict(result) for result in results],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
return 0 if status == "pass" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
210
tools/go_systemd_prod_smoke.py
Executable file
210
tools/go_systemd_prod_smoke.py
Executable file
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Systemd and port smoke checks for the Go native production ECS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
|
||||
DEFAULT_HOST = "115.29.187.205"
|
||||
DEFAULT_USER = "root"
|
||||
DEFAULT_SERVICES = [
|
||||
"lingniu-go-gateway.service",
|
||||
"lingniu-go-history-writer.service",
|
||||
"lingniu-go-stat-writer.service",
|
||||
"lingniu-go-realtime-api.service",
|
||||
]
|
||||
DEFAULT_PORTS = {
|
||||
808: "gateway",
|
||||
32960: "gateway",
|
||||
20210: "realtime-api",
|
||||
}
|
||||
DEFAULT_RELEASE_ROOT = "/opt/lingniu-go-native"
|
||||
DEFAULT_SPOOL_DIR = "/opt/lingniu-go-native/spool/gateway"
|
||||
DEFAULT_BINARIES = ["gateway", "history-writer", "stat-writer", "realtime-api"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Check:
|
||||
name: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServiceState:
|
||||
active: str
|
||||
enabled: str
|
||||
|
||||
|
||||
def parse_service_states(output: str) -> dict[str, ServiceState]:
|
||||
states: dict[str, ServiceState] = {}
|
||||
for line in output.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) >= 2:
|
||||
enabled = parts[2] if len(parts) >= 3 else "enabled"
|
||||
states[parts[0]] = ServiceState(parts[1], enabled)
|
||||
return states
|
||||
|
||||
|
||||
def service_checks(output: str, services: list[str]) -> list[Check]:
|
||||
states = parse_service_states(output)
|
||||
checks: list[Check] = []
|
||||
for service in services:
|
||||
state = states.get(service, ServiceState("missing", "missing"))
|
||||
status = "pass" if state.active == "active" and state.enabled == "enabled" else "fail"
|
||||
checks.append(Check(
|
||||
"service." + service,
|
||||
status,
|
||||
"state=" + state.active + "; enabled=" + state.enabled,
|
||||
))
|
||||
return checks
|
||||
|
||||
|
||||
def port_checks(output: str, ports: dict[int, str]) -> list[Check]:
|
||||
lines = output.splitlines()
|
||||
checks: list[Check] = []
|
||||
for port, process in sorted(ports.items()):
|
||||
matching = [line.strip() for line in lines if has_port(line, port)]
|
||||
if not matching:
|
||||
checks.append(Check("port." + str(port), "fail", "not listening"))
|
||||
continue
|
||||
owned = [line for line in matching if process in line]
|
||||
if owned:
|
||||
checks.append(Check("port." + str(port), "pass", owned[0]))
|
||||
else:
|
||||
checks.append(Check("port." + str(port), "fail", matching[0]))
|
||||
return checks
|
||||
|
||||
|
||||
def has_port(line: str, port: int) -> bool:
|
||||
needle = ":" + str(port)
|
||||
return needle + " " in line or needle + "\t" in line
|
||||
|
||||
|
||||
def spool_check(output: str) -> Check:
|
||||
values = parse_key_values(output)
|
||||
files = int(values.get("files", "-1"))
|
||||
recent = int(values.get("recent", "-1"))
|
||||
status = "pass" if files == 0 and recent == 0 else "fail"
|
||||
return Check("spool.gateway", status, output.strip() or "missing")
|
||||
|
||||
|
||||
def release_check(output: str) -> Check:
|
||||
values = parse_key_values(output)
|
||||
current = values.get("current", "")
|
||||
binaries_present = [values.get(binary, "0") == "1" for binary in DEFAULT_BINARIES]
|
||||
status = "pass" if current and all(binaries_present) else "fail"
|
||||
return Check("release.current", status, output.strip() or "missing")
|
||||
|
||||
|
||||
def parse_key_values(output: str) -> dict[str, str]:
|
||||
values: dict[str, str] = {}
|
||||
for part in output.split():
|
||||
if "=" not in part:
|
||||
continue
|
||||
key, value = part.split("=", 1)
|
||||
values[key] = value
|
||||
return values
|
||||
|
||||
|
||||
def ssh(host: str, user: str, command: str, timeout: float) -> str:
|
||||
target = user + "@" + host if user else host
|
||||
completed = subprocess.run(
|
||||
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", target, command],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def remote_command(services: list[str]) -> str:
|
||||
quoted = " ".join(services)
|
||||
return (
|
||||
"for svc in " + quoted + "; do "
|
||||
"printf '%s ' \"$svc\"; "
|
||||
"printf '%s ' \"$(systemctl is-active \"$svc\" 2>/dev/null || true)\"; "
|
||||
"systemctl is-enabled \"$svc\" 2>/dev/null || true; "
|
||||
"done; "
|
||||
"printf '\\n--PORTS--\\n'; "
|
||||
"ss -lntp; "
|
||||
"printf '\\n--SPOOL--\\n'; "
|
||||
"spool='" + DEFAULT_SPOOL_DIR + "'; "
|
||||
"if test -d \"$spool\"; then "
|
||||
"files=$(find \"$spool\" -type f | wc -l); "
|
||||
"bytes=$(du -sb \"$spool\" 2>/dev/null | awk '{print $1}'); "
|
||||
"recent=$(find \"$spool\" -type f -mmin -5 | wc -l); "
|
||||
"printf 'files=%s bytes=%s recent=%s\\n' \"$files\" \"${bytes:-0}\" \"$recent\"; "
|
||||
"else printf 'files=-1 bytes=0 recent=-1 missing=%s\\n' \"$spool\"; fi; "
|
||||
"printf '\\n--RELEASE--\\n'; "
|
||||
"root='" + DEFAULT_RELEASE_ROOT + "'; "
|
||||
"current=$(readlink -f \"$root/current\" 2>/dev/null || true); "
|
||||
"printf 'current=%s' \"$current\"; "
|
||||
"for bin in " + " ".join(DEFAULT_BINARIES) + "; do "
|
||||
"if test -x \"$root/current/$bin\"; then present=1; else present=0; fi; "
|
||||
"printf ' %s=%s' \"$bin\" \"$present\"; "
|
||||
"done; printf '\\n'"
|
||||
)
|
||||
|
||||
|
||||
def split_remote_output(output: str) -> tuple[str, str, str, str]:
|
||||
ports_marker = "\n--PORTS--\n"
|
||||
spool_marker = "\n--SPOOL--\n"
|
||||
release_marker = "\n--RELEASE--\n"
|
||||
if ports_marker not in output:
|
||||
return output, "", "", ""
|
||||
service_output, rest = output.split(ports_marker, 1)
|
||||
if spool_marker not in rest:
|
||||
return service_output, rest, "", ""
|
||||
port_output, spool_output = rest.split(spool_marker, 1)
|
||||
if release_marker not in spool_output:
|
||||
return service_output, port_output, spool_output, ""
|
||||
spool_output, release_output = spool_output.split(release_marker, 1)
|
||||
return service_output, port_output, spool_output, release_output
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
|
||||
output = ssh(args.host, args.user, remote_command(DEFAULT_SERVICES), args.timeout)
|
||||
service_output, port_output, spool_output, release_output = split_remote_output(output)
|
||||
checks = (
|
||||
service_checks(service_output, DEFAULT_SERVICES)
|
||||
+ port_checks(port_output, DEFAULT_PORTS)
|
||||
+ [spool_check(spool_output)]
|
||||
+ [release_check(release_output)]
|
||||
)
|
||||
status = "fail" if any(check.status == "fail" for check in checks) else "pass"
|
||||
return status, checks
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
||||
parser.add_argument("--user", default=DEFAULT_USER)
|
||||
parser.add_argument("--timeout", type=float, default=20.0)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
try:
|
||||
status, checks = run(args)
|
||||
except Exception as exc:
|
||||
status = "fail"
|
||||
checks = [Check("systemd.ssh", "fail", str(exc))]
|
||||
print(json.dumps({
|
||||
"status": status,
|
||||
"host": args.host,
|
||||
"checks": [asdict(check) for check in checks],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
return 0 if status == "pass" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
62
tools/test_go_kafka_prod_smoke.py
Normal file
62
tools/test_go_kafka_prod_smoke.py
Normal file
@@ -0,0 +1,62 @@
|
||||
import unittest
|
||||
|
||||
from tools import go_kafka_prod_smoke as smoke
|
||||
|
||||
|
||||
class GoKafkaProdSmokeTest(unittest.TestCase):
|
||||
def test_parse_topic_list_marks_required_topics_present(self):
|
||||
topics = smoke.parse_topic_list("""
|
||||
vehicle.raw.go.gb32960.v1
|
||||
vehicle.raw.go.jt808.v1
|
||||
vehicle.raw.go.yutong-mqtt.v1
|
||||
vehicle.event.go.unified.v1
|
||||
""")
|
||||
|
||||
checks = smoke.topic_checks(topics, smoke.DEFAULT_TOPICS)
|
||||
|
||||
self.assertTrue(all(check.status == "pass" for check in checks))
|
||||
|
||||
def test_topic_checks_fail_when_required_topic_missing(self):
|
||||
topics = {"vehicle.raw.go.gb32960.v1"}
|
||||
|
||||
checks = smoke.topic_checks(topics, ["vehicle.raw.go.gb32960.v1", "vehicle.event.go.unified.v1"])
|
||||
|
||||
by_name = {check.name: check for check in checks}
|
||||
self.assertEqual(by_name["topic.vehicle.event.go.unified.v1"].status, "fail")
|
||||
|
||||
def test_parse_consumer_group_describe_reads_lag_rows(self):
|
||||
rows = smoke.parse_consumer_group_describe("""
|
||||
GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG
|
||||
go-realtime-api vehicle.event.go.unified.v1 0 15992 15992 0
|
||||
go-realtime-api vehicle.event.go.unified.v1 1 2036 2036 3
|
||||
""")
|
||||
|
||||
self.assertEqual(rows[0].group, "go-realtime-api")
|
||||
self.assertEqual(rows[1].topic, "vehicle.event.go.unified.v1")
|
||||
self.assertEqual(rows[1].lag, 3)
|
||||
|
||||
def test_lag_checks_fail_when_lag_exceeds_threshold(self):
|
||||
rows = [
|
||||
smoke.ConsumerLag("go-history-writer", "vehicle.raw.go.jt808.v1", 0, 100, 100, 0),
|
||||
smoke.ConsumerLag("go-history-writer", "vehicle.raw.go.jt808.v1", 1, 100, 110, 10),
|
||||
]
|
||||
|
||||
check = smoke.group_lag_check("go-history-writer", rows, max_lag=5)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertIn("max_lag=10", check.message)
|
||||
|
||||
def test_lag_checks_pass_when_group_has_rows_and_lag_is_small(self):
|
||||
rows = [
|
||||
smoke.ConsumerLag("go-stat-writer", "vehicle.raw.go.gb32960.v1", 0, 100, 100, 0),
|
||||
smoke.ConsumerLag("go-stat-writer", "vehicle.raw.go.jt808.v1", 1, 100, 101, 1),
|
||||
]
|
||||
|
||||
check = smoke.group_lag_check("go-stat-writer", rows, max_lag=5)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("max_lag=1", check.message)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
63
tools/test_go_native_deploy.py
Normal file
63
tools/test_go_native_deploy.py
Normal file
@@ -0,0 +1,63 @@
|
||||
import unittest
|
||||
|
||||
from tools import go_native_deploy as deploy
|
||||
|
||||
|
||||
class GoNativeDeployTest(unittest.TestCase):
|
||||
def test_build_commands_cover_four_production_binaries(self):
|
||||
commands = deploy.build_commands("/repo/go/vehicle-gateway", "/tmp/out")
|
||||
|
||||
self.assertEqual([command.output_name for command in commands], [
|
||||
"gateway",
|
||||
"history-writer",
|
||||
"stat-writer",
|
||||
"realtime-api",
|
||||
])
|
||||
for command in commands:
|
||||
self.assertEqual(command.argv[:3], ["go", "build", "-trimpath"])
|
||||
self.assertIn("GOOS=linux", command.env)
|
||||
self.assertIn("GOARCH=amd64", command.env)
|
||||
self.assertIn("CGO_ENABLED=0", command.env)
|
||||
|
||||
def test_remote_switch_command_creates_release_and_restarts_all_services(self):
|
||||
command = deploy.remote_switch_command("409f55b", "/opt/lingniu-go-native", "/tmp/lingniu-go-native-409f55b.tar.gz")
|
||||
|
||||
self.assertIn("/opt/lingniu-go-native/releases/409f55b", command)
|
||||
self.assertIn("ln -sfn", command)
|
||||
self.assertIn("systemctl restart lingniu-go-gateway", command)
|
||||
self.assertIn("systemctl restart lingniu-go-history-writer", command)
|
||||
self.assertIn("systemctl restart lingniu-go-stat-writer", command)
|
||||
self.assertIn("systemctl restart lingniu-go-realtime-api", command)
|
||||
|
||||
def test_acceptance_command_uses_deployed_hosts_and_date(self):
|
||||
command = deploy.acceptance_command(
|
||||
app_host="115.29.187.205",
|
||||
kafka_host="114.55.58.251",
|
||||
date="2026-07-02",
|
||||
timeout=20,
|
||||
)
|
||||
|
||||
self.assertEqual(command[:2], ["python3", "tools/go_prod_acceptance.py"])
|
||||
self.assertIn("--app-host", command)
|
||||
self.assertIn("115.29.187.205", command)
|
||||
self.assertIn("--kafka-host", command)
|
||||
self.assertIn("114.55.58.251", command)
|
||||
self.assertIn("--date", command)
|
||||
self.assertIn("2026-07-02", command)
|
||||
|
||||
def test_parse_spool_status_reads_file_and_recent_counts(self):
|
||||
status = deploy.parse_spool_status("files=42 recent=3")
|
||||
|
||||
self.assertEqual(status.files, 42)
|
||||
self.assertEqual(status.recent, 3)
|
||||
|
||||
def test_spool_status_command_targets_gateway_spool_directory(self):
|
||||
command = deploy.spool_status_command("/opt/lingniu-go-native")
|
||||
|
||||
self.assertIn("/opt/lingniu-go-native/spool/gateway", command)
|
||||
self.assertIn("files=", command)
|
||||
self.assertIn("recent=", command)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
358
tools/test_go_native_prod_smoke.py
Normal file
358
tools/test_go_native_prod_smoke.py
Normal file
@@ -0,0 +1,358 @@
|
||||
import datetime as dt
|
||||
import unittest
|
||||
|
||||
from tools import go_native_prod_smoke as smoke
|
||||
|
||||
|
||||
class GoNativeProdSmokeTest(unittest.TestCase):
|
||||
def test_shanghai_day_window_uses_plus_eight_bounds(self):
|
||||
now = dt.datetime(2026, 7, 2, 1, 35, tzinfo=smoke.SHANGHAI)
|
||||
|
||||
date_from, date_to = smoke.shanghai_day_window(now)
|
||||
|
||||
self.assertEqual(date_from, "2026-07-02T00:00:00+08:00")
|
||||
self.assertEqual(date_to, "2026-07-03T00:00:00+08:00")
|
||||
|
||||
def test_api_url_encodes_plus_eight_date_range(self):
|
||||
url = smoke.api_url(
|
||||
"http://example.test/base/",
|
||||
"/api/history/raw-frames",
|
||||
{
|
||||
"protocol": "JT808",
|
||||
"dateFrom": "2026-07-02T00:00:00+08:00",
|
||||
"dateTo": "2026-07-03T00:00:00+08:00",
|
||||
"limit": 1,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
url,
|
||||
"http://example.test/base/api/history/raw-frames?"
|
||||
"protocol=JT808&dateFrom=2026-07-02T00%3A00%3A00%2B08%3A00"
|
||||
"&dateTo=2026-07-03T00%3A00%3A00%2B08%3A00&limit=1",
|
||||
)
|
||||
|
||||
def test_explicit_today_date_still_enforces_raw_freshness(self):
|
||||
now = dt.datetime(2026, 7, 2, 9, 30, tzinfo=smoke.SHANGHAI)
|
||||
|
||||
window = smoke.resolve_check_window("2026-07-02", 15.0, now)
|
||||
|
||||
self.assertEqual(window.date_from, "2026-07-02T00:00:00+08:00")
|
||||
self.assertEqual(window.date_to, "2026-07-03T00:00:00+08:00")
|
||||
self.assertEqual(window.stat_date, "2026-07-02")
|
||||
self.assertEqual(window.max_raw_age_minutes, 15.0)
|
||||
|
||||
def test_explicit_historical_date_disables_raw_freshness(self):
|
||||
now = dt.datetime(2026, 7, 2, 9, 30, tzinfo=smoke.SHANGHAI)
|
||||
|
||||
window = smoke.resolve_check_window("2026-07-01", 15.0, now)
|
||||
|
||||
self.assertEqual(window.date_from, "2026-07-01T00:00:00+08:00")
|
||||
self.assertEqual(window.date_to, "2026-07-02T00:00:00+08:00")
|
||||
self.assertEqual(window.stat_date, "2026-07-01")
|
||||
self.assertIsNone(window.max_raw_age_minutes)
|
||||
|
||||
def test_total_check_passes_when_total_reaches_minimum(self):
|
||||
check = smoke.check_total(
|
||||
"jt808.raw",
|
||||
{"total": 2, "items": [{"vehicle_key": "JT808:013307811170"}]},
|
||||
minimum=1,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertEqual(check.count, 2)
|
||||
self.assertIn("JT808:013307811170", check.message)
|
||||
|
||||
def test_total_check_fails_when_total_is_below_minimum(self):
|
||||
check = smoke.check_total("gb32960.raw", {"total": 0, "items": []}, minimum=1)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertEqual(check.count, 0)
|
||||
self.assertIn("expected >= 1", check.message)
|
||||
|
||||
def test_overall_status_fails_if_any_check_fails(self):
|
||||
checks = [
|
||||
smoke.Check("a", "pass", 1, 1, "ok"),
|
||||
smoke.Check("b", "fail", 0, 1, "bad"),
|
||||
]
|
||||
|
||||
self.assertEqual(smoke.overall_status(checks), "fail")
|
||||
|
||||
def test_build_checks_cover_three_raw_protocols_and_two_daily_metric_protocols(self):
|
||||
specs = smoke.build_check_specs(
|
||||
date_from="2026-07-02T00:00:00+08:00",
|
||||
date_to="2026-07-03T00:00:00+08:00",
|
||||
stat_date="2026-07-02",
|
||||
min_raw=1,
|
||||
min_history=1,
|
||||
min_stat=1,
|
||||
)
|
||||
|
||||
names = [spec.name for spec in specs]
|
||||
|
||||
self.assertIn("gb32960.raw", names)
|
||||
self.assertIn("jt808.raw", names)
|
||||
self.assertIn("yutong_mqtt.raw", names)
|
||||
self.assertIn("gb32960.locations", names)
|
||||
self.assertIn("gb32960.mileage_points", names)
|
||||
self.assertIn("jt808.locations", names)
|
||||
self.assertIn("jt808.mileage_points", names)
|
||||
self.assertIn("yutong_mqtt.locations", names)
|
||||
self.assertIn("yutong_mqtt.mileage_points", names)
|
||||
self.assertIn("gb32960.daily_mileage", names)
|
||||
self.assertIn("jt808.daily_total_mileage", names)
|
||||
|
||||
def test_raw_checks_order_by_received_at_for_ingest_freshness(self):
|
||||
specs = smoke.build_check_specs(
|
||||
date_from="2026-07-02T00:00:00+08:00",
|
||||
date_to="2026-07-03T00:00:00+08:00",
|
||||
stat_date="2026-07-02",
|
||||
min_raw=1,
|
||||
min_history=1,
|
||||
min_stat=1,
|
||||
max_raw_age_minutes=15,
|
||||
)
|
||||
|
||||
raw_specs = [spec for spec in specs if spec.name.endswith(".raw")]
|
||||
|
||||
self.assertTrue(raw_specs)
|
||||
for spec in raw_specs:
|
||||
self.assertEqual(spec.params.get("orderBy"), "receivedAt")
|
||||
self.assertEqual(spec.params.get("includeTotal"), "false")
|
||||
self.assertNotIn("dateFrom", spec.params)
|
||||
self.assertNotIn("dateTo", spec.params)
|
||||
|
||||
def test_vehicle_identifier_prefers_vin_over_vehicle_key(self):
|
||||
identifier = smoke.vehicle_identifier({
|
||||
"vin": "LB9A32A20R0LS1343",
|
||||
"vehicle_key": "GB32960:ignored",
|
||||
})
|
||||
|
||||
self.assertEqual(identifier, "LB9A32A20R0LS1343")
|
||||
|
||||
def test_vehicle_identifier_falls_back_to_vehicle_key(self):
|
||||
identifier = smoke.vehicle_identifier({
|
||||
"vin": "",
|
||||
"vehicle_key": "JT808:013307811170",
|
||||
})
|
||||
|
||||
self.assertEqual(identifier, "JT808:013307811170")
|
||||
|
||||
def test_realtime_specs_are_built_from_raw_samples_with_vin(self):
|
||||
payloads = {
|
||||
"gb32960.raw": {"items": [{"vin": "LB9A32A20R0LS1343", "vehicle_key": "LB9A32A20R0LS1343"}]},
|
||||
"jt808.raw": {"items": [{"vin": "", "vehicle_key": "JT808:013307811170"}]},
|
||||
"yutong_mqtt.raw": {"items": [{"vin": "LMRKH9AC6R1004108"}]},
|
||||
}
|
||||
|
||||
specs = smoke.build_realtime_specs(payloads)
|
||||
|
||||
self.assertEqual(
|
||||
[(spec.name, spec.path) for spec in specs],
|
||||
[
|
||||
("gb32960.realtime", "/api/realtime/vehicles/LB9A32A20R0LS1343"),
|
||||
("gb32960.realtime_online", "/api/realtime/vehicles/LB9A32A20R0LS1343/online"),
|
||||
("gb32960.realtime_protocol", "/api/realtime/vehicles/LB9A32A20R0LS1343/protocols/GB32960"),
|
||||
("jt808.realtime", "/api/realtime/vehicles/JT808%3A013307811170"),
|
||||
("jt808.realtime_online", "/api/realtime/vehicles/JT808%3A013307811170/online"),
|
||||
("jt808.realtime_protocol", "/api/realtime/vehicles/JT808%3A013307811170/protocols/JT808"),
|
||||
("yutong_mqtt.realtime", "/api/realtime/vehicles/LMRKH9AC6R1004108"),
|
||||
("yutong_mqtt.realtime_online", "/api/realtime/vehicles/LMRKH9AC6R1004108/online"),
|
||||
("yutong_mqtt.realtime_protocol", "/api/realtime/vehicles/LMRKH9AC6R1004108/protocols/YUTONG_MQTT"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_realtime_check_requires_online_true_when_field_exists(self):
|
||||
check = smoke.check_realtime(
|
||||
"gb32960.realtime_online",
|
||||
{"vin": "LB9A32A20R0LS1343", "online": True, "protocols": ["GB32960"]},
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertEqual(check.count, 1)
|
||||
|
||||
def test_realtime_check_fails_when_online_false(self):
|
||||
check = smoke.check_realtime(
|
||||
"gb32960.realtime_online",
|
||||
{"vin": "LB9A32A20R0LS1343", "online": False},
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertIn("online=false", check.message)
|
||||
|
||||
def test_realtime_check_fails_when_snapshot_is_stale(self):
|
||||
now = dt.datetime(2026, 7, 2, 0, 30, 0, tzinfo=dt.timezone.utc)
|
||||
|
||||
check = smoke.check_realtime(
|
||||
"jt808.realtime",
|
||||
{"vehicle_key": "JT808:013307811170", "updated_at_ms": 1782950400000},
|
||||
max_age_minutes=15,
|
||||
now=now,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertIn("updated_age_minutes=30.0", check.message)
|
||||
|
||||
def test_realtime_check_passes_when_snapshot_is_fresh(self):
|
||||
now = dt.datetime(2026, 7, 2, 0, 30, 0, tzinfo=dt.timezone.utc)
|
||||
|
||||
check = smoke.check_realtime(
|
||||
"jt808.realtime",
|
||||
{"vehicle_key": "JT808:013307811170", "updated_at_ms": 1782951600000},
|
||||
max_age_minutes=15,
|
||||
now=now,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("updated_age_minutes=10.0", check.message)
|
||||
|
||||
def test_parse_tdengine_utc_timestamp_as_aware_utc(self):
|
||||
parsed = smoke.parse_tdengine_utc_timestamp("2026-07-01 17:36:02")
|
||||
|
||||
self.assertEqual(parsed, dt.datetime(2026, 7, 1, 17, 36, 2, tzinfo=dt.timezone.utc))
|
||||
|
||||
def test_total_check_fails_when_latest_sample_is_stale(self):
|
||||
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
|
||||
|
||||
check = smoke.check_total(
|
||||
"gb32960.raw",
|
||||
{"total": 1, "items": [{"ts": "2026-07-01 17:00:00"}]},
|
||||
minimum=1,
|
||||
max_age_minutes=30,
|
||||
now=now,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertIn("latest_age_minutes=60.0", check.message)
|
||||
|
||||
def test_total_check_uses_received_at_for_raw_freshness_when_present(self):
|
||||
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
|
||||
|
||||
check = smoke.check_total(
|
||||
"gb32960.raw",
|
||||
{
|
||||
"total": 1,
|
||||
"items": [{
|
||||
"ts": "2026-07-01 17:00:00",
|
||||
"received_at": "2026-07-01 17:58:00",
|
||||
}],
|
||||
},
|
||||
minimum=1,
|
||||
max_age_minutes=15,
|
||||
now=now,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("latest_age_minutes=2.0", check.message)
|
||||
|
||||
def test_total_check_passes_when_latest_sample_is_fresh(self):
|
||||
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
|
||||
|
||||
check = smoke.check_total(
|
||||
"gb32960.raw",
|
||||
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00"}]},
|
||||
minimum=1,
|
||||
max_age_minutes=30,
|
||||
now=now,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("latest_age_minutes=15.0", check.message)
|
||||
|
||||
def test_raw_check_requires_structured_parsed_json(self):
|
||||
check = smoke.check_total(
|
||||
"jt808.raw",
|
||||
{
|
||||
"total": 1,
|
||||
"items": [{
|
||||
"ts": "2026-07-01 17:45:00",
|
||||
"parsed_json": "",
|
||||
}],
|
||||
},
|
||||
minimum=1,
|
||||
require_parsed_json=True,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertIn("missing parsed_json", check.message)
|
||||
|
||||
def test_raw_check_accepts_structured_parsed_json(self):
|
||||
check = smoke.check_total(
|
||||
"gb32960.raw",
|
||||
{
|
||||
"total": 1,
|
||||
"items": [{
|
||||
"ts": "2026-07-01 17:45:00",
|
||||
"parsed_json": "{\"header\":{\"vin\":\"LTEST\"}}",
|
||||
}],
|
||||
},
|
||||
minimum=1,
|
||||
require_parsed_json=True,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("parsed_json=ok", check.message)
|
||||
|
||||
def test_history_check_requires_frame_id_backlink(self):
|
||||
check = smoke.check_total(
|
||||
"jt808.locations",
|
||||
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00", "frame_id": ""}]},
|
||||
minimum=1,
|
||||
require_frame_id=True,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertIn("missing frame_id", check.message)
|
||||
|
||||
def test_history_check_accepts_frame_id_backlink(self):
|
||||
check = smoke.check_total(
|
||||
"jt808.mileage_points",
|
||||
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00", "frame_id": "go_abc"}]},
|
||||
minimum=1,
|
||||
require_frame_id=True,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("frame_id=ok", check.message)
|
||||
|
||||
def test_daily_mileage_formula_requires_latest_minus_first(self):
|
||||
check = smoke.check_total(
|
||||
"jt808.daily_mileage",
|
||||
{
|
||||
"total": 1,
|
||||
"items": [{
|
||||
"metric_key": "daily_mileage_km",
|
||||
"metric_value": 40.0,
|
||||
"first_total_mileage_km": 4434.9,
|
||||
"latest_total_mileage_km": 4481.0,
|
||||
}],
|
||||
},
|
||||
minimum=1,
|
||||
require_metric_formula=True,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertIn("metric_formula mismatch", check.message)
|
||||
|
||||
def test_daily_total_formula_requires_latest_total(self):
|
||||
check = smoke.check_total(
|
||||
"jt808.daily_total_mileage",
|
||||
{
|
||||
"total": 1,
|
||||
"items": [{
|
||||
"metric_key": "daily_total_mileage_km",
|
||||
"metric_value": 4481.0,
|
||||
"first_total_mileage_km": 4434.9,
|
||||
"latest_total_mileage_km": 4481.0,
|
||||
}],
|
||||
},
|
||||
minimum=1,
|
||||
require_metric_formula=True,
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("metric_formula=ok", check.message)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
53
tools/test_go_prod_acceptance.py
Normal file
53
tools/test_go_prod_acceptance.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import argparse
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
from tools import go_prod_acceptance as acceptance
|
||||
|
||||
|
||||
class GoProdAcceptanceTest(unittest.TestCase):
|
||||
def test_build_commands_runs_systemd_kafka_and_http_smokes(self):
|
||||
args = argparse.Namespace(
|
||||
app_host="115.29.187.205",
|
||||
kafka_host="114.55.58.251",
|
||||
ssh_user="root",
|
||||
base_url="http://115.29.187.205:20210",
|
||||
date="2026-07-02",
|
||||
timeout=8.0,
|
||||
max_lag=100,
|
||||
)
|
||||
|
||||
commands = acceptance.build_commands(args)
|
||||
|
||||
self.assertEqual(len(commands), 3)
|
||||
self.assertEqual(commands[0].name, "systemd")
|
||||
self.assertEqual(commands[0].argv[:2], [sys.executable, "tools/go_systemd_prod_smoke.py"])
|
||||
self.assertIn("--host", commands[0].argv)
|
||||
self.assertIn("115.29.187.205", commands[0].argv)
|
||||
self.assertEqual(commands[1].name, "kafka")
|
||||
self.assertIn("tools/go_kafka_prod_smoke.py", commands[1].argv)
|
||||
self.assertIn("--max-lag", commands[1].argv)
|
||||
self.assertEqual(commands[2].name, "http")
|
||||
self.assertIn("tools/go_native_prod_smoke.py", commands[2].argv)
|
||||
self.assertIn("--base-url", commands[2].argv)
|
||||
|
||||
def test_overall_status_fails_when_any_child_fails(self):
|
||||
results = [
|
||||
acceptance.ChildResult("systemd", "pass", 0, {"status": "pass"}),
|
||||
acceptance.ChildResult("kafka", "fail", 1, {"status": "fail"}),
|
||||
]
|
||||
|
||||
self.assertEqual(acceptance.overall_status(results), "fail")
|
||||
|
||||
def test_overall_status_passes_when_all_children_pass(self):
|
||||
results = [
|
||||
acceptance.ChildResult("systemd", "pass", 0, {"status": "pass"}),
|
||||
acceptance.ChildResult("kafka", "pass", 0, {"status": "pass"}),
|
||||
acceptance.ChildResult("http", "pass", 0, {"status": "pass"}),
|
||||
]
|
||||
|
||||
self.assertEqual(acceptance.overall_status(results), "pass")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
101
tools/test_go_systemd_prod_smoke.py
Normal file
101
tools/test_go_systemd_prod_smoke.py
Normal file
@@ -0,0 +1,101 @@
|
||||
import unittest
|
||||
|
||||
from tools import go_systemd_prod_smoke as smoke
|
||||
|
||||
|
||||
class GoSystemdProdSmokeTest(unittest.TestCase):
|
||||
def test_service_checks_require_all_go_units_active(self):
|
||||
output = """
|
||||
lingniu-go-gateway.service active enabled
|
||||
lingniu-go-history-writer.service active enabled
|
||||
lingniu-go-stat-writer.service active enabled
|
||||
lingniu-go-realtime-api.service active enabled
|
||||
"""
|
||||
|
||||
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
|
||||
|
||||
self.assertTrue(all(check.status == "pass" for check in checks))
|
||||
|
||||
def test_service_checks_fail_when_unit_is_missing_or_inactive(self):
|
||||
output = """
|
||||
lingniu-go-gateway.service active enabled
|
||||
lingniu-go-history-writer.service failed enabled
|
||||
"""
|
||||
|
||||
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
|
||||
by_name = {check.name: check for check in checks}
|
||||
|
||||
self.assertEqual(by_name["service.lingniu-go-history-writer.service"].status, "fail")
|
||||
self.assertEqual(by_name["service.lingniu-go-stat-writer.service"].status, "fail")
|
||||
|
||||
def test_service_checks_fail_when_unit_is_not_enabled(self):
|
||||
output = """
|
||||
lingniu-go-gateway.service active disabled
|
||||
lingniu-go-history-writer.service active enabled
|
||||
lingniu-go-stat-writer.service active enabled
|
||||
lingniu-go-realtime-api.service active enabled
|
||||
"""
|
||||
|
||||
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
|
||||
by_name = {check.name: check for check in checks}
|
||||
|
||||
self.assertEqual(by_name["service.lingniu-go-gateway.service"].status, "fail")
|
||||
self.assertIn("enabled=disabled", by_name["service.lingniu-go-gateway.service"].message)
|
||||
|
||||
def test_port_checks_require_expected_go_processes(self):
|
||||
output = """
|
||||
LISTEN 0 4096 *:32960 *:* users:(("gateway",pid=10,fd=3))
|
||||
LISTEN 0 4096 *:808 *:* users:(("gateway",pid=10,fd=4))
|
||||
LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
|
||||
"""
|
||||
|
||||
checks = smoke.port_checks(output, smoke.DEFAULT_PORTS)
|
||||
|
||||
self.assertTrue(all(check.status == "pass" for check in checks))
|
||||
|
||||
def test_port_checks_fail_when_java_owns_ingest_port(self):
|
||||
output = """
|
||||
LISTEN 0 4096 *:808 *:* users:(("java",pid=20,fd=4))
|
||||
LISTEN 0 4096 *:32960 *:* users:(("gateway",pid=10,fd=3))
|
||||
LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
|
||||
"""
|
||||
|
||||
checks = smoke.port_checks(output, smoke.DEFAULT_PORTS)
|
||||
by_name = {check.name: check for check in checks}
|
||||
|
||||
self.assertEqual(by_name["port.808"].status, "fail")
|
||||
self.assertIn("java", by_name["port.808"].message)
|
||||
|
||||
def test_spool_check_passes_when_no_pending_files(self):
|
||||
check = smoke.spool_check("files=0 bytes=208896 recent=0")
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("files=0", check.message)
|
||||
|
||||
def test_spool_check_fails_when_pending_or_recent_files_exist(self):
|
||||
for output in ["files=2 bytes=300 recent=0", "files=0 bytes=208896 recent=1"]:
|
||||
check = smoke.spool_check(output)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
|
||||
def test_release_check_passes_when_current_release_has_all_binaries(self):
|
||||
check = smoke.release_check(
|
||||
"current=/opt/lingniu-go-native/releases/8def635 "
|
||||
"gateway=1 history-writer=1 stat-writer=1 realtime-api=1"
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "pass")
|
||||
self.assertIn("8def635", check.message)
|
||||
|
||||
def test_release_check_fails_when_binary_is_missing(self):
|
||||
check = smoke.release_check(
|
||||
"current=/opt/lingniu-go-native/releases/8def635 "
|
||||
"gateway=1 history-writer=0 stat-writer=1 realtime-api=1"
|
||||
)
|
||||
|
||||
self.assertEqual(check.status, "fail")
|
||||
self.assertIn("history-writer=0", check.message)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user