46 Commits

Author SHA1 Message Date
lingniu
cbb6f3b741 perf: skip raw freshness count scan 2026-07-02 09:30:15 +08:00
lingniu
42de422935 test: check raw freshness without event date filter 2026-07-02 09:27:22 +08:00
lingniu
318486b1e9 fix: verify raw freshness by received time 2026-07-02 09:16:45 +08:00
lingniu
0a024f9a76 chore: wait for gateway spool before acceptance 2026-07-02 09:11:50 +08:00
lingniu
ea9d9f5d35 chore: add go native ecs deploy script 2026-07-02 08:50:19 +08:00
lingniu
409f55b5ae test: verify go native release layout 2026-07-02 08:47:05 +08:00
lingniu
8def635bf5 test: verify realtime snapshots are fresh 2026-07-02 08:43:27 +08:00
lingniu
c211447747 test: verify daily metric formulas 2026-07-02 08:38:32 +08:00
lingniu
1741aacaaa test: verify history rows link raw frames 2026-07-02 08:35:34 +08:00
lingniu
27f4ad3468 test: verify raw parsed json in production smoke 2026-07-02 02:00:50 +08:00
lingniu
601150b364 test: verify gateway spool is drained 2026-07-02 01:57:53 +08:00
lingniu
5b7f386c5a test: verify go services are enabled 2026-07-02 01:55:39 +08:00
lingniu
f4d9c9ef5a test: add go production acceptance smoke 2026-07-02 01:53:43 +08:00
lingniu
96514898eb test: add go systemd production smoke 2026-07-02 01:51:41 +08:00
lingniu
0ef88e1a4c test: add kafka production smoke 2026-07-02 01:49:02 +08:00
lingniu
0fd4d319f9 docs: record go kafka production topics 2026-07-02 01:44:27 +08:00
lingniu
4e8434c511 test: verify mqtt history tables in production smoke 2026-07-02 01:41:08 +08:00
lingniu
576644eb65 test: verify gb32960 history tables in production smoke 2026-07-02 01:39:45 +08:00
lingniu
9bd48c5781 test: require fresh raw data in production smoke 2026-07-02 01:38:16 +08:00
lingniu
2214abe9c5 test: include realtime in production smoke 2026-07-02 01:36:14 +08:00
lingniu
1477b997c9 test: add go native production smoke 2026-07-02 01:34:28 +08:00
lingniu
4ea8d18972 docs: document go native ecs deployment 2026-07-02 01:31:58 +08:00
lingniu
947e645090 fix: query history with utc tdengine filters 2026-07-02 01:28:45 +08:00
lingniu
78b805e3a2 fix: normalize timezone history filters 2026-07-02 01:24:51 +08:00
lingniu
a7b24f4c0b feat: expose realtime snapshot protocols 2026-07-02 01:23:03 +08:00
lingniu
eb124d929b fix: accept datetime-local history filters 2026-07-02 01:19:19 +08:00
lingniu
4711287655 fix: return empty arrays for query APIs 2026-07-02 01:16:48 +08:00
lingniu
2c224396c9 fix: return real pagination totals 2026-07-02 01:14:06 +08:00
lingniu
4f2cded760 fix: treat yutong mqtt mileage as meters 2026-07-02 01:08:59 +08:00
lingniu
7ad1a25e8e fix: normalize yutong mqtt mileage units 2026-07-02 01:04:14 +08:00
lingniu
3ee1d6f52b fix: stabilize yutong mqtt ingestion 2026-07-02 01:01:33 +08:00
lingniu
6d1d0aa85e feat: write protocol acknowledgements from go gateway 2026-07-02 00:54:31 +08:00
lingniu
055373c405 fix: support extended gb32960 platform login 2026-07-02 00:47:01 +08:00
lingniu
e37dd39c89 fix: enrich jt808 location additional parsing 2026-07-02 00:36:51 +08:00
lingniu
dd73fdbb5b feat: add vehicle mileage history query 2026-07-02 00:29:49 +08:00
lingniu
fb9ab8525a feat: add vehicle location history query 2026-07-02 00:27:20 +08:00
lingniu
cc89e0d537 fix: filter raw frames by vehicle key 2026-07-02 00:24:53 +08:00
lingniu
f3ecf430cc fix: cache realtime snapshots by vehicle key 2026-07-02 00:21:50 +08:00
lingniu
bcf7cdae2e fix: key daily metrics by vehicle identity 2026-07-02 00:18:35 +08:00
lingniu
9045b871d6 fix: align jt808 parser with protocol variants 2026-07-02 00:14:02 +08:00
lingniu
b0150a5503 docs: record yutong mqtt eof diagnosis 2026-07-02 00:04:23 +08:00
lingniu
8f3b337fd6 docs: add native go deployment runbook 2026-07-02 00:02:21 +08:00
lingniu
594ab2d9c7 feat: support yutong mqtt tls config 2026-07-01 23:50:30 +08:00
lingniu
229ffcf61f docs: record go production port verification 2026-07-01 23:46:33 +08:00
lingniu
593713283c fix: query tdengine raw frames with literals 2026-07-01 23:42:09 +08:00
lingniu
6bee0499ae feat: expose raw frame query api 2026-07-01 23:39:10 +08:00
45 changed files with 5574 additions and 190 deletions

View File

@@ -44,11 +44,21 @@ services:
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt}
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-}
VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding}
ports:
- "${GO_GB32960_TCP_PORT:-32960}:32960"
- "${GO_JT808_TCP_PORT:-808}:808"
volumes:
- "${KAFKA_SPOOL_HOST_DIR:-/opt/lingniu-go/spool/gateway}:${KAFKA_SPOOL_DIR:-/data/spool/gateway}"
- "${YUTONG_MQTT_CERT_HOST_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:${YUTONG_MQTT_CERT_CONTAINER_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:ro"
go-history-writer:
<<: *restart-policy
@@ -96,6 +106,9 @@ services:
REDIS_DB: ${REDIS_DB:-50}
ONLINE_TTL_SECONDS: ${ONLINE_TTL_SECONDS:-600}
MYSQL_DSN: ${MYSQL_DSN:-}
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
TDENGINE_DSN: ${TDENGINE_DSN:-}
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
ports:
- "${GO_REALTIME_HTTP_PORT:-20210}:20210"

66
deploy/systemd/README.md Normal file
View File

@@ -0,0 +1,66 @@
# Go Vehicle Gateway Native Deployment
本目录用于 ECS 原生部署。当前 goal 后续不再使用 Docker/Portainer 作为 Go 接入链路的部署方式。
## Runtime Layout
```text
/opt/lingniu-go-native/
current -> /opt/lingniu-go-native/releases/<git-short-sha>
releases/<git-short-sha>/
gateway
history-writer
stat-writer
realtime-api
env/
gateway.env
history-writer.env
stat-writer.env
realtime-api.env
spool/gateway/
```
## Services
| systemd unit | Binary | Purpose |
|---|---|---|
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP `32960`、JT808 TCP `808`、宇通 MQTT 接入,写 Kafka RAW/unified |
| `lingniu-go-history-writer.service` | `history-writer` | 消费 RAW topic写 TDengine `raw_frames` 和核心时序表 |
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 RAW topic写 MySQL `vehicle_daily_metric` |
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费 unified topic写 Redis并提供 realtime/raw/stat 查询 API |
## Build
在开发机或 CI 上构建 Linux amd64 二进制:
```bash
cd go/vehicle-gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/gateway ./cmd/gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/history-writer ./cmd/history-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/stat-writer ./cmd/stat-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/realtime-api ./cmd/realtime-api
```
## Cutover Notes
1. 先上传二进制到新 release 目录并更新 `current` symlink。
2. 从现有生产 env 生成四个 systemd 专用 env 文件;不要把密钥写入仓库。
3. 停止旧 Docker Go 容器或 Java 容器,释放 `808``32960``20210`
4. 执行:
```bash
systemctl daemon-reload
systemctl enable lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
```
## Verification
```bash
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=JT808&limit=1'
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=GB32960&limit=1'
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=YUTONG_MQTT&limit=1'
```

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Vehicle Gateway
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/gateway.env
ExecStart=/opt/lingniu-go-native/current/gateway
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go History Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/history-writer.env
ExecStart=/opt/lingniu-go-native/current/history-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Realtime API
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/realtime-api.env
ExecStart=/opt/lingniu-go-native/current/realtime-api
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Stat Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/stat-writer.env
ExecStart=/opt/lingniu-go-native/current/stat-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,227 @@
# 当前 ECS Go 原生部署说明
更新时间2026-07-02 01:30 CST
本文档记录当前 `lingniu-vehicle-ingest` 的生产运行面。当前 goal 的接入链路已经切到 Go 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。密钥只维护在 ECS 环境文件或受控凭据中,不写入 Git。
## 部署范围
| systemd 服务 | 二进制 | 说明 | 对外端口 |
| --- | --- | --- | --- |
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP、JT808 TCP、宇通 MQTT 接入,解析后写 Kafka RAW 和统一事件 | TCP `32960`、TCP `808` |
| `lingniu-go-history-writer.service` | `history-writer` | 消费 Kafka RAW写 TDengine RAW、位置点、里程点核心表 | 无 HTTP |
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 Kafka RAW按总里程差值法写 MySQL 每日指标 | 无 HTTP |
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费统一事件写 Redis并提供实时、历史、统计查询 API | HTTP `20210` |
信达 Push 已废弃,不参与当前 Go 生产链路。旧 Java/Docker 服务不应占用 `808``32960``20210`
## 应用 ECS
| 项 | 值 |
| --- | --- |
| 公网 IP | `115.29.187.205` |
| 登录用户 | `root` |
| 部署目录 | `/opt/lingniu-go-native` |
| 当前 release | `/opt/lingniu-go-native/current` |
| 环境文件目录 | `/opt/lingniu-go-native/env` |
| systemd unit 目录 | `/etc/systemd/system` |
| 对外服务 | `32960``808``20210` |
运行目录结构:
```text
/opt/lingniu-go-native/
current -> /opt/lingniu-go-native/releases/<git-short-sha>
releases/<git-short-sha>/
gateway
history-writer
stat-writer
realtime-api
env/
gateway.env
history-writer.env
stat-writer.env
realtime-api.env
spool/gateway/
```
## 中间件
| 组件 | 内网地址 | 公网地址 | 用途 |
| --- | --- | --- | --- |
| Kafka | `172.17.111.56:9092` | `114.55.58.251:9092` | RAW 和统一事件消息总线 |
| TDengine | `172.17.111.57:6041` | `115.29.185.82:6041` | RAW、位置、里程点时序热存储 |
| MySQL RDS | `rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306` | `rm-bp179zbv481rnw3e2no.mysql.rds.aliyuncs.com:3306` | 身份映射、JT808 注册、每日指标 |
| Redis RDS | `r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379` | 无 | 准实时车辆状态缓存 |
生产应用之间访问中间件优先使用内网地址。RDS 白名单需要允许应用 ECS 私网访问。
## Kafka Topic
| Topic | 生产者 | 消费者 | 内容 |
| --- | --- | --- | --- |
| `vehicle.raw.go.gb32960.v1` | `gateway` | `history-writer``stat-writer` | GB32960 完整 RAW 记录 |
| `vehicle.raw.go.jt808.v1` | `gateway` | `history-writer``stat-writer` | JT808 完整 RAW 记录 |
| `vehicle.raw.go.yutong-mqtt.v1` | `gateway` | `history-writer` | 宇通 MQTT 完整 RAW 记录 |
| `vehicle.event.go.unified.v1` | `gateway` | `realtime-api` | 三类协议统一事件,用于 Redis 实时状态 |
`gateway` 配置了 Kafka retry生产 env 中启用 `KAFKA_SPOOL_DIR` 时,本地 spool 可在 Kafka 短暂不可用后回放。
Kafka 消费组:
| Consumer Group | Topic | 当前验证 |
| --- | --- | --- |
| `go-history-writer` | 三个 `vehicle.raw.go.*` topic | 2026-07-02 验证 lag 为 `0` |
| `go-stat-writer` | `vehicle.raw.go.gb32960.v1``vehicle.raw.go.jt808.v1` | 2026-07-02 验证 lag 为 `0` |
| `go-realtime-api` | `vehicle.event.go.unified.v1` | 2026-07-02 验证 lag 为 `0` |
可重复 smoke
```bash
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
```
该脚本通过 SSH 登录 Kafka ECS检查 Go 生产 topic 是否存在,并校验 `go-history-writer``go-stat-writer``go-realtime-api` 的消费组 lag。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
## TDengine 数据层
默认数据库:`lingniu_vehicle_ts`
| 表 | 类型 | 说明 |
| --- | --- | --- |
| `raw_frames` | stable | 完整 RAW 帧,包含 `raw_hex``parsed_json``fields_json``source_endpoint` 和协议/车辆 tags |
| `vehicle_locations` | stable | 最小化位置历史,保留 `frame_id` 回链 RAW |
| `vehicle_mileage_points` | stable | 最小化总里程点,供历史查询和统计复核 |
设计原则:
- 完整结构化 payload 只落在 `raw_frames.parsed_json` / `fields_json`
- `vehicle_locations``vehicle_mileage_points` 只保留核心查询字段,不重复保存完整 JSON。
- `telemetry_fields` 不由当前服务维护,字段配置解析由独立子服务处理。
- API 传入东八区时间时,服务会转换为 TDengine 当前存储使用的 UTC 字面量再查询。
## MySQL 数据层
默认业务库:`lingniu_vehicle_data`
| 表 | 维护方 | 说明 |
| --- | --- | --- |
| `vehicle_identity_binding` | 人工/外部主数据 | VIN 映射主表,用 `phone``device_id``plate` 降级定位 VIN |
| `jt808_registration` | `gateway` 自动写入 | 仅 JT808 注册和鉴权记录,以 `phone` 作为主键,记录设备、车牌、厂家、鉴权、来源端点 |
| `vehicle_daily_metric` | `stat-writer` 自动写入 | 每日指标表,保存 `daily_mileage_km``daily_total_mileage_km` |
每日里程算法:
```text
daily_mileage_km = 当日最大 total_mileage_km - 当日最小 total_mileage_km
daily_total_mileage_km = 当日最大 total_mileage_km
```
统计按 `Asia/Shanghai` 自然日归档,当前支持 GB32960、JT808、宇通 MQTT 中能解析出 `total_mileage_km` 的数据。JT808 的总里程来自位置附加信息 `0x01`,单位按协议转换为 km宇通 MQTT 的 `TOTAL_MILEAGE` 原始单位为米,入库前转换为 km。
## Redis 实时层
`realtime-api` 消费 `vehicle.event.go.unified.v1`,按 VIN/vehicle key 维护准实时快照。主要能力:
| API | 说明 |
| --- | --- |
| `/api/realtime/vehicles/{vin}` | 查询车辆合并实时快照 |
| `/api/realtime/vehicles/{vin}/online` | 查询车辆是否在线 |
| `/api/realtime/vehicles/{vin}/protocols/{protocol}` | 查询指定协议的实时快照 |
Redis 只作为实时缓存;历史和统计以 TDengine/MySQL 为准。
## API 入口
公网基础地址:
```text
http://115.29.187.205:20210
```
常用查询:
```bash
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=YUTONG_MQTT&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/locations?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/mileage-points?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/stats/daily-metrics?dateFrom=2026-07-02&dateTo=2026-07-02&limit=20'
```
分页参数统一使用 `limit``offset`
部署后推荐直接运行 Go 原生生产 smoke
```bash
python3 tools/go_prod_acceptance.py --date 2026-07-02
```
该聚合脚本会串行执行 systemd/端口检查、Kafka topic/consumer lag 检查、HTTP 数据链路检查,任一子检查失败时整体退出码为非 0。需要 SSH 可登录应用 ECS 和 Kafka ECS。
也可以单独运行子检查:
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
```
`go_systemd_prod_smoke.py` 通过 SSH 检查四个 Go systemd 服务是否 `active``enabled`,并确认 `808``32960``gateway` 监听、`20210``realtime-api` 监听,同时要求 `/opt/lingniu-go-native/current` 指向有效 release、四个生产二进制均可执行、`/opt/lingniu-go-native/spool/gateway` 没有待回放文件,避免旧 Java/Docker 进程占用生产端口、部署目录错乱或 Kafka spool 静默堆积。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
`go_native_prod_smoke.py` 通过 `20210` HTTP API 验证 GB32960、JT808、宇通 MQTT 的 RAW 查询及结构化 `parsed_json`,三类协议的位置和里程点查询及 `frame_id` RAW 回链GB32960/JT808 的 `daily_mileage_km``daily_total_mileage_km` 统计公式,以及三类协议的 Redis realtime snapshot、online、protocol 查询。实时 snapshot/protocol 查询会校验 `updated_at_ms` 新鲜度;默认检查今天东八区数据时,还会要求三类 RAW 最新样本不超过 15 分钟,避免旧数据误判为接收正常;任一检查不达标时退出码为非 0。
## 部署命令
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收:
```bash
python3 tools/go_native_deploy.py --date 2026-07-02
```
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout``--spool-poll-interval` 调整。
脚本执行的发布动作等价于:
```bash
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/history-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/stat-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/realtime-api
scp release.tar.gz root@115.29.187.205:/tmp/
ln -sfn /opt/lingniu-go-native/releases/<git-short-sha> /opt/lingniu-go-native/current
systemctl daemon-reload
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
```
只更新查询 API 时,可以仅替换 `/opt/lingniu-go-native/current/realtime-api` 并重启:
```bash
systemctl restart lingniu-go-realtime-api.service
```
## 运行检查
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-history-writer --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-stat-writer --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-realtime-api --since '5 minutes ago' --no-pager
```
2026-07-02 01:30 CST 已验证:
- 四个 Go systemd 服务均为 `active``enabled`
- `/opt/lingniu-go-native/current` 指向有效 release四个生产二进制均存在且可执行。
- `gateway` 正在监听 `32960``808`
- `realtime-api` 正在监听 `20210`
- gateway Kafka spool 当前无待回放文件。
- GB32960、JT808、YUTONG_MQTT 的东八区 RAW 查询均可命中生产数据,且最新 RAW 样本包含结构化 `parsed_json`
- GB32960、JT808、YUTONG_MQTT 的最新 RAW 样本均在 15 分钟内。
- GB32960、JT808、YUTONG_MQTT 的 `vehicle_locations``vehicle_mileage_points` 查询均可命中生产数据,且样本包含 `frame_id` 回链 RAW。
- `vehicle_daily_metric` 可查到 `daily_mileage_km``daily_total_mileage_km`,且公式满足 `daily_mileage_km = latest_total_mileage_km - first_total_mileage_km``daily_total_mileage_km = latest_total_mileage_km`
- Redis realtime 可查到 GB32960、JT808、YUTONG_MQTT 的在线状态和协议快照,且 snapshot/protocol 快照最近刷新。

View File

@@ -657,3 +657,232 @@ http://115.29.187.205:20210/api/realtime/vehicles/LKLG7C4E3NA774736/online
```
说明:本次发布重启窗口产生的 spool 正在回放下降,且最近 1 分钟没有新增gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。
## 2026-07-01 23:50 生产端口接管与 RAW 查询 API 复验
部署版本:
- Git commit`5937132`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5937132-20260701234220`
生产端口状态:
```text
go-vehicle-gateway 0.0.0.0:808->808/tcp
go-vehicle-gateway 0.0.0.0:32960->32960/tcp
```
旧 Java 容器状态:
```text
gb32960-ingest-app restart=no status=exited
jt808-ingest-app restart=no status=exited
yutong-mqtt-app restart=no status=exited
vehicle-history-app restart=no status=exited
vehicle-analytics-app restart=no status=exited
vehicle-state-app restart=no status=exited
telemetry-field-parser-app restart=no status=exited
```
生产连接确认:
- JT808 端口 `808` 已收到外部连接:`222.66.200.68``115.231.168.135`
- GB32960 端口 `32960` 已收到外部连接:`8.134.95.166``117.160.0.65`
- ECS 当前没有 `8089` 监听。
RAW 查询 API
```text
GET /api/history/raw-frames
```
生产可访问样例:
```text
http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
```
接口复验结果:
```text
JT808_RAW:
total=1
protocol=JT808
vin=LKLG7C4E3NA774736
phone=013079963379
message_id=512
message_id_hex=0x0200
parse_status=OK
raw_hex_len=94
parsed_json_len=651
fields_json_len=186
GB32960_RAW:
total=1
protocol=GB32960
vin=LB9A32A21R0LS1707
message_id=2
message_id_hex=0x0002
parse_status=OK
raw_hex_len=1548
parsed_json_len=3698
fields_json_len=563
```
同轮回归验证:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
online=true
protocols=["JT808"]
GET /api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2026-07-01&limit=1
total=1
protocol=GB32960
metric_key=daily_mileage_km
calculation_method=TOTAL_MILEAGE_DIFF
```
说明:
- RAW 查询已返回 `raw_hex``parsed_json``fields_json`,可以用于查看完整解析结果。
- 日统计和实时查询在生产端口接管后仍可用。
## 2026-07-02 00:00 原生 systemd 部署与宇通 MQTT 复验
根据本次 goal 的新要求Go 接入链路后续不再使用 Docker 部署。本轮已中断 Docker 镜像构建路径,改为 ECS 原生 Linux 二进制 + systemd。
部署版本:
- Git commit`594ab2d`
- 部署目录:`/opt/lingniu-go-native`
- 当前 release`/opt/lingniu-go-native/releases/594ab2d`
- 当前指针:`/opt/lingniu-go-native/current -> /opt/lingniu-go-native/releases/594ab2d`
systemd 服务:
```text
lingniu-go-gateway.service active
lingniu-go-history-writer.service active
lingniu-go-stat-writer.service active
lingniu-go-realtime-api.service active
```
端口确认:
```text
[::]:32960 gateway
[::]:808 gateway
[::]:20210 realtime-api
```
Go Docker 容器已停止:
```text
go-vehicle-gateway Exited
go-history-writer Exited
go-stat-writer Exited
go-realtime-api Exited
```
宇通 MQTT
```text
yutong mqtt client started
mqtt subscribed broker=ssl://cpxlm.axxc.cn:38883 topic=/ytforward/shln/+ qos=1
```
说明MQTT broker 存在短连接 `EOF` 后自动重连现象,但订阅后已收到真实数据并落 RAW。
RAW 查询复验:
```text
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=3
total=3
protocol=YUTONG_MQTT
vin=LMRKH9AC1R1004131
vehicle_key=LMRKH9AC1R1004131
parse_status=OK
parsed_json_len=1104
fields_json_len=116
```
同轮回归:
```text
GET /api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
total=1
parse_status=OK
GET /api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
total=1
parse_status=OK
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
online=true
protocols=["JT808"]
```
本轮落地文件:
- `deploy/systemd/README.md`
- `deploy/systemd/lingniu-go-gateway.service`
- `deploy/systemd/lingniu-go-history-writer.service`
- `deploy/systemd/lingniu-go-stat-writer.service`
- `deploy/systemd/lingniu-go-realtime-api.service`
## 2026-07-02 00:04 宇通 MQTT EOF 诊断
现象:
Go 原生部署后,`lingniu-go-gateway` 日志中出现多次:
```text
mqtt connection lost error=EOF
mqtt connection lost error=write: broken pipe
mqtt subscribed topic=/ytforward/shln/+ qos=1
```
10 分钟窗口统计:
```text
mqtt subscribed = 33
mqtt connection lost = 32
```
对照旧 Java `yutong-mqtt-app` 历史日志,停止前也存在同类行为:
```text
mqtt endpoint [yutong] received topic=/ytforward/shln/1 bytes=574
mqtt endpoint [yutong] connection lost
Caused by: java.io.EOFException
mqtt endpoint [yutong] re-subscribed topic=/ytforward/shln/+ qos=1
mqtt endpoint [yutong] received topic=/ytforward/shln/3 bytes=586
```
判断:
- EOF/短连接不是 Go TLS 支持或 systemd 原生部署新引入的问题。
- 旧 Java 和新 Go 都表现为订阅、收到数据、broker 断开、自动重连。
- 当前链路可持续收到宇通 MQTT 数据并写入 TDengine RAW。
复验:
```text
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=1
total=1
protocol=YUTONG_MQTT
vin=LMRKH9ACXR1004094
parse_status=OK
parsed_json_len=1080
fields_json_len=115
```
运维建议:
- 单条 EOF 不作为不可用判断。
-`YUTONG_MQTT` RAW 入库增长、最近成功 `mqtt subscribed` 时间、服务 `systemctl is-active` 作为可用性判断。
- 如果 EOF 频率继续升高且 RAW 不再增长,再联系宇通侧确认 broker 长连接策略、同一 clientId 并发限制或网络出口策略。

View File

@@ -18,32 +18,40 @@ go build ./cmd/stat-writer
go build ./cmd/realtime-api
```
## 镜像构建
## 生产原生部署验证
```bash
cd go/vehicle-gateway
docker build -t vehicle-gateway-go:local .
python3 tools/go_prod_acceptance.py --date 2026-07-02
```
同一个镜像包含四个二进制
该命令会聚合执行
- `/app/gateway`
- `/app/history-writer`
- `/app/stat-writer`
- `/app/realtime-api`
- `go_systemd_prod_smoke.py`:应用 ECS systemd 服务 active/enabled、端口归属和 gateway spool
- `go_kafka_prod_smoke.py`Kafka topic 和消费组 lag
- `go_native_prod_smoke.py`HTTP API、RAW `parsed_json`、历史核心表 `frame_id` 回链、每日指标公式、Redis 实时新鲜度、TDengine/MySQL/Redis 数据链路
## Portainer 部署
Go 旁路 stack 文件:
如需单独检查 systemd
```bash
deploy/portainer/docker-compose-go.yml
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
```
必填变量
脚本通过 SSH 检查
- `lingniu-go-gateway.service`
- `lingniu-go-history-writer.service`
- `lingniu-go-stat-writer.service`
- `lingniu-go-realtime-api.service`
- 上述服务均为 `active``enabled`
- `808``32960``gateway` 监听
- `20210``realtime-api` 监听
- `/opt/lingniu-go-native/spool/gateway` 无待回放文件
## 生产环境变量
当前 goal 的生产面使用 ECS 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。以下环境变量维护在 `/opt/lingniu-go-native/env/*.env`
```bash
LINGNIU_GO_IMAGE_VERSION=<image-version>
KAFKA_BROKERS=172.17.111.56:9092
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
@@ -68,23 +76,44 @@ YUTONG_MQTT_PASSWORD=<password>
## ECS 进程检查
```bash
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' \
| egrep 'go-vehicle-gateway|go-history-writer|go-stat-writer|go-realtime-api|NAMES'
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
```
## Kafka 验证
优先使用 smoke 脚本做可重复检查:
```bash
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
```
脚本会检查 `vehicle.raw.go.gb32960.v1``vehicle.raw.go.jt808.v1``vehicle.raw.go.yutong-mqtt.v1``vehicle.event.go.unified.v1` 是否存在,并汇总 `go-history-writer``go-stat-writer``go-realtime-api` 的消费 lag。运行环境需要 SSH 免密或已建立可用的 SSH 认证方式。
手工核对命令:
```bash
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.jt808.v1 --max-messages 1 --timeout-ms 10000
--topic vehicle.raw.go.jt808.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.gb32960.v1 --max-messages 1 --timeout-ms 10000
--topic vehicle.raw.go.gb32960.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.event.unified.v1 --max-messages 1 --timeout-ms 10000
--topic vehicle.raw.go.yutong-mqtt.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.event.go.unified.v1 --max-messages 1 --timeout-ms 10000
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-history-writer
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-stat-writer
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-realtime-api
```
## TDengine 验证

View File

@@ -46,12 +46,14 @@ func main() {
Addr: env("GB32960_TCP_ADDR", ":32960"),
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Respond: gb32960.AutoResponse,
},
{
Protocol: envelope.ProtocolJT808,
Addr: env("JT808_TCP_ADDR", ":808"),
Extract: jt808.ExtractFrames,
Parse: jt808.ParseFrame,
Respond: jt808.NewAutoResponder(env("JT808_REGISTER_AUTH_CODE", "g7gps")).Respond,
},
}
@@ -81,16 +83,23 @@ func main() {
}
if envBool("YUTONG_MQTT_ENABLED", false) {
client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{
EndpointName: env("YUTONG_MQTT_ENDPOINT", "yutong"),
Broker: env("YUTONG_MQTT_URI", ""),
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
Username: env("YUTONG_MQTT_USERNAME", ""),
Password: env("YUTONG_MQTT_PASSWORD", ""),
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
Sink: sink,
Resolver: resolver,
Logger: logger,
EndpointName: env("YUTONG_MQTT_ENDPOINT", env("YUTONG_MQTT_ENDPOINT_NAME", "yutong")),
Broker: env("YUTONG_MQTT_URI", ""),
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
Username: env("YUTONG_MQTT_USERNAME", ""),
Password: env("YUTONG_MQTT_PASSWORD", ""),
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
CleanSession: envBool("YUTONG_MQTT_CLEAN_SESSION", false),
KeepAlive: time.Duration(envInt("YUTONG_MQTT_KEEP_ALIVE_SECONDS", 20)) * time.Second,
ConnectTimeout: time.Duration(envInt("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", 10)) * time.Second,
TLSCACertPath: env("YUTONG_MQTT_TLS_CA_PEM", ""),
TLSClientCertPath: env("YUTONG_MQTT_TLS_CLIENT_PEM", ""),
TLSClientKeyPath: env("YUTONG_MQTT_TLS_CLIENT_KEY", ""),
TLSHostnameVerification: envBool("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", true),
Sink: sink,
Resolver: resolver,
Logger: logger,
})
if err != nil {
logger.Error("build yutong mqtt client failed", "error", err)

View File

@@ -15,8 +15,10 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/redis/go-redis/v9"
"github.com/segmentio/kafka-go"
_ "github.com/taosdata/driver-go/v3/taosWS"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
@@ -74,6 +76,37 @@ func main() {
logger.Warn("MYSQL_DSN is empty; stats query api disabled")
}
defer closeStats()
closeHistory := func() {}
if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" {
driver := env("TDENGINE_DRIVER", "taosWS")
db, err := sql.Open(driver, dsn)
if err != nil {
logger.Error("history tdengine open failed", "driver", driver, "error", err)
os.Exit(1)
}
if err := db.PingContext(ctx); err != nil {
_ = db.Close()
logger.Error("history tdengine ping failed", "driver", driver, "error", err)
os.Exit(1)
}
closeHistory = func() { _ = db.Close() }
database := env("TDENGINE_DATABASE", history.DefaultDatabase)
mux.Handle("/api/history/raw-frames", history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)))
mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database)))
mux.Handle("/api/history/mileage-points", history.NewMileagePointHandler(history.NewMileagePointRepository(db, database)))
logger.Info("history query enabled", "driver", driver, "database", database)
} else {
historyUnavailable := func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"})
}
mux.HandleFunc("/api/history/raw-frames", historyUnavailable)
mux.HandleFunc("/api/history/locations", historyUnavailable)
mux.HandleFunc("/api/history/mileage-points", historyUnavailable)
logger.Warn("TDENGINE_DSN is empty; history query api disabled")
}
defer closeHistory()
server := &http.Server{
Addr: env("HTTP_ADDR", ":20210"),

View File

@@ -2,9 +2,13 @@ package gateway
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"errors"
"fmt"
"log/slog"
"os"
"strings"
"time"
@@ -17,16 +21,23 @@ import (
)
type MQTTClientConfig struct {
EndpointName string
Broker string
ClientID string
Username string
Password string
Topics []string
QoS byte
Sink eventbus.Sink
Resolver identity.Resolver
Logger *slog.Logger
EndpointName string
Broker string
ClientID string
Username string
Password string
Topics []string
QoS byte
CleanSession bool
KeepAlive time.Duration
ConnectTimeout time.Duration
TLSCACertPath string
TLSClientCertPath string
TLSClientKeyPath string
TLSHostnameVerification bool
Sink eventbus.Sink
Resolver identity.Resolver
Logger *slog.Logger
}
type MQTTClient struct {
@@ -60,17 +71,47 @@ func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
}
func (c *MQTTClient) Start(ctx context.Context) error {
opts, err := c.buildOptions(ctx)
if err != nil {
return err
}
c.client = mqtt.NewClient(opts)
token := c.client.Connect()
if token.Wait() && token.Error() != nil {
return token.Error()
}
go func() {
<-ctx.Done()
if c.client != nil && c.client.IsConnected() {
c.client.Disconnect(250)
}
}()
return nil
}
func (c *MQTTClient) buildOptions(ctx context.Context) (*mqtt.ClientOptions, error) {
keepAlive := c.cfg.KeepAlive
if keepAlive <= 0 {
keepAlive = 20 * time.Second
}
connectTimeout := c.cfg.ConnectTimeout
if connectTimeout <= 0 {
connectTimeout = 10 * time.Second
}
opts := mqtt.NewClientOptions().
AddBroker(c.cfg.Broker).
SetClientID(c.cfg.ClientID).
SetUsername(c.cfg.Username).
SetPassword(c.cfg.Password).
SetCleanSession(false).
SetCleanSession(c.cfg.CleanSession).
SetAutoReconnect(true).
SetConnectRetry(true).
SetConnectRetryInterval(5 * time.Second).
SetKeepAlive(20 * time.Second).
SetConnectTimeout(10 * time.Second)
SetResumeSubs(true).
SetOrderMatters(false).
SetKeepAlive(keepAlive).
SetConnectTimeout(connectTimeout)
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
c.handleMessage(ctx, message.Topic(), message.Payload())
@@ -89,18 +130,49 @@ func (c *MQTTClient) Start(ctx context.Context) error {
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
}
c.client = mqtt.NewClient(opts)
token := c.client.Connect()
if token.Wait() && token.Error() != nil {
return token.Error()
tlsConfig, err := c.buildTLSConfig()
if err != nil {
return nil, err
}
go func() {
<-ctx.Done()
if c.client != nil && c.client.IsConnected() {
c.client.Disconnect(250)
if tlsConfig != nil {
opts.SetTLSConfig(tlsConfig)
}
return opts, nil
}
func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
caPath := strings.TrimSpace(c.cfg.TLSCACertPath)
certPath := strings.TrimSpace(c.cfg.TLSClientCertPath)
keyPath := strings.TrimSpace(c.cfg.TLSClientKeyPath)
if caPath == "" && certPath == "" && keyPath == "" {
return nil, nil
}
config := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: !c.cfg.TLSHostnameVerification,
}
if caPath != "" {
caPEM, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("read mqtt ca certificate: %w", err)
}
}()
return nil
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(caPEM) {
return nil, fmt.Errorf("parse mqtt ca certificate %s", caPath)
}
config.RootCAs = roots
}
if certPath != "" || keyPath != "" {
if certPath == "" || keyPath == "" {
return nil, errors.New("mqtt client certificate and key must be configured together")
}
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, fmt.Errorf("load mqtt client certificate: %w", err)
}
config.Certificates = []tls.Certificate{cert}
}
return config, nil
}
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {

View File

@@ -2,8 +2,17 @@ package gateway
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"log/slog"
"math/big"
"os"
"path/filepath"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
@@ -59,3 +68,116 @@ func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
}
}
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
dir := t.TempDir()
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "ssl://mqtt.example.test:8883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
QoS: 1,
Sink: &recordingSink{},
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
TLSCACertPath: caPath,
TLSClientCertPath: certPath,
TLSClientKeyPath: keyPath,
TLSHostnameVerification: false,
CleanSession: true,
KeepAlive: 20 * time.Second,
ConnectTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
opts, err := client.buildOptions(context.Background())
if err != nil {
t.Fatalf("buildOptions() error = %v", err)
}
if opts.TLSConfig == nil {
t.Fatal("TLSConfig is nil")
}
if opts.TLSConfig.RootCAs == nil {
t.Fatal("RootCAs is nil")
}
if len(opts.TLSConfig.Certificates) != 1 {
t.Fatalf("client certificates = %d, want 1", len(opts.TLSConfig.Certificates))
}
if !opts.TLSConfig.InsecureSkipVerify {
t.Fatal("InsecureSkipVerify should be true when hostname verification is disabled")
}
if !opts.CleanSession {
t.Fatal("CleanSession should be true")
}
if opts.Order {
t.Fatal("OrderMatters should be false so MQTT network handling is not blocked by Kafka/DB work")
}
if !opts.ResumeSubs {
t.Fatal("ResumeSubs should be true to avoid subscribe failures during reconnect churn")
}
if got := opts.KeepAlive; got != 20 {
t.Fatalf("KeepAlive = %d, want 20", got)
}
if got := opts.ConnectTimeout; got != 10*time.Second {
t.Fatalf("ConnectTimeout = %v, want 10s", got)
}
}
func writeTestTLSMaterial(t *testing.T, dir string) (string, string, string) {
t.Helper()
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate ca key: %v", err)
}
caTemplate := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "test-ca"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
BasicConstraintsValid: true,
IsCA: true,
}
caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
if err != nil {
t.Fatalf("create ca cert: %v", err)
}
clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate client key: %v", err)
}
clientTemplate := &x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{CommonName: "test-client"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey)
if err != nil {
t.Fatalf("create client cert: %v", err)
}
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "client.pem")
keyPath := filepath.Join(dir, "client-key.pem")
writePEM(t, caPath, "CERTIFICATE", caDER)
writePEM(t, certPath, "CERTIFICATE", clientDER)
keyDER := x509.MarshalPKCS1PrivateKey(clientKey)
writePEM(t, keyPath, "RSA PRIVATE KEY", keyDER)
return caPath, certPath, keyPath
}
func writePEM(t *testing.T, path, typ string, der []byte) {
t.Helper()
file, err := os.Create(path)
if err != nil {
t.Fatalf("create %s: %v", path, err)
}
defer file.Close()
if err := pem.Encode(file, &pem.Block{Type: typ, Bytes: der}); err != nil {
t.Fatalf("write pem %s: %v", path, err)
}
}

View File

@@ -21,11 +21,14 @@ type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
type TCPProtocol struct {
Protocol envelope.Protocol
Addr string
Extract FrameExtractor
Parse FrameParser
Respond FrameResponder
}
type TCPServer struct {
@@ -153,7 +156,7 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
}
pending = remainder
for _, frame := range frames {
s.handleFrame(ctx, frame, source)
s.handleFrame(ctx, conn, frame, source)
}
}
if err != nil {
@@ -174,7 +177,7 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
}
}
func (s *TCPServer) handleFrame(ctx context.Context, raw []byte, source string) {
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string) {
receivedAtMS := time.Now().UnixMilli()
env, err := s.protocol.Parse(raw, receivedAtMS, source)
if err != nil {
@@ -213,6 +216,21 @@ func (s *TCPServer) handleFrame(ctx context.Context, raw []byte, source string)
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if s.protocol.Respond == nil {
return
}
response, ok, err := s.protocol.Respond(raw, env)
if err != nil {
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if !ok || len(response) == 0 {
return
}
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := conn.Write(response); err != nil {
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
}
}
func (p TCPProtocol) String() string {

View File

@@ -3,6 +3,7 @@ package gateway
import (
"context"
"encoding/hex"
"io"
"log/slog"
"net"
"testing"
@@ -73,6 +74,37 @@ func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
}
}
func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
frame := buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil)
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolGB32960,
Addr: ":0",
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
return []byte("ACK"), true, nil
},
}, sink)
client, done := runPipe(t, server)
if _, err := client.Write(frame); err != nil {
t.Fatalf("client.Write() error = %v", err)
}
buf := make([]byte, 3)
if _, err := io.ReadFull(client, buf); err != nil {
t.Fatalf("read response error = %v", err)
}
if string(buf) != "ACK" {
t.Fatalf("response = %q", string(buf))
}
_ = client.Close()
<-done
}
func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer {
t.Helper()
server, err := NewTCPServer(TCPServerConfig{

View File

@@ -0,0 +1,908 @@
package history
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
)
type Queryer interface {
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
}
type RawFrameQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
MessageID string
OrderBy string
IncludeTotal bool
DateFrom string
DateTo string
Limit int
Offset int
}
type RawFrameRow struct {
TS string `json:"ts"`
FrameID string `json:"frame_id"`
EventID string `json:"event_id"`
MessageID int64 `json:"message_id"`
MessageIDHex string `json:"message_id_hex"`
EventTime string `json:"event_time"`
ReceivedAt string `json:"received_at"`
RawSizeBytes int64 `json:"raw_size_bytes"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
ParsedJSON string `json:"parsed_json,omitempty"`
FieldsJSON string `json:"fields_json,omitempty"`
ParseStatus string `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
SourceEndpoint string `json:"source_endpoint"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type LocationQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
DateFrom string
DateTo string
Limit int
Offset int
}
type LocationRow struct {
TS string `json:"ts"`
EventID string `json:"event_id"`
FrameID string `json:"frame_id"`
ReceivedAt string `json:"received_at"`
Longitude float64 `json:"longitude"`
Latitude float64 `json:"latitude"`
AltitudeM *float64 `json:"altitude_m,omitempty"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
DirectionDeg *int64 `json:"direction_deg,omitempty"`
AlarmFlag *int64 `json:"alarm_flag,omitempty"`
StatusFlag *int64 `json:"status_flag,omitempty"`
TotalMileageKM *float64 `json:"total_mileage_km,omitempty"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type MileagePointQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
DateFrom string
DateTo string
Limit int
Offset int
}
type MileagePointRow struct {
TS string `json:"ts"`
EventID string `json:"event_id"`
FrameID string `json:"frame_id"`
ReceivedAt string `json:"received_at"`
TotalMileageKM float64 `json:"total_mileage_km"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type RawFrameRepository struct {
db Queryer
database string
}
func NewRawFrameRepository(db Queryer, database string) *RawFrameRepository {
if db == nil {
panic("raw frame query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &RawFrameRepository{db: db, database: database}
}
type LocationRepository struct {
db Queryer
database string
}
type MileagePointRepository struct {
db Queryer
database string
}
func NewLocationRepository(db Queryer, database string) *LocationRepository {
if db == nil {
panic("location query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &LocationRepository{db: db, database: database}
}
func NewMileagePointRepository(db Queryer, database string) *MileagePointRepository {
if db == nil {
panic("mileage point query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &MileagePointRepository{db: db, database: database}
}
func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) {
query = normalizeRawFrameQuery(query)
sqlText, args := buildRawFrameSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]RawFrameRow, 0)
for rows.Next() {
var row RawFrameRow
var ts scanDateTime
var eventTime scanDateTime
var receivedAt scanDateTime
if err := rows.Scan(
&ts,
&row.FrameID,
&row.EventID,
&row.MessageID,
&eventTime,
&receivedAt,
&row.RawSizeBytes,
&row.RawHex,
&row.RawText,
&row.ParsedJSON,
&row.FieldsJSON,
&row.ParseStatus,
&row.ParseError,
&row.SourceEndpoint,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.EventTime = eventTime.String
row.ReceivedAt = receivedAt.String
row.MessageIDHex = fmt.Sprintf("0x%04X", row.MessageID)
out = append(out, row)
}
return out, rows.Err()
}
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
query = normalizeRawFrameQuery(query)
sqlText, args := buildRawFrameCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]LocationRow, error) {
query = normalizeLocationQuery(query)
sqlText, args := buildLocationSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]LocationRow, 0)
for rows.Next() {
var row LocationRow
var ts scanDateTime
var receivedAt scanDateTime
var altitude sql.NullFloat64
var speed sql.NullFloat64
var direction sql.NullInt64
var alarm sql.NullInt64
var status sql.NullInt64
var mileage sql.NullFloat64
if err := rows.Scan(
&ts,
&row.EventID,
&row.FrameID,
&receivedAt,
&row.Longitude,
&row.Latitude,
&altitude,
&speed,
&direction,
&alarm,
&status,
&mileage,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.ReceivedAt = receivedAt.String
row.AltitudeM = nullableFloat(altitude)
row.SpeedKMH = nullableFloat(speed)
row.DirectionDeg = nullableInt(direction)
row.AlarmFlag = nullableInt(alarm)
row.StatusFlag = nullableInt(status)
row.TotalMileageKM = nullableFloat(mileage)
out = append(out, row)
}
return out, rows.Err()
}
func (r *LocationRepository) Count(ctx context.Context, query LocationQuery) (int64, error) {
query = normalizeLocationQuery(query)
sqlText, args := buildLocationCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func (r *MileagePointRepository) Query(ctx context.Context, query MileagePointQuery) ([]MileagePointRow, error) {
query = normalizeMileagePointQuery(query)
sqlText, args := buildMileagePointSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]MileagePointRow, 0)
for rows.Next() {
var row MileagePointRow
var ts scanDateTime
var receivedAt scanDateTime
var speed sql.NullFloat64
var longitude sql.NullFloat64
var latitude sql.NullFloat64
if err := rows.Scan(
&ts,
&row.EventID,
&row.FrameID,
&receivedAt,
&row.TotalMileageKM,
&speed,
&longitude,
&latitude,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.ReceivedAt = receivedAt.String
row.SpeedKMH = nullableFloat(speed)
row.Longitude = nullableFloat(longitude)
row.Latitude = nullableFloat(latitude)
out = append(out, row)
}
return out, rows.Err()
}
func (r *MileagePointRepository) Count(ctx context.Context, query MileagePointQuery) (int64, error) {
query = normalizeMileagePointQuery(query)
sqlText, args := buildMileagePointCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func countRows(ctx context.Context, db Queryer, sqlText string, args ...any) (int64, error) {
rows, err := db.QueryContext(ctx, sqlText, args...)
if err != nil {
return 0, err
}
defer rows.Close()
var total int64
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, rows.Err()
}
func (r *RawFrameRepository) tableName() string {
if r.database == "" {
return "raw_frames"
}
return r.database + ".raw_frames"
}
func (r *LocationRepository) tableName() string {
if r.database == "" {
return "vehicle_locations"
}
return r.database + ".vehicle_locations"
}
func (r *MileagePointRepository) tableName() string {
if r.database == "" {
return "vehicle_mileage_points"
}
return r.database + ".vehicle_mileage_points"
}
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.MessageID = strings.TrimSpace(query.MessageID)
query.OrderBy = normalizeRawFrameOrderBy(query.OrderBy)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func normalizeLocationQuery(query LocationQuery) LocationQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func normalizeMileagePointQuery(query MileagePointQuery) MileagePointQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
where := rawFrameWhere(query)
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY " + rawFrameOrderColumn(query.OrderBy) + " DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := rawFrameWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
where := locationWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildLocationCountSQL(table string, query LocationQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := locationWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func buildMileagePointSQL(table string, query MileagePointQuery) (string, []any) {
where := mileagePointWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildMileagePointCountSQL(table string, query MileagePointQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := mileagePointWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func rawFrameWhere(query RawFrameQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.MessageID != "" {
if parsed, ok := parseMessageID(query.MessageID); ok {
add("message_id = " + strconv.FormatInt(parsed, 10))
}
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
func normalizeRawFrameOrderBy(value string) string {
value = strings.ToLower(strings.TrimSpace(value))
switch value {
case "receivedat", "received_at":
return "receivedAt"
default:
return "ts"
}
}
func rawFrameOrderColumn(value string) string {
if normalizeRawFrameOrderBy(value) == "receivedAt" {
return "received_at"
}
return "ts"
}
func locationWhere(query LocationQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
func mileagePointWhere(query MileagePointQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
type RawFrameHandler struct {
repository *RawFrameRepository
}
type LocationHandler struct {
repository *LocationRepository
}
type MileagePointHandler struct {
repository *MileagePointRepository
}
func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler {
if repository == nil {
panic("raw frame repository must not be nil")
}
return &RawFrameHandler{repository: repository}
}
func NewLocationHandler(repository *LocationRepository) *LocationHandler {
if repository == nil {
panic("location repository must not be nil")
}
return &LocationHandler{repository: repository}
}
func NewMileagePointHandler(repository *MileagePointRepository) *MileagePointHandler {
if repository == nil {
panic("mileage point repository must not be nil")
}
return &MileagePointHandler{repository: repository}
}
func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/raw-frames" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseRawFrameQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
var total int64
if query.IncludeTotal {
total, err = h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
if !query.IncludeTotal {
total = int64(len(rows))
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/locations" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseLocationQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func (h *MileagePointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/mileage-points" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseMileagePointQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return RawFrameQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return RawFrameQuery{}, err
}
query := RawFrameQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
MessageID: values.Get("messageId"),
OrderBy: values.Get("orderBy"),
IncludeTotal: values.Get("includeTotal") != "false",
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
if strings.TrimSpace(query.MessageID) != "" {
if _, ok := parseMessageID(query.MessageID); !ok {
return RawFrameQuery{}, errors.New("messageId must be decimal or hex like 0x0200")
}
}
return normalizeRawFrameQuery(query), nil
}
func parseMileagePointQuery(r *http.Request) (MileagePointQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return MileagePointQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return MileagePointQuery{}, err
}
query := MileagePointQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return MileagePointQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
return normalizeMileagePointQuery(query), nil
}
func parseLocationQuery(r *http.Request) (LocationQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return LocationQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return LocationQuery{}, err
}
query := LocationQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
return normalizeLocationQuery(query), nil
}
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return fallback, nil
}
value, err := strconv.Atoi(raw)
if err != nil || value < min || value > max {
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
}
return value, nil
}
func parseMessageID(value string) (int64, bool) {
value = strings.TrimSpace(strings.ToLower(value))
if value == "" {
return 0, false
}
if strings.HasPrefix(value, "0x") {
parsed, err := strconv.ParseInt(strings.TrimPrefix(value, "0x"), 16, 32)
return parsed, err == nil
}
parsed, err := strconv.ParseInt(value, 10, 32)
return parsed, err == nil
}
func validDateTime(value string) bool {
value = normalizeDateTimeLiteral(value)
if value == "" {
return true
}
for _, layout := range []string{"2006-01-02", "2006-01-02 15:04:05", time.RFC3339} {
if _, err := time.Parse(layout, value); err == nil {
return true
}
}
return false
}
func normalizeDateTimeLiteral(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} {
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
}
}
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
}
return value
}
func safeIdentifier(value string) bool {
if value == "" {
return false
}
for _, r := range value {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
continue
}
return false
}
return true
}
type scanDateTime struct {
String string
}
func (s *scanDateTime) Scan(value any) error {
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
return nil
}
func formatSQLTime(value any, layout string) string {
switch typed := value.(type) {
case nil:
return ""
case time.Time:
return typed.Format(layout)
case []byte:
return strings.TrimSpace(string(typed))
case string:
return strings.TrimSpace(typed)
default:
return strings.TrimSpace(fmt.Sprint(typed))
}
}
func nullableFloat(value sql.NullFloat64) *float64 {
if !value.Valid {
return nil
}
return &value.Float64
}
func nullableInt(value sql.NullInt64) *int64 {
if !value.Valid {
return nil
}
return &value.Int64
}
func writeHistoryError(w http.ResponseWriter, status int, message string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
}

View File

@@ -0,0 +1,423 @@
package history
import (
"context"
"database/sql"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
)
func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
"go_frame", "event-1", 0x0200,
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
time.Date(2026, 7, 1, 23, 25, 37, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
64, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "222.66.200.68:29646", "JT808", "013079963379", "LKLG7C4E3NA774736", "013079963379", "9963379",
))
repository := NewRawFrameRepository(db, "lingniu_vehicle_ts")
rows, err := repository.Query(context.Background(), RawFrameQuery{
Protocol: "JT808",
VIN: "LKLG7C4E3NA774736",
DateFrom: "2026-07-01 00:00:00",
DateTo: "2026-07-01 23:59:59",
Limit: 20,
})
if err != nil {
t.Fatalf("Query() error = %v", err)
}
if len(rows) != 1 {
t.Fatalf("row count = %d", len(rows))
}
if rows[0].MessageID != 512 || rows[0].MessageIDHex != "0x0200" || rows[0].ParsedJSON == "" {
t.Fatalf("unexpected row: %#v", rows[0])
}
if rows[0].TS != "2026-07-01 23:25:36" {
t.Fatalf("timestamp = %q", rows[0].TS)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(38))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-01 22:28:25", "go_frame", "event-2", 2, "2026-07-01 22:28:25", "2026-07-01 22:28:25",
128, "2323", "", `{"command":"REALTIME"}`, `{"total_mileage_km":53490.9}`,
"OK", "", "8.134.95.166:53702", "GB32960", "LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"message_id_hex":"0x0002"`, `"total":38`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(11))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "go_frame", "event-3", 0x0200, "2026-07-02 00:18:22", "2026-07-02 00:22:43",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"total_mileage_km":8792.8}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"phone":"013307811350"`, `"total":11`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=NOT_EXISTS&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
t.Fatalf("expected empty items array, got: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestLocationHandlerReturnsLocationsByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(17))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
"JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
8792.8, 8.0, 121.07764, 30.585928,
"JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewMileagePointHandler(NewMileagePointRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/mileage-points?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"total_mileage_km":8792.8`, `"speed_kmh":8`, `"total":19`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
}
func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) {
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?dateFrom=2026-07-01T00:00:00&dateTo=2026-07-02T00:00:00", nil)
query, err := parseRawFrameQuery(request)
if err != nil {
t.Fatalf("parseRawFrameQuery() error = %v", err)
}
if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" {
t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo)
}
}
func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) {
for raw, want := range map[string]string{
"2026-07-01T00:00:00": "2026-06-30 16:00:00",
"2026-07-01 00:00:00": "2026-06-30 16:00:00",
"2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00",
"2026-06-30T16:00:00Z": "2026-06-30 16:00:00",
} {
if got := normalizeDateTimeLiteral(raw); got != want {
t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want)
}
}
}
func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
for raw, want := range map[string]int64{
"512": 512,
"0x0200": 512,
"0X0100": 256,
} {
got, ok := parseMessageID(raw)
if !ok || got != want {
t.Fatalf("parseMessageID(%q) = %d,%v want %d,true", raw, got, ok, want)
}
}
}
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
DateFrom: "2026-07-02 00:00:00",
DateTo: "2026-07-02 23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"ts >= '2026-07-01 16:00:00'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildLocationSQL("lingniu_vehicle_ts.vehicle_locations", LocationQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
DateFrom: "2026-07-02 00:00:00",
DateTo: "2026-07-02 23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"ts >= '2026-07-01 16:00:00'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
VIN: "VIN'1",
MessageID: "0x0200",
DateFrom: "2026-07-01T00:00:00",
DateTo: "2026-07-01T23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"vin = 'VIN''1'",
"message_id = 512",
"ts >= '2026-06-30 16:00:00'",
"ts <= '2026-07-01 15:59:59'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildRawFrameSQLCanOrderByReceivedAt(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
Protocol: "JT808",
OrderBy: "receivedAt",
Limit: 1,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
if !strings.Contains(sqlText, "ORDER BY received_at DESC LIMIT 1 OFFSET 0") {
t.Fatalf("sql should order by received_at: %s", sqlText)
}
}

View File

@@ -133,7 +133,7 @@ func rawValues(env envelope.FrameEnvelope) []any {
messageIDInt(env.MessageID),
eventTime,
received,
rawSize(env.RawHex),
rawSizeBytes(env),
env.RawHex,
env.RawText,
jsonString(env.Parsed),
@@ -200,12 +200,15 @@ func messageIDInt(value string) int64 {
return parsed
}
func rawSize(rawHex string) int {
rawHex = strings.TrimSpace(rawHex)
func rawSizeBytes(env envelope.FrameEnvelope) int {
rawHex := strings.TrimSpace(env.RawHex)
if len(rawHex)%2 != 0 {
return len(rawHex) / 2
}
return len(rawHex) / 2
if rawHex != "" {
return len(rawHex) / 2
}
return len([]byte(env.RawText))
}
func jsonString(value any) string {

View File

@@ -75,6 +75,18 @@ func TestWriterSkipsSparseDerivedRows(t *testing.T) {
}
}
func TestRawSizeBytesUsesRawTextWhenHexIsEmpty(t *testing.T) {
env := envelope.FrameEnvelope{RawText: `{"code":"0F80","data":{"speed":12}}`}
if got, want := rawSizeBytes(env), len([]byte(env.RawText)); got != want {
t.Fatalf("raw text size = %d, want %d", got, want)
}
env.RawHex = "7E0200"
if got, want := rawSizeBytes(env), 3; got != want {
t.Fatalf("raw hex size = %d, want %d", got, want)
}
}
func sampleEnvelope() envelope.FrameEnvelope {
return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,

View File

@@ -38,7 +38,7 @@ func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error)
return frames, append([]byte(nil), remaining...), nil
}
bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
frameLen := headerLen + bodyLen + 1
frameLen := frameLength(remaining, bodyLen)
if len(remaining) < frameLen {
return frames, append([]byte(nil), remaining...), nil
}
@@ -59,7 +59,8 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
}
bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
if len(raw) != headerLen+bodyLen+1 {
actualBodyLen := len(raw) - headerLen - 1
if len(raw) != headerLen+bodyLen+1 && !isExtendedPlatformLogin(raw[2], bodyLen, actualBodyLen) {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
}
if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
@@ -69,15 +70,16 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
command := raw[2]
responseFlag := raw[3]
vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
body := raw[headerLen : headerLen+bodyLen]
body := raw[headerLen : headerLen+actualBodyLen]
parsed := map[string]any{
"header": map[string]any{
"version": version,
"command": fmt.Sprintf("0x%02X", command),
"response_flag": fmt.Sprintf("0x%02X", responseFlag),
"vin": vin,
"encrypt": raw[21],
"body_length": bodyLen,
"version": version,
"command": fmt.Sprintf("0x%02X", command),
"response_flag": fmt.Sprintf("0x%02X", responseFlag),
"vin": vin,
"encrypt": raw[21],
"body_length": bodyLen,
"actual_body_length": actualBodyLen,
},
}
fields := map[string]any{}
@@ -91,6 +93,16 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
parsed["device_time"] = eventTime.Format(time.RFC3339)
}
parsed["data_units"] = units
} else if command == 0x05 {
login := parsePlatformLogin(body)
parsed["platform_login"] = login
if deviceTime, ok := login["login_time"].(string); ok && deviceTime != "" {
fields["device_time"] = deviceTime
parsed["device_time"] = deviceTime
}
if account, ok := login["username"].(string); ok && account != "" {
fields["platform_account"] = account
}
}
env := envelope.FrameEnvelope{
@@ -113,7 +125,7 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
if len(body) < 6 {
return time.Time{}, nil
}
eventTime := parseBCDTime(body[:6])
eventTime := parseGBTime(body[:6])
cursor := 6
var units []map[string]any
for cursor < len(body) {
@@ -303,6 +315,54 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
return eventTime, units
}
func frameLength(frameStart []byte, bodyLen int) int {
standardLen := headerLen + bodyLen + 1
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
return standardLen
}
// Some field platform-login frames declare the standard 41-byte body but
// carry a 32-byte password, making the actual body 53 bytes. Only consume
// the extended form when its own BCC verifies so the splitter does not eat
// bytes from the next frame.
extendedLen := headerLen + 53 + 1
if len(frameStart) < extendedLen {
return standardLen
}
extended := frameStart[:extendedLen]
if bcc(extended[2:len(extended)-1]) == extended[len(extended)-1] {
return extendedLen
}
return standardLen
}
func isExtendedPlatformLogin(command byte, declaredBodyLen int, actualBodyLen int) bool {
return command == 0x05 && declaredBodyLen == 41 && actualBodyLen == 53
}
func parsePlatformLogin(body []byte) map[string]any {
login := map[string]any{
"raw_body_length": len(body),
}
if len(body) < 9 {
login["error"] = "truncated"
return login
}
loginTime := parseGBTime(body[:6])
if !loginTime.IsZero() {
login["login_time"] = loginTime.Format(time.RFC3339)
}
login["serial_no"] = binary.BigEndian.Uint16(body[6:8])
if len(body) >= 21 {
login["username"] = trimASCII(body[8:20])
}
if len(body) > 21 {
passwordEnd := len(body) - 1
login["password"] = trimASCII(body[20:passwordEnd])
login["encrypt_rule"] = body[passwordEnd]
}
return login
}
func parseVehicleData(data []byte) map[string]any {
return map[string]any{
"vehicle_status": int(data[0]),
@@ -751,24 +811,24 @@ func version(a byte, b byte) (string, bool) {
}
}
func parseBCDTime(data []byte) time.Time {
func parseGBTime(data []byte) time.Time {
if len(data) != 6 {
return time.Time{}
}
year := 2000 + bcdByte(data[0])
month := time.Month(bcdByte(data[1]))
day := bcdByte(data[2])
hour := bcdByte(data[3])
minute := bcdByte(data[4])
second := bcdByte(data[5])
year := 2000 + int(data[0])
month := time.Month(data[1])
day := int(data[2])
hour := int(data[3])
minute := int(data[4])
second := int(data[5])
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
return time.Time{}
}
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
}
func bcdByte(value byte) int {
return int(value>>4)*10 + int(value&0x0f)
func trimASCII(data []byte) string {
return strings.TrimRight(string(data), "\x00 ")
}
func bcc(data []byte) byte {

View File

@@ -12,7 +12,7 @@ import (
const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57})
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39})
second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
stream := append([]byte{0x00, 0x01}, first...)
stream = append(stream, second[:len(second)-3]...)
@@ -49,8 +49,81 @@ func TestParseFrameRejectsBadBCC(t *testing.T) {
}
}
func TestExtractFramesSupportsExtendedPlatformLogin(t *testing.T) {
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x1f, 0x00, 0x01}
body = append(body, fixedASCII("Hyundai", 12)...)
body = append(body, fixedASCII("f2e3445d7cda409fb4f278f6fb890734", 32)...)
body = append(body, 0x01)
frame := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
frames, remainder, err := ExtractFrames(append(frame, []byte{0x23, 0x23, 0x05}...))
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 3 {
t.Fatalf("unexpected split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
if len(frames[0]) != headerLen+53+1 {
t.Fatalf("extended frame len = %d", len(frames[0]))
}
env, err := ParseFrame(frames[0], 1782745114999, "8.134.95.166:39376")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x05" || env.ParseStatus != envelope.ParseOK {
t.Fatalf("unexpected env: %#v", env)
}
header := env.Parsed["header"].(map[string]any)
if header["body_length"] != 41 || header["actual_body_length"] != 53 {
t.Fatalf("unexpected length header: %#v", header)
}
login := env.Parsed["platform_login"].(map[string]any)
if login["username"] != "Hyundai" || login["password"] != "f2e3445d7cda409fb4f278f6fb890734" {
t.Fatalf("unexpected platform login: %#v", login)
}
if login["login_time"] != "2026-07-02T00:38:31+08:00" {
t.Fatalf("unexpected login time: %#v", login)
}
if env.Fields["platform_account"] != "Hyundai" {
t.Fatalf("platform account not exposed: %#v", env.Fields)
}
}
func TestAutoResponseEchoesRawVINAndOriginalTime(t *testing.T) {
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x0f, 0x00, 0x01}
body = append(body, fixedASCII("Hyundai", 12)...)
body = append(body, fixedASCII("ack-test-password", 32)...)
body = append(body, 0x01)
request := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
env, err := ParseFrame(request, 1782914969584, "8.134.95.166:39376")
if err != nil {
t.Fatal(err)
}
response, ok, err := AutoResponse(request, env)
if err != nil {
t.Fatalf("AutoResponse() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
if response[2] != 0x05 || response[3] != 0x01 {
t.Fatalf("unexpected response header: %x", response[:4])
}
if hex.EncodeToString(response[4:21]) != "0000000000000000000000000000000000" {
t.Fatalf("raw vin not echoed: %x", response[4:21])
}
if hex.EncodeToString(response[24:30]) != "1a070200260f" {
t.Fatalf("timestamp body not preserved: %x", response[24:30])
}
if got, want := bcc(response[2:len(response)-1]), response[len(response)-1]; got != want {
t.Fatalf("response bcc got=0x%02x want=0x%02x", got, want)
}
}
func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
body := []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57}
body := []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39}
body = append(body, 0x01)
body = append(body,
0x01, // vehicle status
@@ -163,13 +236,23 @@ func assertIntField(t *testing.T, env envelope.FrameEnvelope, key string, want i
}
func buildFrame(command byte, response byte, vin string, body []byte) []byte {
return buildFrameWithDeclaredLength(command, response, vin, body, len(body))
}
func buildFrameWithDeclaredLength(command byte, response byte, vin string, body []byte, declaredBodyLen int) []byte {
frame := []byte{'#', '#', command, response}
vinBytes := []byte(vin)
if len(vinBytes) < 17 {
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
}
frame = append(frame, vinBytes[:17]...)
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
frame = append(frame, 0x01, byte(declaredBodyLen>>8), byte(declaredBodyLen))
frame = append(frame, body...)
return append(frame, bcc(frame[2:]))
}
func fixedASCII(value string, size int) []byte {
out := make([]byte, size)
copy(out, []byte(value))
return out
}

View File

@@ -0,0 +1,80 @@
package gb32960
import (
"encoding/binary"
"errors"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrResponseFrameTooShort = errors.New("gb32960 response frame too short")
const (
responseSuccess = byte(0x01)
responseCommand = byte(0xfe)
encryptNone = byte(0x01)
)
// AutoResponse builds the protocol ACK for accepted upstream GB/T 32960 frames.
// The ACK is written only after the caller has durably published the frame.
func AutoResponse(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if env.ParseStatus == envelope.ParseBadFrame {
return nil, false, nil
}
if len(raw) < headerLen+1 {
return nil, false, ErrResponseFrameTooShort
}
if raw[3] != responseCommand {
return nil, false, nil
}
command := raw[2]
if !shouldRespond(command) {
return nil, false, nil
}
body := []byte(nil)
if timestamp := responseTime(raw, command); !timestamp.IsZero() {
body = encodeGBTime(timestamp)
}
return buildResponse(raw[0], command, responseSuccess, raw[4:21], body), true, nil
}
func shouldRespond(command byte) bool {
switch command {
case 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08:
return true
default:
return false
}
}
func responseTime(raw []byte, command byte) time.Time {
bodyLen := len(raw) - headerLen - 1
if bodyLen >= 6 {
return parseGBTime(raw[headerLen : headerLen+6])
}
return time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600))
}
func encodeGBTime(t time.Time) []byte {
local := t.In(time.FixedZone("Asia/Shanghai", 8*3600))
return []byte{
byte(local.Year() - 2000),
byte(local.Month()),
byte(local.Day()),
byte(local.Hour()),
byte(local.Minute()),
byte(local.Second()),
}
}
func buildResponse(start byte, command byte, responseFlag byte, vin17 []byte, body []byte) []byte {
out := make([]byte, 0, headerLen+len(body)+1)
out = append(out, start, start, command, responseFlag)
out = append(out, vin17[:17]...)
out = append(out, encryptNone, 0, 0)
binary.BigEndian.PutUint16(out[22:24], uint16(len(body)))
out = append(out, body...)
return append(out, bcc(out[2:]))
}

View File

@@ -10,8 +10,8 @@ import (
"time"
"unicode/utf8"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"golang.org/x/text/encoding/simplifiedchinese"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var (
@@ -97,16 +97,19 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
phoneEnd = 15
serialStart = 15
}
if props&0x2000 != 0 {
subpackage := props&0x2000 != 0
if subpackage {
headerLen += 4
}
if len(raw) < headerLen+int(bodySize)+1 {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
}
if subpackage {
packageInfo = map[string]any{
"total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]),
"index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]),
}
}
if len(raw) < headerLen+int(bodySize)+1 {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
}
phoneBCD := raw[phoneStart:phoneEnd]
phone := bcdString(phoneBCD)
@@ -126,7 +129,7 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
"phone": phone,
"phone_trim_zero": phoneTrimZero,
"sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
"subpackage": props&0x2000 != 0,
"subpackage": subpackage,
"package_info": packageInfo,
"encryption_flags": (props >> 10) & 0x07,
},
@@ -155,7 +158,7 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
}
} else if messageID == 0x0100 {
registration, err := parseRegistration(body)
registration, err := parseRegistration(body, versioned)
if err != nil {
return envelope.FrameEnvelope{}, err
}
@@ -167,7 +170,7 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
fields["plate"] = plate
}
} else if messageID == 0x0102 {
authentication := parseAuthentication(body)
authentication := parseAuthentication(body, versioned)
parsed["authentication"] = authentication
if token, ok := authentication["token"].(string); ok {
fields["auth_token"] = token
@@ -195,37 +198,107 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
return env, nil
}
func parseRegistration(body []byte) (map[string]any, error) {
if len(body) < 37 {
func parseRegistration(body []byte, versioned bool) (map[string]any, error) {
schemaVersion := "2011_2013"
manufacturerSize := 5
deviceTypeSize := 20
deviceIDSize := 7
if versioned {
schemaVersion = "2019"
manufacturerSize = 11
deviceTypeSize = 30
deviceIDSize = 30
}
minLen := 2 + 2 + manufacturerSize + deviceTypeSize + deviceIDSize + 1
if len(body) < minLen {
return nil, fmt.Errorf("%w: registration body len=%d", ErrFrameTooShort, len(body))
}
manufacturerStart := 4
deviceTypeStart := manufacturerStart + manufacturerSize
deviceIDStart := deviceTypeStart + deviceTypeSize
plateColorStart := deviceIDStart + deviceIDSize
registration := map[string]any{
"province": binary.BigEndian.Uint16(body[0:2]),
"city": binary.BigEndian.Uint16(body[2:4]),
"manufacturer": fixedText(body[4:9]),
"device_type": fixedText(body[9:29]),
"device_id": fixedText(body[29:36]),
"plate_color": body[36],
"plate": freeText(body[37:]),
"schema_version": schemaVersion,
"province": binary.BigEndian.Uint16(body[0:2]),
"city": binary.BigEndian.Uint16(body[2:4]),
"manufacturer": fixedText(body[manufacturerStart:deviceTypeStart]),
"device_type": fixedText(body[deviceTypeStart:deviceIDStart]),
"device_id": fixedText(body[deviceIDStart:plateColorStart]),
"plate_color": body[plateColorStart],
"plate": freeText(body[plateColorStart+1:]),
}
return registration, nil
}
func parseAuthentication(body []byte) map[string]any {
token := freeText(body)
out := map[string]any{"token": token}
return out
func parseAuthentication(body []byte, versioned bool) map[string]any {
if versioned {
if out, ok := parseVersionedAuthentication(body); ok {
return out
}
if out, ok := parseDelimitedVersionedAuthentication(body); ok {
return out
}
}
return map[string]any{"token": freeText(body)}
}
func parseVersionedAuthentication(body []byte) (map[string]any, bool) {
if len(body) < 17 {
return nil, false
}
tokenLen := int(body[0])
imeiStart := 1 + tokenLen
softwareLenStart := imeiStart + 15
if tokenLen <= 0 || len(body) < softwareLenStart+1 {
return nil, false
}
softwareLen := int(body[softwareLenStart])
softwareStart := softwareLenStart + 1
if len(body) < softwareStart+softwareLen {
return nil, false
}
return map[string]any{
"schema_version": "2019",
"token": freeText(body[1:imeiStart]),
"imei": fixedText(body[imeiStart:softwareLenStart]),
"software_version": freeText(body[softwareStart : softwareStart+softwareLen]),
}, true
}
func parseDelimitedVersionedAuthentication(body []byte) (map[string]any, bool) {
parts := bytes.Split(body, []byte{0xff, 0xff})
if len(parts) < 2 {
return nil, false
}
out := map[string]any{
"schema_version": "2019_delimited",
"token": freeText(parts[0]),
"imei": freeText(parts[1]),
}
if len(parts) > 2 {
out["software_version"] = freeText(parts[2])
}
return out, true
}
func parseLocation(body []byte) (Location, error) {
if len(body) < 28 {
return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
}
status := binary.BigEndian.Uint32(body[4:8])
latitude := float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000
longitude := float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000
if status&(1<<2) != 0 {
latitude = -latitude
}
if status&(1<<3) != 0 {
longitude = -longitude
}
location := Location{
AlarmFlag: binary.BigEndian.Uint32(body[0:4]),
StatusFlag: binary.BigEndian.Uint32(body[4:8]),
Latitude: float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000,
Longitude: float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000,
StatusFlag: status,
Latitude: latitude,
Longitude: longitude,
AltitudeM: binary.BigEndian.Uint16(body[16:18]),
SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10,
DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
@@ -256,14 +329,105 @@ func parseAdditional(data []byte, location *Location) []AdditionalItem {
Length: size,
ValueHex: strings.ToUpper(hex.EncodeToString(value)),
}
// JT/T 808-2011 table 27: id 0x01 is GPS total mileage, DWORD, 0.1 km.
if id == 0x01 && size == 4 {
mileage := float64(binary.BigEndian.Uint32(value)) / 10
switch {
case id == 0x01 && size == 4:
rawMileage := binary.BigEndian.Uint32(value)
mileage := float64(rawMileage) / 10
location.TotalMileageKM = &mileage
item.Parsed = map[string]any{
"name": envelope.FieldTotalMileageKM,
"value": mileage,
"unit": "km",
"name": envelope.FieldTotalMileageKM,
"raw_value_tenth_km": rawMileage,
"value": mileage,
"unit": "km",
}
case id == 0x02 && size == 2:
rawFuel := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "fuel_l",
"raw_value_tenth_l": rawFuel,
"value": float64(rawFuel) / 10,
"unit": "L",
}
case id == 0x03 && size == 2:
rawSpeed := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "recorder_speed_kmh",
"raw_value_tenth_kmh": rawSpeed,
"value": float64(rawSpeed) / 10,
"unit": "km/h",
}
case id == 0x04 && size == 2:
item.Parsed = map[string]any{
"name": "manual_alarm_event_id",
"value": binary.BigEndian.Uint16(value),
}
case id == 0x05 && size == 30:
item.Parsed = map[string]any{
"name": "tire_pressure",
"values": parseTirePressure(value),
}
case id == 0x06 && size == 2:
item.Parsed = map[string]any{
"name": "carriage_temperature_c",
"value": int16(binary.BigEndian.Uint16(value)),
"unit": "C",
}
case id == 0x11 && (size == 1 || size == 5):
parsed := map[string]any{
"name": "overspeed_alarm",
"location_type": value[0],
}
if size == 5 {
parsed["area_id"] = binary.BigEndian.Uint32(value[1:5])
}
item.Parsed = parsed
case id == 0x12 && size == 6:
item.Parsed = map[string]any{
"name": "area_route_alarm",
"location_type": value[0],
"area_id": binary.BigEndian.Uint32(value[1:5]),
"direction": value[5],
}
case id == 0x13 && size == 7:
item.Parsed = map[string]any{
"name": "road_section_driving_time_alarm",
"road_section_id": binary.BigEndian.Uint32(value[0:4]),
"driving_time_seconds": binary.BigEndian.Uint16(value[4:6]),
"result": value[6],
"result_description": roadSectionDrivingTimeResult(value[6]),
"driving_time_result_source": "jt808_0x0200_additional_0x13",
}
case id == 0x25 && size == 4:
rawStatus := binary.BigEndian.Uint32(value)
item.Parsed = map[string]any{
"name": "extended_vehicle_signal_status",
"value": rawStatus,
"bits": parseExtendedVehicleSignal(rawStatus),
}
case id == 0x2a && size == 2:
rawStatus := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "io_status",
"value": rawStatus,
"bits": parseIOStatus(rawStatus),
}
case id == 0x2b && size == 4:
analog := binary.BigEndian.Uint32(value)
item.Parsed = map[string]any{
"name": "analog",
"ad0": analog & 0xffff,
"ad1": analog >> 16,
"value": analog,
}
case id == 0x30 && size == 1:
item.Parsed = map[string]any{
"name": "network_signal_strength",
"value": value[0],
}
case id == 0x31 && size == 1:
item.Parsed = map[string]any{
"name": "gnss_satellite_count",
"value": value[0],
}
}
out = append(out, item)
@@ -308,6 +472,55 @@ func additionalToMaps(items []AdditionalItem) []map[string]any {
return out
}
func parseTirePressure(data []byte) []map[string]any {
out := make([]map[string]any, 0, len(data))
for index, value := range data {
out = append(out, map[string]any{
"position": index,
"value": value,
"valid": value != 0xff,
})
}
return out
}
func parseExtendedVehicleSignal(value uint32) map[string]bool {
return map[string]bool{
"low_beam": value&0x0001 != 0,
"high_beam": value&0x0002 != 0,
"right_turn": value&0x0004 != 0,
"left_turn": value&0x0008 != 0,
"brake": value&0x0010 != 0,
"reverse_gear": value&0x0020 != 0,
"fog_light": value&0x0040 != 0,
"clearance_light": value&0x0080 != 0,
"horn": value&0x0100 != 0,
"air_conditioner": value&0x0200 != 0,
"neutral": value&0x0400 != 0,
"retarder": value&0x0800 != 0,
"abs": value&0x1000 != 0,
"heater": value&0x2000 != 0,
"clutch": value&0x4000 != 0,
}
}
func parseIOStatus(value uint16) map[string]bool {
return map[string]bool{
"deep_sleep": value&0x0001 != 0,
"sleep": value&0x0002 != 0,
}
}
func roadSectionDrivingTimeResult(value byte) string {
if value == 0 {
return "insufficient"
}
if value == 1 {
return "too_long"
}
return "unknown"
}
func bcdString(data []byte) string {
out := make([]byte, 0, len(data)*2)
for _, value := range data {

View File

@@ -10,6 +10,7 @@ import (
const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E"
const fullAdditionalLocationFrame = "7e020000800123456789017fff000004000000080006eeb6ad02633df701380003006320070719235901040000000b02020016030200210402002c051e3737370000000000000000000000000000000000000000000000000000001105420000004212064d0000004d4d1307000000580058582504000000632a02000a2b040000001430011e31012806020001927e"
func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
@@ -63,6 +64,13 @@ func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
t.Fatalf("unexpected first additional item: %#v", additional[0])
}
parsed, ok := additional[0]["parsed"].(map[string]any)
if !ok {
t.Fatalf("mileage parsed field missing: %#v", additional[0])
}
if parsed["raw_value_tenth_km"] != uint32(102412) || parsed["value"] != 10241.2 {
t.Fatalf("unexpected mileage parsed value: %#v", parsed)
}
}
func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
@@ -115,6 +123,89 @@ func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
}
}
func TestParseFrameParsesCommonLocationAdditionalItems(t *testing.T) {
stream, err := hex.DecodeString(fullAdditionalLocationFrame)
if err != nil {
t.Fatal(err)
}
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782914967713, "127.0.0.1:808")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Phone != "012345678901" {
t.Fatalf("phone = %q", env.Phone)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.1)
location := env.Parsed["location"].(map[string]any)
additional := location["additional"].([]map[string]any)
byID := map[string]map[string]any{}
for _, item := range additional {
byID[item["id"].(string)] = item["parsed"].(map[string]any)
}
if byID["0x01"]["raw_value_tenth_km"] != uint32(11) || byID["0x01"]["value"] != 1.1 {
t.Fatalf("mileage parsed = %#v", byID["0x01"])
}
if byID["0x05"]["name"] != "tire_pressure" {
t.Fatalf("tire pressure missing: %#v", byID["0x05"])
}
tireValues := byID["0x05"]["values"].([]map[string]any)
if tireValues[0]["value"] != byte(0x37) || tireValues[0]["valid"] != true {
t.Fatalf("tire pressure first value = %#v", tireValues[0])
}
if byID["0x11"]["location_type"] != byte(0x42) || byID["0x11"]["area_id"] != uint32(0x42) {
t.Fatalf("overspeed parsed = %#v", byID["0x11"])
}
if byID["0x12"]["location_type"] != byte(0x4d) || byID["0x12"]["area_id"] != uint32(0x4d) || byID["0x12"]["direction"] != byte(0x4d) {
t.Fatalf("area route parsed = %#v", byID["0x12"])
}
if byID["0x13"]["road_section_id"] != uint32(0x58) || byID["0x13"]["driving_time_seconds"] != uint16(0x58) {
t.Fatalf("road section parsed = %#v", byID["0x13"])
}
ioBits := byID["0x2A"]["bits"].(map[string]bool)
if ioBits["deep_sleep"] || !ioBits["sleep"] {
t.Fatalf("io bits = %#v", ioBits)
}
}
func TestParseFrameRejectsTruncatedSubpackageHeader(t *testing.T) {
payload := []byte{0x02, 0x00, 0x20, 0x00}
payload = append(payload, encodeBCD("013307795425", 6)...)
payload = binary.BigEndian.AppendUint16(payload, 1)
payload = append(payload, checksum(payload))
_, err := ParseFrame(payload, 1782914967713, "115.231.168.135:43625")
if err == nil {
t.Fatal("expected truncated subpackage header to fail")
}
}
func TestParseLocationAppliesSouthWestStatusBits(t *testing.T) {
body := make([]byte, 28)
binary.BigEndian.PutUint32(body[4:8], (1<<2)|(1<<3))
binary.BigEndian.PutUint32(body[8:12], 12345678)
binary.BigEndian.PutUint32(body[12:16], 98765432)
location, err := parseLocation(body)
if err != nil {
t.Fatalf("parseLocation() error = %v", err)
}
if location.Latitude != -12.345678 {
t.Fatalf("latitude = %v", location.Latitude)
}
if location.Longitude != -98.765432 {
t.Fatalf("longitude = %v", location.Longitude)
}
}
func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
@@ -153,6 +244,38 @@ func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
}
}
func TestParseFrameParsesJT8082019RegistrationIdentity(t *testing.T) {
body := make([]byte, 0, 84)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTONG00001", 11)
body = appendFixedASCII(body, "ZK6105CHEVNPG4-2019", 30)
body = appendFixedASCII(body, "DEV201900000000000000000000001", 30)
body = append(body, 2)
body = append(body, []byte("TEST2019")...)
env, err := ParseFrame(buildVersionedFrame(0x0100, "14894135060", 11, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Phone != "14894135060" {
t.Fatalf("phone = %q", env.Phone)
}
if env.DeviceID != "DEV201900000000000000000000001" {
t.Fatalf("device id = %q", env.DeviceID)
}
if env.Plate != "TEST2019" {
t.Fatalf("plate = %q", env.Plate)
}
registration, ok := env.Parsed["registration"].(map[string]any)
if !ok {
t.Fatalf("registration missing: %#v", env.Parsed)
}
if registration["schema_version"] != "2019" || registration["manufacturer"] != "YUTONG00001" {
t.Fatalf("registration fields = %#v", registration)
}
}
func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
@@ -172,6 +295,26 @@ func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
}
}
func TestParseFrameParsesJT8082019Authentication(t *testing.T) {
body := []byte{5}
body = append(body, []byte("TOKEN")...)
body = appendFixedASCII(body, "123456789012345", 15)
body = append(body, 4)
body = append(body, []byte("v1.2")...)
env, err := ParseFrame(buildVersionedFrame(0x0102, "14894135060", 12, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
auth, ok := env.Parsed["authentication"].(map[string]any)
if !ok {
t.Fatalf("authentication missing: %#v", env.Parsed)
}
if auth["token"] != "TOKEN" || auth["imei"] != "123456789012345" || auth["software_version"] != "v1.2" {
t.Fatalf("authentication = %#v", auth)
}
}
func TestParseFrameParsesAuthenticationToken(t *testing.T) {
env, err := ParseFrame(buildFrame(0x0102, "064646848757", 8, []byte("g7gps")), 1782918600000, "115.231.168.135:43625")
if err != nil {
@@ -189,6 +332,86 @@ func TestParseFrameParsesAuthenticationToken(t *testing.T) {
}
}
func TestAutoResponderBuildsRegisterAck(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTNG", 5)
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
body = appendFixedASCII(body, "DEV0001", 7)
body = append(body, 2)
body = append(body, []byte("TEST123")...)
request := buildFrame(0x0100, "013079963379", 9, body)
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatal(err)
}
response, ok, err := NewAutoResponder("g7gps").Respond(request, env)
if err != nil {
t.Fatalf("Respond() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
frames, remainder, err := ExtractFrames(response)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
payload := frames[0]
if binary.BigEndian.Uint16(payload[0:2]) != 0x8100 {
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
}
if hex.EncodeToString(payload[4:10]) != "013079963379" {
t.Fatalf("phone bcd = %x", payload[4:10])
}
if binary.BigEndian.Uint16(payload[12:14]) != 9 || payload[14] != 0 || string(payload[15:20]) != "g7gps" {
t.Fatalf("unexpected register ack body: %x", payload[12:len(payload)-1])
}
}
func TestAutoResponderBuildsVersionedGeneralAck(t *testing.T) {
request := buildVersionedFrame(0x0200, "14894135060", 12, make([]byte, 28))
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatal(err)
}
response, ok, err := NewAutoResponder("").Respond(request, env)
if err != nil {
t.Fatalf("Respond() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
frames, remainder, err := ExtractFrames(response)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
payload := frames[0]
if binary.BigEndian.Uint16(payload[0:2]) != 0x8001 {
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
}
if binary.BigEndian.Uint16(payload[2:4])&0x4000 == 0 || payload[4] != 1 {
t.Fatalf("versioned header not preserved: %x", payload[:5])
}
if hex.EncodeToString(payload[5:15]) != "00000000014894135060" {
t.Fatalf("phone bcd = %x", payload[5:15])
}
bodyStart := 17
if binary.BigEndian.Uint16(payload[bodyStart:bodyStart+2]) != 12 ||
binary.BigEndian.Uint16(payload[bodyStart+2:bodyStart+4]) != 0x0200 ||
payload[bodyStart+4] != 0 {
t.Fatalf("unexpected general ack body: %x", payload[bodyStart:len(payload)-1])
}
}
func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
@@ -217,6 +440,18 @@ func buildFrame(messageID uint16, phone string, sequence uint16, body []byte) []
return payload
}
func buildVersionedFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
payload := make([]byte, 0, 17+len(body)+1)
payload = binary.BigEndian.AppendUint16(payload, messageID)
payload = binary.BigEndian.AppendUint16(payload, 0x4000|uint16(len(body)))
payload = append(payload, 1)
payload = append(payload, encodeBCD(phone, 10)...)
payload = binary.BigEndian.AppendUint16(payload, sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return payload
}
func appendFixedASCII(out []byte, value string, size int) []byte {
start := len(out)
out = append(out, make([]byte, size)...)

View File

@@ -0,0 +1,130 @@
package jt808
import (
"encoding/binary"
"errors"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrResponseHeaderTooShort = errors.New("jt808 response header too short")
const (
msgTerminalGeneralResponse = uint16(0x0001)
msgTerminalRegister = uint16(0x0100)
msgPlatformGeneralResponse = uint16(0x8001)
msgPlatformRegisterAck = uint16(0x8100)
)
type AutoResponder struct {
authCode string
}
func NewAutoResponder(authCode string) AutoResponder {
if authCode == "" {
authCode = "g7gps"
}
return AutoResponder{authCode: authCode}
}
func (r AutoResponder) Respond(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if env.ParseStatus == envelope.ParseBadFrame {
return nil, false, nil
}
header, err := parseResponseHeader(raw)
if err != nil {
return nil, false, err
}
if header.messageID == msgTerminalGeneralResponse {
return nil, false, nil
}
if header.messageID == msgTerminalRegister {
body := make([]byte, 3, 3+len(r.authCode))
binary.BigEndian.PutUint16(body[0:2], header.sequence)
body[2] = 0
body = append(body, []byte(r.authCode)...)
return encodeResponse(msgPlatformRegisterAck, header, body), true, nil
}
body := make([]byte, 5)
binary.BigEndian.PutUint16(body[0:2], header.sequence)
binary.BigEndian.PutUint16(body[2:4], header.messageID)
body[4] = 0
return encodeResponse(msgPlatformGeneralResponse, header, body), true, nil
}
type responseHeader struct {
messageID uint16
versioned bool
protocolVersion byte
phoneBCD []byte
sequence uint16
}
func parseResponseHeader(raw []byte) (responseHeader, error) {
if len(raw) < 12 {
return responseHeader{}, ErrResponseHeaderTooShort
}
props := binary.BigEndian.Uint16(raw[2:4])
versioned := props&0x4000 != 0
phoneStart := 4
phoneLen := 6
protocolVersion := byte(0)
if versioned {
if len(raw) < 17 {
return responseHeader{}, ErrResponseHeaderTooShort
}
protocolVersion = raw[4]
phoneStart = 5
phoneLen = 10
}
sequenceStart := phoneStart + phoneLen
if len(raw) < sequenceStart+2 {
return responseHeader{}, ErrResponseHeaderTooShort
}
phone := append([]byte(nil), raw[phoneStart:sequenceStart]...)
return responseHeader{
messageID: binary.BigEndian.Uint16(raw[0:2]),
versioned: versioned,
protocolVersion: protocolVersion,
phoneBCD: phone,
sequence: binary.BigEndian.Uint16(raw[sequenceStart : sequenceStart+2]),
}, nil
}
func encodeResponse(messageID uint16, header responseHeader, body []byte) []byte {
props := uint16(len(body)) & 0x03ff
if header.versioned {
props |= 0x4000
}
payload := make([]byte, 0, 4+1+len(header.phoneBCD)+2+len(body)+1)
payload = appendU16(payload, messageID)
payload = appendU16(payload, props)
if header.versioned {
payload = append(payload, header.protocolVersion)
}
payload = append(payload, header.phoneBCD...)
payload = appendU16(payload, header.sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return frame(payload)
}
func appendU16(out []byte, value uint16) []byte {
return append(out, byte(value>>8), byte(value))
}
func frame(payload []byte) []byte {
out := []byte{0x7e}
for _, value := range payload {
switch value {
case 0x7e:
out = append(out, 0x7d, 0x02)
case 0x7d:
out = append(out, 0x7d, 0x01)
default:
out = append(out, value)
}
}
out = append(out, 0x7e)
return out
}

View File

@@ -69,7 +69,7 @@ func fieldsFromData(data map[string]any) map[string]any {
fields[envelope.FieldSpeedKMH] = speed
}
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
fields[envelope.FieldTotalMileageKM] = mileage
fields[envelope.FieldTotalMileageKM] = normalizeTotalMileageKM(mileage)
}
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
fields[envelope.FieldSOCPercent] = soc
@@ -166,6 +166,14 @@ func parseTimeMS(raw string, fallback int64) int64 {
if raw == "" {
return fallback
}
for _, layout := range []string{
"2006-01-02 15:04:05.000",
"2006-01-02 15:04:05",
} {
if parsed, err := time.ParseInLocation(layout, raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
return parsed.UnixMilli()
}
}
if len(raw) == 14 {
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
return parsed.UnixMilli()
@@ -187,3 +195,7 @@ func looksLikeVIN(value string) bool {
value = strings.TrimSpace(value)
return len(value) == 17
}
func normalizeTotalMileageKM(value float64) float64 {
return value / 1000
}

View File

@@ -36,7 +36,7 @@ func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
t.Fatalf("plate = %q", env.Plate)
}
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123456.7)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123.4567)
assertFloatField(t, env, envelope.FieldSOCPercent, 70)
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
@@ -64,6 +64,35 @@ func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
}
func TestParseMessageNormalizesProductionMileageMeters(t *testing.T) {
payload := []byte(`{
"device":"LMRKH9AC3R1004101",
"time":"2026-07-02 01:02:04.085",
"data":{"TOTAL_MILEAGE":56905000,"METER_SPEED":19}
}`)
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 56905)
if env.EventTimeMS != 1782925324085 {
t.Fatalf("event time = %d, want 1782925324085", env.EventTimeMS)
}
}
func TestParseMessageTreatsSmallYutongMileageAsMeters(t *testing.T) {
payload := []byte(`{
"device":"LMRKH9AC3R1004101",
"time":"2026-07-02 01:02:04.085",
"data":{"TOTAL_MILEAGE":1024,"METER_SPEED":19}
}`)
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.024)
}
func TestParseMessageRejectsMalformedJSON(t *testing.T) {
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
if !errors.Is(err, ErrInvalidJSON) {

View File

@@ -7,18 +7,21 @@ import (
)
type Snapshot struct {
VIN string `json:"vin"`
Protocol envelope.Protocol `json:"protocol,omitempty"`
EventID string `json:"event_id,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
UpdatedAtMS int64 `json:"updated_at_ms"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Protocol envelope.Protocol `json:"protocol,omitempty"`
Protocols []envelope.Protocol `json:"protocols,omitempty"`
EventID string `json:"event_id,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
UpdatedAtMS int64 `json:"updated_at_ms"`
}
type OnlineStatus struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Online bool `json:"online"`
LastSeenMS int64 `json:"last_seen_ms"`

View File

@@ -28,7 +28,8 @@ func NewRepository(client *redis.Client, cfg Config) *Repository {
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
vin := strings.TrimSpace(env.VIN)
if vin == "" {
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil
}
nowMS := time.Now().UnixMilli()
@@ -37,6 +38,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
eventMS = env.ReceivedAtMS
}
protocolSnapshot := Snapshot{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
EventID: env.StableEventID(),
@@ -47,16 +49,23 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
FieldTimesMS: fieldTimes(env.Fields, eventMS),
UpdatedAtMS: nowMS,
}
if err := r.setJSON(ctx, protocolKey(vin, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
if err := r.setJSON(ctx, protocolKey(vehicleKey, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
return err
}
merged, err := r.GetMerged(ctx, vin)
protocols, err := r.addProtocol(ctx, vehicleKey, env.Protocol)
if err != nil {
return err
}
merged, err := r.GetMerged(ctx, vehicleKey)
if err != nil && !errors.Is(err, redis.Nil) {
return err
}
if merged.VIN == "" {
merged = Snapshot{VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
if merged.VehicleKey == "" {
merged = Snapshot{VehicleKey: vehicleKey, VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
} else if merged.VIN == "" && vin != "" {
merged.VIN = vin
}
mergeFields(&merged, env.Fields, eventMS)
if eventMS >= merged.EventTimeMS {
@@ -65,49 +74,48 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
merged.ReceivedAtMS = env.ReceivedAtMS
merged.SourceEndpoint = env.SourceEndpoint
}
merged.Protocols = protocols
merged.UpdatedAtMS = nowMS
if err := r.setJSON(ctx, mergedKey(vin), merged, r.cfg.ttl()); err != nil {
if err := r.setJSON(ctx, mergedKey(vehicleKey), merged, r.cfg.ttl()); err != nil {
return err
}
protocols, err := r.addProtocol(ctx, vin, env.Protocol)
if err != nil {
return err
}
online := OnlineStatus{
VehicleKey: vehicleKey,
VIN: vin,
Online: true,
LastSeenMS: env.ReceivedAtMS,
Protocols: protocols,
TTLSeconds: int64(r.cfg.ttl().Seconds()),
}
if err := r.setJSON(ctx, onlineKey(vin), online, r.cfg.ttl()); err != nil {
if err := r.setJSON(ctx, onlineKey(vehicleKey), online, r.cfg.ttl()); err != nil {
return err
}
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vin}).Err()
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vehicleKey}).Err()
}
func (r *Repository) GetMerged(ctx context.Context, vin string) (Snapshot, error) {
return r.getSnapshot(ctx, mergedKey(vin))
func (r *Repository) GetMerged(ctx context.Context, vehicleKey string) (Snapshot, error) {
return r.getSnapshot(ctx, mergedKey(vehicleKey))
}
func (r *Repository) GetProtocol(ctx context.Context, vin string, protocol envelope.Protocol) (Snapshot, error) {
return r.getSnapshot(ctx, protocolKey(vin, protocol))
func (r *Repository) GetProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) (Snapshot, error) {
return r.getSnapshot(ctx, protocolKey(vehicleKey, protocol))
}
func (r *Repository) IsOnline(ctx context.Context, vin string) (OnlineStatus, error) {
func (r *Repository) IsOnline(ctx context.Context, vehicleKey string) (OnlineStatus, error) {
vehicleKey = strings.TrimSpace(vehicleKey)
var status OnlineStatus
payload, err := r.client.Get(ctx, onlineKey(vin)).Bytes()
payload, err := r.client.Get(ctx, onlineKey(vehicleKey)).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return OnlineStatus{VIN: vin, Online: false}, nil
return OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}, nil
}
return status, err
}
if err := json.Unmarshal(payload, &status); err != nil {
return status, err
}
ttl := r.client.TTL(ctx, onlineKey(vin)).Val()
ttl := r.client.TTL(ctx, onlineKey(vehicleKey)).Val()
status.Online = ttl > 0
status.TTLSeconds = int64(ttl.Seconds())
return status, nil
@@ -130,8 +138,8 @@ func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl tim
return r.client.Set(ctx, key, payload, ttl).Err()
}
func (r *Repository) addProtocol(ctx context.Context, vin string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
key := protocolsKey(vin)
func (r *Repository) addProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
key := protocolsKey(vehicleKey)
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
return nil, err
}
@@ -207,18 +215,18 @@ func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
return out
}
func mergedKey(vin string) string {
return "vehicle:latest:" + strings.TrimSpace(vin)
func mergedKey(vehicleKey string) string {
return "vehicle:latest:" + strings.TrimSpace(vehicleKey)
}
func protocolKey(vin string, protocol envelope.Protocol) string {
return "vehicle:latest:" + strings.TrimSpace(vin) + ":" + string(protocol)
func protocolKey(vehicleKey string, protocol envelope.Protocol) string {
return "vehicle:latest:" + strings.TrimSpace(vehicleKey) + ":" + string(protocol)
}
func onlineKey(vin string) string {
return "vehicle:online:" + strings.TrimSpace(vin)
func onlineKey(vehicleKey string) string {
return "vehicle:online:" + strings.TrimSpace(vehicleKey)
}
func protocolsKey(vin string) string {
return "vehicle:protocols:" + strings.TrimSpace(vin)
func protocolsKey(vehicleKey string) string {
return "vehicle:protocols:" + strings.TrimSpace(vehicleKey)
}

View File

@@ -54,6 +54,9 @@ func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
if merged.Fields[envelope.FieldLatitude] != 30.5 {
t.Fatalf("new latitude missing: %#v", merged.Fields)
}
if got := protocolNames(merged.Protocols); strings.Join(got, ",") != "GB32960,JT808" {
t.Fatalf("merged protocols = %#v", got)
}
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
if err != nil {
t.Fatalf("GetProtocol() error = %v", err)
@@ -94,6 +97,44 @@ func TestRepositoryOnlineStatus(t *testing.T) {
}
}
func TestRepositoryUpdatesPhoneOnlyVehicleKey(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811170",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10003.7,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
vehicleKey := "JT808:013307811170"
merged, err := repo.GetMerged(ctx, vehicleKey)
if err != nil {
t.Fatalf("GetMerged() error = %v", err)
}
if merged.VehicleKey != vehicleKey || merged.VIN != "" {
t.Fatalf("unexpected identity: %#v", merged)
}
if merged.Fields[envelope.FieldTotalMileageKM] != 10003.7 {
t.Fatalf("merged fields = %#v", merged.Fields)
}
status, err := repo.IsOnline(ctx, vehicleKey)
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if !status.Online || status.VehicleKey != vehicleKey {
t.Fatalf("online status = %#v", status)
}
}
func TestRepositoryDoesNotOverwritePositiveMileageWithZero(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
@@ -159,6 +200,30 @@ func TestHandlerReturnsMergedSnapshot(t *testing.T) {
}
}
func TestHandlerReturnsPhoneOnlyVehicleKeySnapshot(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811170",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/JT808:013307811170", nil)
rec := httptest.NewRecorder()
NewHandler(repo).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
if !stringsContains(rec.Body.String(), `"vehicle_key":"JT808:013307811170"`) {
t.Fatalf("unexpected body: %s", rec.Body.String())
}
}
func newTestRepository(t *testing.T) (*Repository, func()) {
t.Helper()
server, err := miniredis.Run()
@@ -175,3 +240,11 @@ func newTestRepository(t *testing.T) (*Repository, func()) {
func stringsContains(value string, pattern string) bool {
return strings.Contains(value, pattern)
}
func protocolNames(protocols []envelope.Protocol) []string {
names := make([]string, 0, len(protocols))
for _, protocol := range protocols {
names = append(names, string(protocol))
}
return names
}

View File

@@ -26,6 +26,7 @@ type Writer struct {
}
type MetricSample struct {
VehicleKey string
VIN string
Protocol envelope.Protocol
StatDate string
@@ -45,8 +46,13 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
}
func (w *Writer) EnsureSchema(ctx context.Context) error {
_, err := w.exec.ExecContext(ctx, DailyMetricTableSQL)
return err
if _, err := w.exec.ExecContext(ctx, DailyMetricTableSQL); err != nil {
return err
}
if db, ok := w.exec.(metadataDB); ok {
return migrateDailyMetricTable(ctx, db)
}
return nil
}
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -56,6 +62,7 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
}
for _, sample := range samples {
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL,
sample.VehicleKey,
sample.VIN,
sample.StatDate,
string(sample.Protocol),
@@ -71,7 +78,8 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
vin := strings.TrimSpace(env.VIN)
if vin == "" {
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil, nil
}
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
@@ -94,6 +102,7 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
return []MetricSample{
{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
StatDate: statDate,
@@ -102,6 +111,7 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
TotalMileageKM: totalMileage,
},
{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
StatDate: statDate,
@@ -114,10 +124,11 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
const upsertDailyMetricSQL = `
INSERT INTO vehicle_daily_metric
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
(vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit,
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
VALUES (?, ?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
ON DUPLICATE KEY UPDATE
vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
first_total_mileage_km = CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
@@ -158,6 +169,82 @@ ON DUPLICATE KEY UPDATE
updated_at = CURRENT_TIMESTAMP
`
type metadataDB interface {
Execer
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func migrateDailyMetricTable(ctx context.Context, db metadataDB) error {
columnExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND column_name = 'vehicle_key'`)
if err != nil {
return err
}
if !columnExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD COLUMN vehicle_key VARCHAR(96) NOT NULL DEFAULT '' AFTER id`); err != nil {
return err
}
}
if _, err := db.ExecContext(ctx, `UPDATE vehicle_daily_metric SET vehicle_key = vin WHERE vehicle_key = ''`); err != nil {
return err
}
vehicleIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric_vehicle'`)
if err != nil {
return err
}
if !vehicleIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key)`); err != nil {
return err
}
}
oldIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric'`)
if err != nil {
return err
}
if oldIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric DROP INDEX uk_daily_metric`); err != nil {
return err
}
}
vinIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'idx_vin'`)
if err != nil {
return err
}
if !vinIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD KEY idx_vin (vin)`); err != nil {
return err
}
}
return nil
}
func informationSchemaExists(ctx context.Context, db metadataDB, query string) (bool, error) {
var count int
if err := db.QueryRowContext(ctx, query).Scan(&count); err != nil {
return false, err
}
return count > 0, nil
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
if env.Fields == nil {
return 0, false

View File

@@ -31,6 +31,9 @@ func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
}
if samples[0].VehicleKey != "LNBVIN00000000001" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 {
t.Fatalf("unexpected daily total sample: %#v", samples[1])
}
@@ -39,7 +42,31 @@ func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
}
}
func TestSamplesFromEnvelopeSkipsMissingVINOrMileage(t *testing.T) {
func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811254",
EventTimeMS: time.Date(2026, 7, 2, 0, 3, 0, 0, loc).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: 10985.7,
},
}, loc)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 2 {
t.Fatalf("sample count = %d", len(samples))
}
if samples[0].VehicleKey != "JT808:013307811254" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[0].VIN != "" {
t.Fatalf("vin should stay empty for unresolved JT808 identity, got %q", samples[0].VIN)
}
}
func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
@@ -48,7 +75,7 @@ func TestSamplesFromEnvelopeSkipsMissingVINOrMileage(t *testing.T) {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 0 {
t.Fatalf("expected no samples without vin, got %#v", samples)
t.Fatalf("expected no samples without vehicle key, got %#v", samples)
}
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
@@ -103,6 +130,9 @@ func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") {
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
}
if !strings.Contains(exec.calls[0].query, "vehicle_key") {
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
}
if len(exec.calls) != 3 {
t.Fatalf("exec calls = %d", len(exec.calls))
}
@@ -112,6 +142,9 @@ func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
}
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
t.Fatalf("first upsert arg should be vehicle_key, got %#v", got)
}
}
type execCall struct {

View File

@@ -17,16 +17,18 @@ type Queryer interface {
}
type MetricQuery struct {
VIN string
Protocol string
MetricKey string
DateFrom string
DateTo string
Limit int
Offset int
VehicleKey string
VIN string
Protocol string
MetricKey string
DateFrom string
DateTo string
Limit int
Offset int
}
type MetricRow struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
@@ -61,7 +63,7 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
}
defer rows.Close()
var out []MetricRow
out := make([]MetricRow, 0)
for rows.Next() {
var row MetricRow
var statDate scanDate
@@ -70,6 +72,7 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
var first sql.NullFloat64
var latest sql.NullFloat64
if err := rows.Scan(
&row.VehicleKey,
&row.VIN,
&statDate,
&row.Protocol,
@@ -99,7 +102,26 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
return out, rows.Err()
}
func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, error) {
query = normalizeMetricQuery(query)
sqlText, args := buildMetricCountSQL(query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return 0, err
}
defer rows.Close()
var total int64
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, rows.Err()
}
func normalizeMetricQuery(query MetricQuery) MetricQuery {
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.MetricKey = strings.TrimSpace(query.MetricKey)
@@ -112,6 +134,26 @@ func normalizeMetricQuery(query MetricQuery) MetricQuery {
}
func buildMetricSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
args = append(args, query.Limit, query.Offset)
return sqlText, args
}
func buildMetricCountSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT COUNT(*) FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, args
}
func buildMetricWhere(query MetricQuery) ([]string, []any) {
var where []string
var args []any
add := func(clause string, value any) {
@@ -121,6 +163,9 @@ func buildMetricSQL(query MetricQuery) (string, []any) {
if query.VIN != "" {
add("vin = ?", query.VIN)
}
if query.VehicleKey != "" {
add("vehicle_key = ?", query.VehicleKey)
}
if query.Protocol != "" {
add("protocol = ?", query.Protocol)
}
@@ -133,13 +178,7 @@ func buildMetricSQL(query MetricQuery) (string, []any) {
if query.DateTo != "" {
add("stat_date <= ?", query.DateTo)
}
sqlText := `SELECT vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY stat_date DESC, vin ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
args = append(args, query.Limit, query.Offset)
return sqlText, args
return where, args
}
type MetricHandler struct {
@@ -167,6 +206,11 @@ func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeMetricError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeMetricError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeMetricError(w, http.StatusInternalServerError, err.Error())
@@ -175,7 +219,7 @@ func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": len(rows),
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
@@ -192,13 +236,14 @@ func parseMetricQuery(r *http.Request) (MetricQuery, error) {
return MetricQuery{}, err
}
query := MetricQuery{
VIN: values.Get("vin"),
Protocol: values.Get("protocol"),
MetricKey: values.Get("metricKey"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Protocol: values.Get("protocol"),
MetricKey: values.Get("metricKey"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDate(query.DateFrom) || !validDate(query.DateTo) {
return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD")

View File

@@ -18,14 +18,14 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km",
"LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km",
12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
))
@@ -46,6 +46,9 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 {
t.Fatalf("unexpected row: %#v", rows[0])
}
if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
t.Fatalf("vehicle key = %q", rows[0].VehicleKey)
}
if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" {
t.Fatalf("unexpected time formatting: %#v", rows[0])
}
@@ -60,14 +63,17 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km",
"LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km",
53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25",
))
@@ -81,7 +87,7 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":1`} {
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":42`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
@@ -91,6 +97,81 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
}
}
func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("JT808:013307811254", "JT808").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("JT808:013307811254", "JT808", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"JT808:013307811254", "", "2026-07-02", "JT808", MetricDailyTotalMileageKM, 10985.7, "km",
10985.7, 10985.7, 1, "TOTAL_MILEAGE_DIFF", "2026-07-02 00:03:00", "2026-07-02 00:03:00",
))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vehicleKey=JT808:013307811254&protocol=JT808", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"metric_value":10985.7`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("YUTONG_MQTT").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("YUTONG_MQTT", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?protocol=YUTONG_MQTT", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
t.Fatalf("expected empty items array, got: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerRejectsInvalidPagination(t *testing.T) {
handler := NewMetricHandler(NewMetricRepository(&sql.DB{}))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?limit=2001", nil)

View File

@@ -2,7 +2,8 @@ package stats
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vin VARCHAR(32) NOT NULL,
vehicle_key VARCHAR(96) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '',
stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL,
metric_key VARCHAR(64) NOT NULL,
@@ -14,7 +15,8 @@ const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
calculation_method VARCHAR(64) NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key),
KEY idx_vin (vin),
KEY idx_stat_date (stat_date),
KEY idx_protocol_metric (protocol, metric_key)
)`

162
tools/go_kafka_prod_smoke.py Executable file
View File

@@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""Kafka topic and consumer lag smoke checks for the Go native production stack."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
DEFAULT_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_BOOTSTRAP = "127.0.0.1:9092"
DEFAULT_KAFKA_BIN = "/opt/kafka/current/bin"
DEFAULT_TOPICS = [
"vehicle.raw.go.gb32960.v1",
"vehicle.raw.go.jt808.v1",
"vehicle.raw.go.yutong-mqtt.v1",
"vehicle.event.go.unified.v1",
]
DEFAULT_GROUPS = [
"go-history-writer",
"go-stat-writer",
"go-realtime-api",
]
@dataclass(frozen=True)
class Check:
name: str
status: str
message: str
@dataclass(frozen=True)
class ConsumerLag:
group: str
topic: str
partition: int
current_offset: int | None
log_end_offset: int
lag: int | None
def parse_topic_list(output: str) -> set[str]:
return {line.strip() for line in output.splitlines() if line.strip()}
def topic_checks(existing: set[str], required: list[str]) -> list[Check]:
checks: list[Check] = []
for topic in required:
if topic in existing:
checks.append(Check("topic." + topic, "pass", "present"))
else:
checks.append(Check("topic." + topic, "fail", "missing"))
return checks
def parse_consumer_group_describe(output: str) -> list[ConsumerLag]:
rows: list[ConsumerLag] = []
for line in output.splitlines():
parts = line.split()
if len(parts) < 6 or parts[0] == "GROUP":
continue
group, topic = parts[0], parts[1]
try:
partition = int(parts[2])
current = parse_optional_int(parts[3])
log_end = int(parts[4])
lag = parse_optional_int(parts[5])
except ValueError:
continue
rows.append(ConsumerLag(group, topic, partition, current, log_end, lag))
return rows
def parse_optional_int(value: str) -> int | None:
value = value.strip()
if value == "-":
return None
return int(value)
def group_lag_check(group: str, rows: list[ConsumerLag], max_lag: int) -> Check:
group_rows = [row for row in rows if row.group == group]
if not group_rows:
return Check("group." + group, "fail", "no rows")
concrete_lags = [row.lag for row in group_rows if row.lag is not None]
if not concrete_lags:
return Check("group." + group, "fail", "no concrete lag rows")
max_seen = max(concrete_lags)
topics = sorted({row.topic for row in group_rows})
status = "pass" if max_seen <= max_lag else "fail"
return Check(
"group." + group,
status,
"max_lag=" + str(max_seen) + "; threshold=" + str(max_lag) + "; topics=" + ",".join(topics),
)
def ssh(host: str, user: str, command: str, timeout: float) -> str:
target = user + "@" + host if user else host
completed = subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", target, command],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
return completed.stdout
def remote_topic_command(kafka_bin: str, bootstrap: str) -> str:
return kafka_bin.rstrip("/") + "/kafka-topics.sh --bootstrap-server " + bootstrap + " --list"
def remote_group_command(kafka_bin: str, bootstrap: str, group: str) -> str:
return kafka_bin.rstrip("/") + "/kafka-consumer-groups.sh --bootstrap-server " + bootstrap + " --describe --group " + group
def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
topic_output = ssh(args.host, args.user, remote_topic_command(args.kafka_bin, args.bootstrap), args.timeout)
checks = topic_checks(parse_topic_list(topic_output), DEFAULT_TOPICS)
for group in DEFAULT_GROUPS:
group_output = ssh(args.host, args.user, remote_group_command(args.kafka_bin, args.bootstrap, group), args.timeout)
checks.append(group_lag_check(group, parse_consumer_group_describe(group_output), args.max_lag))
status = "fail" if any(check.status == "fail" for check in checks) else "pass"
return status, checks
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--bootstrap", default=DEFAULT_BOOTSTRAP)
parser.add_argument("--kafka-bin", default=DEFAULT_KAFKA_BIN)
parser.add_argument("--max-lag", type=int, default=100)
parser.add_argument("--timeout", type=float, default=20.0)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
status, checks = run(args)
except Exception as exc:
status = "fail"
checks = [Check("kafka.ssh", "fail", str(exc))]
print(json.dumps({
"status": status,
"host": args.host,
"bootstrap": args.bootstrap,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

253
tools/go_native_deploy.py Normal file
View File

@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Build and deploy the Go native vehicle gateway release to ECS."""
from __future__ import annotations
import argparse
import os
import pathlib
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
from dataclasses import dataclass
DEFAULT_APP_HOST = "115.29.187.205"
DEFAULT_KAFKA_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_RELEASE_ROOT = "/opt/lingniu-go-native"
DEFAULT_GO_DIR = "go/vehicle-gateway"
BINARIES = [
("gateway", "./cmd/gateway"),
("history-writer", "./cmd/history-writer"),
("stat-writer", "./cmd/stat-writer"),
("realtime-api", "./cmd/realtime-api"),
]
SERVICES = [
"lingniu-go-gateway",
"lingniu-go-history-writer",
"lingniu-go-stat-writer",
"lingniu-go-realtime-api",
]
@dataclass(frozen=True)
class BuildCommand:
output_name: str
argv: list[str]
env: list[str]
cwd: str
@dataclass(frozen=True)
class SpoolStatus:
files: int
recent: int
def git_short_sha(cwd: str) -> str:
return run_capture(["git", "rev-parse", "--short=7", "HEAD"], cwd=cwd).strip()
def build_commands(go_dir: str, output_dir: str) -> list[BuildCommand]:
commands: list[BuildCommand] = []
env = ["CGO_ENABLED=0", "GOOS=linux", "GOARCH=amd64"]
for output_name, package in BINARIES:
commands.append(BuildCommand(
output_name=output_name,
argv=[
"go",
"build",
"-trimpath",
"-ldflags=-s -w",
"-o",
str(pathlib.Path(output_dir) / output_name),
package,
],
env=env,
cwd=go_dir,
))
return commands
def build_release(go_dir: str, output_dir: str) -> None:
for command in build_commands(go_dir, output_dir):
env = os.environ.copy()
for item in command.env:
key, value = item.split("=", 1)
env[key] = value
run(command.argv, cwd=command.cwd, env=env)
def create_archive(source_dir: str, archive_path: str) -> None:
with tarfile.open(archive_path, "w:gz") as archive:
for binary, _ in BINARIES:
archive.add(pathlib.Path(source_dir) / binary, arcname=binary)
def remote_switch_command(release: str, release_root: str, remote_archive: str) -> str:
if not safe_release_name(release):
raise ValueError("release must contain only letters, numbers, dot, underscore, or dash")
release_dir = release_root.rstrip("/") + "/releases/" + release
root = release_root.rstrip("/")
commands = [
"set -euo pipefail",
"mkdir -p " + shell_quote(release_dir),
"tar -xzf " + shell_quote(remote_archive) + " -C " + shell_quote(release_dir),
"chmod +x " + shell_quote(release_dir) + "/*",
"ln -sfn " + shell_quote(release_dir) + " " + shell_quote(root + "/current"),
"systemctl daemon-reload",
]
commands.extend("systemctl restart " + service for service in SERVICES)
commands.append("rm -f " + shell_quote(remote_archive))
return " && ".join(commands)
def acceptance_command(app_host: str, kafka_host: str, date: str, timeout: float) -> list[str]:
command = [
"python3",
"tools/go_prod_acceptance.py",
"--app-host",
app_host,
"--kafka-host",
kafka_host,
"--timeout",
str(int(timeout) if timeout == int(timeout) else timeout),
]
if date:
command.extend(["--date", date])
return command
def deploy(args: argparse.Namespace) -> None:
repo = pathlib.Path(args.repo).resolve()
go_dir = str(repo / args.go_dir)
release = args.release or git_short_sha(str(repo))
with tempfile.TemporaryDirectory(prefix="lingniu-go-native-") as tmp:
output_dir = pathlib.Path(tmp) / "out"
output_dir.mkdir()
archive_path = pathlib.Path(tmp) / ("lingniu-go-native-" + release + ".tar.gz")
build_release(go_dir, str(output_dir))
create_archive(str(output_dir), str(archive_path))
remote_archive = "/tmp/" + archive_path.name
target = ssh_target(args.user, args.app_host)
run(["scp", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", str(archive_path), target + ":" + remote_archive])
run([
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
target,
remote_switch_command(release, args.release_root, remote_archive),
])
wait_for_spool_drain(target, args.release_root, args.spool_drain_timeout, args.spool_poll_interval)
if not args.skip_acceptance:
run(acceptance_command(args.app_host, args.kafka_host, args.date, args.timeout), cwd=str(repo))
def wait_for_spool_drain(target: str, release_root: str, timeout: float, poll_interval: float) -> None:
deadline = time.monotonic() + timeout
last = SpoolStatus(files=-1, recent=-1)
while time.monotonic() <= deadline:
output = run_capture([
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
target,
spool_status_command(release_root),
])
last = parse_spool_status(output)
print(f"spool files={last.files} recent={last.recent}", flush=True)
if last.files == 0:
return
time.sleep(max(1.0, poll_interval))
raise RuntimeError(f"gateway spool did not drain: files={last.files} recent={last.recent}")
def spool_status_command(release_root: str) -> str:
spool_dir = release_root.rstrip("/") + "/spool/gateway"
return (
"spool=" + shell_quote(spool_dir) + "; "
"files=$(find \"$spool\" -type f 2>/dev/null | wc -l); "
"recent=$(find \"$spool\" -type f -mmin -5 2>/dev/null | wc -l); "
"printf 'files=%s recent=%s\\n' \"$files\" \"$recent\""
)
def parse_spool_status(output: str) -> SpoolStatus:
values: dict[str, int] = {}
for part in output.split():
if "=" not in part:
continue
key, value = part.split("=", 1)
try:
values[key] = int(value)
except ValueError:
continue
return SpoolStatus(files=values.get("files", -1), recent=values.get("recent", -1))
def ssh_target(user: str, host: str) -> str:
return user + "@" + host if user else host
def shell_quote(value: str) -> str:
return "'" + value.replace("'", "'\"'\"'") + "'"
def safe_release_name(value: str) -> bool:
value = value.strip()
if not value:
return False
return all(r.isalnum() or r in "._-" for r in value)
def run(argv: list[str], cwd: str | None = None, env: dict[str, str] | None = None) -> None:
subprocess.run(argv, cwd=cwd, env=env, check=True)
def run_capture(argv: list[str], cwd: str | None = None) -> str:
return subprocess.run(argv, cwd=cwd, check=True, stdout=subprocess.PIPE, text=True).stdout
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", default=".")
parser.add_argument("--go-dir", default=DEFAULT_GO_DIR)
parser.add_argument("--app-host", default=DEFAULT_APP_HOST)
parser.add_argument("--kafka-host", default=DEFAULT_KAFKA_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--release-root", default=DEFAULT_RELEASE_ROOT)
parser.add_argument("--release", default="")
parser.add_argument("--date", default="")
parser.add_argument("--timeout", type=float, default=20.0)
parser.add_argument("--spool-drain-timeout", type=float, default=900.0)
parser.add_argument("--spool-poll-interval", type=float, default=10.0)
parser.add_argument("--skip-acceptance", action="store_true")
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
deploy(args)
except subprocess.CalledProcessError as exc:
print("command failed: " + " ".join(exc.cmd), file=sys.stderr)
return exc.returncode or 1
except (OSError, shutil.Error) as exc:
print("deploy failed: " + str(exc), file=sys.stderr)
return 1
except RuntimeError as exc:
print("deploy failed: " + str(exc), file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

544
tools/go_native_prod_smoke.py Executable file
View File

@@ -0,0 +1,544 @@
#!/usr/bin/env python3
"""HTTP smoke checks for the Go native production vehicle ingest stack."""
from __future__ import annotations
import argparse
import datetime as dt
import json
import sys
import urllib.parse
import urllib.request
from dataclasses import asdict, dataclass
from typing import Any
SHANGHAI = dt.timezone(dt.timedelta(hours=8))
DEFAULT_BASE_URL = "http://115.29.187.205:20210"
DEFAULT_REALTIME_MAX_AGE_MINUTES = 15.0
@dataclass(frozen=True)
class CheckSpec:
name: str
path: str
params: dict[str, Any]
minimum: int
kind: str = "total"
max_age_minutes: float | None = None
require_parsed_json: bool = False
require_frame_id: bool = False
require_metric_formula: bool = False
@dataclass(frozen=True)
class Check:
name: str
status: str
count: int
minimum: int
message: str
@dataclass(frozen=True)
class CheckWindow:
date_from: str
date_to: str
stat_date: str
max_raw_age_minutes: float | None
def shanghai_day_window(now: dt.datetime | None = None) -> tuple[str, str]:
now = now or dt.datetime.now(tz=SHANGHAI)
local = now.astimezone(SHANGHAI)
start = dt.datetime(local.year, local.month, local.day, tzinfo=SHANGHAI)
end = start + dt.timedelta(days=1)
return start.isoformat(timespec="seconds"), end.isoformat(timespec="seconds")
def resolve_check_window(
raw_date: str | None,
max_raw_age_minutes: float,
now: dt.datetime | None = None,
) -> CheckWindow:
current = (now or dt.datetime.now(tz=SHANGHAI)).astimezone(SHANGHAI)
today = current.date()
if raw_date:
day = dt.date.fromisoformat(raw_date)
start = dt.datetime(day.year, day.month, day.day, tzinfo=SHANGHAI)
date_from, date_to = shanghai_day_window(start)
freshness = max_raw_age_minutes if day == today else None
return CheckWindow(date_from, date_to, raw_date, freshness)
date_from, date_to = shanghai_day_window(current)
return CheckWindow(date_from, date_to, date_from[:10], max_raw_age_minutes)
def api_url(base_url: str, path: str, params: dict[str, Any]) -> str:
url = base_url.rstrip("/") + path
if not params:
return url
return url + "?" + urllib.parse.urlencode(params)
def query_json(base_url: str, path: str, params: dict[str, Any], timeout: float) -> dict[str, Any]:
request = urllib.request.Request(
api_url(base_url, path, params),
headers={"Accept": "application/json"},
)
with urllib.request.urlopen(request, timeout=timeout) as response:
return json.loads(response.read().decode("utf-8"))
def check_total(
name: str,
payload: dict[str, Any],
minimum: int,
max_age_minutes: float | None = None,
now: dt.datetime | None = None,
require_parsed_json: bool = False,
require_frame_id: bool = False,
require_metric_formula: bool = False,
) -> Check:
count = int(payload.get("total") or 0)
items = payload.get("items") or []
sample = sample_summary(items)
age_message = latest_age_message(items, now)
if count >= minimum:
parsed_message = ""
if max_age_minutes is not None:
latest_age = latest_age_minutes(items, now)
if latest_age is None:
return Check(name, "fail", count, minimum, f"count={count}; missing latest ts; sample={sample}")
if latest_age > max_age_minutes:
return Check(
name,
"fail",
count,
minimum,
f"count={count}; latest_age_minutes={latest_age:.1f}; "
f"expected <= {max_age_minutes:.1f}; sample={sample}",
)
if require_parsed_json:
parsed_message = parsed_json_message(items)
if not parsed_message.startswith("parsed_json=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {parsed_message}; sample={sample}")
frame_message = ""
if require_frame_id:
frame_message = frame_id_message(items)
if not frame_message.startswith("frame_id=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {frame_message}; sample={sample}")
metric_message = ""
if require_metric_formula:
metric_message = metric_formula_message(items)
if not metric_message.startswith("metric_formula=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {metric_message}; sample={sample}")
details = [f"count={count}", age_message]
if parsed_message:
details.append(parsed_message)
if frame_message:
details.append(frame_message)
if metric_message:
details.append(metric_message)
return Check(name, "pass", count, minimum, "; ".join(details) + f"; sample={sample}")
return Check(name, "fail", count, minimum, f"count={count}; expected >= {minimum}; {age_message}; sample={sample}")
def parse_tdengine_utc_timestamp(value: str) -> dt.datetime:
value = str(value or "").strip()
for layout in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S"):
try:
return dt.datetime.strptime(value[:19], layout).replace(tzinfo=dt.timezone.utc)
except ValueError:
continue
raise ValueError(f"unsupported timestamp: {value}")
def latest_age_minutes(items: list[Any], now: dt.datetime | None = None) -> float | None:
if not items or not isinstance(items[0], dict):
return None
timestamp = items[0].get("received_at") or items[0].get("ts")
if not timestamp:
return None
current = now or dt.datetime.now(tz=dt.timezone.utc)
if current.tzinfo is None:
current = current.replace(tzinfo=dt.timezone.utc)
latest = parse_tdengine_utc_timestamp(str(timestamp))
return max(0.0, (current.astimezone(dt.timezone.utc) - latest).total_seconds() / 60)
def latest_age_message(items: list[Any], now: dt.datetime | None = None) -> str:
latest_age = latest_age_minutes(items, now)
if latest_age is None:
return "latest_age_minutes=unknown"
return f"latest_age_minutes={latest_age:.1f}"
def parsed_json_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing parsed_json"
raw = items[0].get("parsed_json")
if isinstance(raw, dict) and raw:
return "parsed_json=ok"
if not isinstance(raw, str) or not raw.strip() or raw.strip().lower() == "null":
return "missing parsed_json"
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return "invalid parsed_json"
if isinstance(parsed, dict) and parsed:
return "parsed_json=ok"
return "invalid parsed_json"
def frame_id_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing frame_id"
frame_id = str(items[0].get("frame_id") or "").strip()
if frame_id:
return "frame_id=ok"
return "missing frame_id"
def metric_formula_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing metric row"
row = items[0]
try:
metric_value = float(row.get("metric_value"))
first_total = float(row.get("first_total_mileage_km"))
latest_total = float(row.get("latest_total_mileage_km"))
except (TypeError, ValueError):
return "missing metric formula fields"
metric_key = str(row.get("metric_key") or "")
if metric_key == "daily_mileage_km":
expected = max(0.0, latest_total - first_total)
elif metric_key == "daily_total_mileage_km":
expected = latest_total
else:
return "unsupported metric_key=" + metric_key
if abs(metric_value - expected) <= 0.001:
return "metric_formula=ok"
return f"metric_formula mismatch value={metric_value:.3f} expected={expected:.3f}"
def check_realtime(
name: str,
payload: dict[str, Any],
max_age_minutes: float | None = None,
now: dt.datetime | None = None,
) -> Check:
if payload.get("online") is False:
return Check(name, "fail", 0, 1, "online=false")
updated_message = ""
if max_age_minutes is not None:
updated_age = realtime_updated_age_minutes(payload, now)
if updated_age is None:
return Check(name, "fail", 0, 1, "missing updated_at_ms")
updated_message = f"updated_age_minutes={updated_age:.1f}"
if updated_age > max_age_minutes:
return Check(
name,
"fail",
0,
1,
updated_message + f"; expected <= {max_age_minutes:.1f}",
)
identifier = payload.get("vin") or payload.get("vehicle_key") or ""
protocols = payload.get("protocols") or []
protocol = payload.get("protocol") or ""
fields = payload.get("fields") or {}
message = json.dumps(
{
key: value
for key, value in {
"identifier": identifier,
"protocol": protocol,
"protocols": protocols,
"field_count": len(fields) if isinstance(fields, dict) else 0,
"online": payload.get("online"),
}.items()
if value not in (None, "", [])
},
ensure_ascii=False,
sort_keys=True,
)
if updated_message:
message = updated_message + "; " + message
return Check(name, "pass", 1, 1, message)
def realtime_updated_age_minutes(payload: dict[str, Any], now: dt.datetime | None = None) -> float | None:
try:
updated_ms = int(payload.get("updated_at_ms"))
except (TypeError, ValueError):
return None
current = now or dt.datetime.now(tz=dt.timezone.utc)
if current.tzinfo is None:
current = current.replace(tzinfo=dt.timezone.utc)
updated = dt.datetime.fromtimestamp(updated_ms / 1000, tz=dt.timezone.utc)
return max(0.0, (current.astimezone(dt.timezone.utc) - updated).total_seconds() / 60)
def sample_summary(items: list[Any]) -> str:
if not items:
return "{}"
first = items[0]
if not isinstance(first, dict):
return json.dumps(first, ensure_ascii=False, sort_keys=True)
keep = {
key: first.get(key)
for key in (
"ts",
"stat_date",
"protocol",
"vehicle_key",
"vin",
"phone",
"message_id_hex",
"metric_key",
"metric_value",
"total_mileage_km",
"latest_total_mileage_km",
)
if first.get(key) not in (None, "")
}
return json.dumps(keep, ensure_ascii=False, sort_keys=True)
def overall_status(checks: list[Check]) -> str:
return "fail" if any(check.status == "fail" for check in checks) else "pass"
def build_check_specs(
*,
date_from: str,
date_to: str,
stat_date: str,
min_raw: int,
min_history: int,
min_stat: int,
max_raw_age_minutes: float | None = None,
) -> list[CheckSpec]:
raw_params = {"limit": 1, "orderBy": "receivedAt", "includeTotal": "false"}
gb32960_history_params = {"protocol": "GB32960", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
jt808_history_params = {"protocol": "JT808", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
yutong_mqtt_history_params = {"protocol": "YUTONG_MQTT", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
return [
CheckSpec(
"gb32960.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "GB32960"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec(
"jt808.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "JT808"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec(
"yutong_mqtt.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "YUTONG_MQTT"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec("gb32960.locations", "/api/history/locations", gb32960_history_params, min_history, require_frame_id=True),
CheckSpec("gb32960.mileage_points", "/api/history/mileage-points", gb32960_history_params, min_history, require_frame_id=True),
CheckSpec("jt808.locations", "/api/history/locations", jt808_history_params, min_history, require_frame_id=True),
CheckSpec("jt808.mileage_points", "/api/history/mileage-points", jt808_history_params, min_history, require_frame_id=True),
CheckSpec("yutong_mqtt.locations", "/api/history/locations", yutong_mqtt_history_params, min_history, require_frame_id=True),
CheckSpec("yutong_mqtt.mileage_points", "/api/history/mileage-points", yutong_mqtt_history_params, min_history, require_frame_id=True),
CheckSpec(
"gb32960.daily_mileage",
"/api/stats/daily-metrics",
{
"protocol": "GB32960",
"metricKey": "daily_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"gb32960.daily_total_mileage",
"/api/stats/daily-metrics",
{
"protocol": "GB32960",
"metricKey": "daily_total_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"jt808.daily_mileage",
"/api/stats/daily-metrics",
{
"protocol": "JT808",
"metricKey": "daily_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"jt808.daily_total_mileage",
"/api/stats/daily-metrics",
{
"protocol": "JT808",
"metricKey": "daily_total_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
]
def vehicle_identifier(item: dict[str, Any]) -> str:
for key in ("vin", "vehicle_key"):
value = str(item.get(key) or "").strip()
if value and value.lower() != "unknown":
return value
return ""
def build_realtime_specs(payloads: dict[str, dict[str, Any]]) -> list[CheckSpec]:
configs = [
("gb32960", "gb32960.raw", "GB32960"),
("jt808", "jt808.raw", "JT808"),
("yutong_mqtt", "yutong_mqtt.raw", "YUTONG_MQTT"),
]
specs: list[CheckSpec] = []
for prefix, source_name, protocol in configs:
items = payloads.get(source_name, {}).get("items") or []
if not items or not isinstance(items[0], dict):
continue
identifier = vehicle_identifier(items[0])
if not identifier:
continue
encoded = urllib.parse.quote(identifier, safe="")
base_path = "/api/realtime/vehicles/" + encoded
specs.extend([
CheckSpec(
prefix + ".realtime",
base_path,
{},
1,
"realtime",
max_age_minutes=DEFAULT_REALTIME_MAX_AGE_MINUTES,
),
CheckSpec(prefix + ".realtime_online", base_path + "/online", {}, 1, "realtime"),
CheckSpec(
prefix + ".realtime_protocol",
base_path + "/protocols/" + protocol,
{},
1,
"realtime",
max_age_minutes=DEFAULT_REALTIME_MAX_AGE_MINUTES,
),
])
return specs
def run_checks(base_url: str, specs: list[CheckSpec], timeout: float) -> list[Check]:
checks: list[Check] = []
for spec in specs:
try:
payload = query_json(base_url, spec.path, spec.params, timeout)
if spec.kind == "realtime":
checks.append(check_realtime(spec.name, payload, spec.max_age_minutes))
else:
checks.append(check_total(
spec.name,
payload,
spec.minimum,
spec.max_age_minutes,
require_parsed_json=spec.require_parsed_json,
require_frame_id=spec.require_frame_id,
require_metric_formula=spec.require_metric_formula,
))
except Exception as exc: # pragma: no cover - exercised by live failures.
checks.append(Check(spec.name, "fail", 0, spec.minimum, f"request failed: {exc}"))
return checks
def query_payloads(base_url: str, specs: list[CheckSpec], timeout: float) -> tuple[list[Check], dict[str, dict[str, Any]]]:
checks: list[Check] = []
payloads: dict[str, dict[str, Any]] = {}
for spec in specs:
try:
payload = query_json(base_url, spec.path, spec.params, timeout)
payloads[spec.name] = payload
checks.append(check_total(
spec.name,
payload,
spec.minimum,
spec.max_age_minutes,
require_parsed_json=spec.require_parsed_json,
require_frame_id=spec.require_frame_id,
require_metric_formula=spec.require_metric_formula,
))
except Exception as exc:
checks.append(Check(spec.name, "fail", 0, spec.minimum, f"request failed: {exc}"))
return checks, payloads
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--date", help="Shanghai local date, YYYY-MM-DD. Defaults to today.")
parser.add_argument("--timeout", type=float, default=5.0)
parser.add_argument("--min-raw", type=int, default=1)
parser.add_argument("--min-history", type=int, default=1)
parser.add_argument("--min-stat", type=int, default=1)
parser.add_argument(
"--max-raw-age-minutes",
type=float,
default=15.0,
help="Require latest RAW sample age to be at most this many minutes when checking today's date.",
)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
window = resolve_check_window(args.date, args.max_raw_age_minutes)
specs = build_check_specs(
date_from=window.date_from,
date_to=window.date_to,
stat_date=window.stat_date,
min_raw=args.min_raw,
min_history=args.min_history,
min_stat=args.min_stat,
max_raw_age_minutes=window.max_raw_age_minutes,
)
checks, payloads = query_payloads(args.base_url, specs, args.timeout)
checks.extend(run_checks(args.base_url, build_realtime_specs(payloads), args.timeout))
status = overall_status(checks)
print(json.dumps({
"status": status,
"baseUrl": args.base_url,
"dateFrom": window.date_from,
"dateTo": window.date_to,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

136
tools/go_prod_acceptance.py Executable file
View File

@@ -0,0 +1,136 @@
#!/usr/bin/env python3
"""Run the Go native production acceptance smoke suite."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
from typing import Any
DEFAULT_APP_HOST = "115.29.187.205"
DEFAULT_KAFKA_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_BASE_URL = "http://115.29.187.205:20210"
@dataclass(frozen=True)
class ChildCommand:
name: str
argv: list[str]
@dataclass(frozen=True)
class ChildResult:
name: str
status: str
exit_code: int
payload: dict[str, Any]
def build_commands(args: argparse.Namespace) -> list[ChildCommand]:
timeout = str(args.timeout)
return [
ChildCommand("systemd", [
sys.executable,
"tools/go_systemd_prod_smoke.py",
"--host",
args.app_host,
"--user",
args.ssh_user,
"--timeout",
timeout,
]),
ChildCommand("kafka", [
sys.executable,
"tools/go_kafka_prod_smoke.py",
"--host",
args.kafka_host,
"--user",
args.ssh_user,
"--max-lag",
str(args.max_lag),
"--timeout",
timeout,
]),
ChildCommand("http", [
sys.executable,
"tools/go_native_prod_smoke.py",
"--base-url",
args.base_url,
"--timeout",
timeout,
] + (["--date", args.date] if args.date else [])),
]
def run_child(command: ChildCommand, timeout: float) -> ChildResult:
completed = subprocess.run(
command.argv,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
payload = parse_json_output(completed.stdout)
if completed.stderr.strip():
payload.setdefault("stderr", completed.stderr.strip())
status = payload.get("status")
if status not in {"pass", "fail"}:
status = "pass" if completed.returncode == 0 else "fail"
return ChildResult(command.name, status, completed.returncode, payload)
def parse_json_output(output: str) -> dict[str, Any]:
output = output.strip()
if not output:
return {}
start = output.find("{")
end = output.rfind("}")
if start < 0 or end < start:
return {"rawOutput": output}
try:
return json.loads(output[start:end + 1])
except json.JSONDecodeError:
return {"rawOutput": output}
def overall_status(results: list[ChildResult]) -> str:
if not results:
return "fail"
return "fail" if any(result.status != "pass" or result.exit_code != 0 for result in results) else "pass"
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--app-host", default=DEFAULT_APP_HOST)
parser.add_argument("--kafka-host", default=DEFAULT_KAFKA_HOST)
parser.add_argument("--ssh-user", default=DEFAULT_USER)
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--date", help="Shanghai local date, YYYY-MM-DD. Defaults to today in go_native_prod_smoke.py.")
parser.add_argument("--timeout", type=float, default=20.0)
parser.add_argument("--max-lag", type=int, default=100)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
results: list[ChildResult] = []
for command in build_commands(args):
try:
results.append(run_child(command, args.timeout + 10))
except Exception as exc:
results.append(ChildResult(command.name, "fail", 1, {"status": "fail", "error": str(exc)}))
status = overall_status(results)
print(json.dumps({
"status": status,
"checks": [asdict(result) for result in results],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

210
tools/go_systemd_prod_smoke.py Executable file
View File

@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""Systemd and port smoke checks for the Go native production ECS."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
DEFAULT_HOST = "115.29.187.205"
DEFAULT_USER = "root"
DEFAULT_SERVICES = [
"lingniu-go-gateway.service",
"lingniu-go-history-writer.service",
"lingniu-go-stat-writer.service",
"lingniu-go-realtime-api.service",
]
DEFAULT_PORTS = {
808: "gateway",
32960: "gateway",
20210: "realtime-api",
}
DEFAULT_RELEASE_ROOT = "/opt/lingniu-go-native"
DEFAULT_SPOOL_DIR = "/opt/lingniu-go-native/spool/gateway"
DEFAULT_BINARIES = ["gateway", "history-writer", "stat-writer", "realtime-api"]
@dataclass(frozen=True)
class Check:
name: str
status: str
message: str
@dataclass(frozen=True)
class ServiceState:
active: str
enabled: str
def parse_service_states(output: str) -> dict[str, ServiceState]:
states: dict[str, ServiceState] = {}
for line in output.splitlines():
parts = line.split()
if len(parts) >= 2:
enabled = parts[2] if len(parts) >= 3 else "enabled"
states[parts[0]] = ServiceState(parts[1], enabled)
return states
def service_checks(output: str, services: list[str]) -> list[Check]:
states = parse_service_states(output)
checks: list[Check] = []
for service in services:
state = states.get(service, ServiceState("missing", "missing"))
status = "pass" if state.active == "active" and state.enabled == "enabled" else "fail"
checks.append(Check(
"service." + service,
status,
"state=" + state.active + "; enabled=" + state.enabled,
))
return checks
def port_checks(output: str, ports: dict[int, str]) -> list[Check]:
lines = output.splitlines()
checks: list[Check] = []
for port, process in sorted(ports.items()):
matching = [line.strip() for line in lines if has_port(line, port)]
if not matching:
checks.append(Check("port." + str(port), "fail", "not listening"))
continue
owned = [line for line in matching if process in line]
if owned:
checks.append(Check("port." + str(port), "pass", owned[0]))
else:
checks.append(Check("port." + str(port), "fail", matching[0]))
return checks
def has_port(line: str, port: int) -> bool:
needle = ":" + str(port)
return needle + " " in line or needle + "\t" in line
def spool_check(output: str) -> Check:
values = parse_key_values(output)
files = int(values.get("files", "-1"))
recent = int(values.get("recent", "-1"))
status = "pass" if files == 0 and recent == 0 else "fail"
return Check("spool.gateway", status, output.strip() or "missing")
def release_check(output: str) -> Check:
values = parse_key_values(output)
current = values.get("current", "")
binaries_present = [values.get(binary, "0") == "1" for binary in DEFAULT_BINARIES]
status = "pass" if current and all(binaries_present) else "fail"
return Check("release.current", status, output.strip() or "missing")
def parse_key_values(output: str) -> dict[str, str]:
values: dict[str, str] = {}
for part in output.split():
if "=" not in part:
continue
key, value = part.split("=", 1)
values[key] = value
return values
def ssh(host: str, user: str, command: str, timeout: float) -> str:
target = user + "@" + host if user else host
completed = subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", target, command],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
return completed.stdout
def remote_command(services: list[str]) -> str:
quoted = " ".join(services)
return (
"for svc in " + quoted + "; do "
"printf '%s ' \"$svc\"; "
"printf '%s ' \"$(systemctl is-active \"$svc\" 2>/dev/null || true)\"; "
"systemctl is-enabled \"$svc\" 2>/dev/null || true; "
"done; "
"printf '\\n--PORTS--\\n'; "
"ss -lntp; "
"printf '\\n--SPOOL--\\n'; "
"spool='" + DEFAULT_SPOOL_DIR + "'; "
"if test -d \"$spool\"; then "
"files=$(find \"$spool\" -type f | wc -l); "
"bytes=$(du -sb \"$spool\" 2>/dev/null | awk '{print $1}'); "
"recent=$(find \"$spool\" -type f -mmin -5 | wc -l); "
"printf 'files=%s bytes=%s recent=%s\\n' \"$files\" \"${bytes:-0}\" \"$recent\"; "
"else printf 'files=-1 bytes=0 recent=-1 missing=%s\\n' \"$spool\"; fi; "
"printf '\\n--RELEASE--\\n'; "
"root='" + DEFAULT_RELEASE_ROOT + "'; "
"current=$(readlink -f \"$root/current\" 2>/dev/null || true); "
"printf 'current=%s' \"$current\"; "
"for bin in " + " ".join(DEFAULT_BINARIES) + "; do "
"if test -x \"$root/current/$bin\"; then present=1; else present=0; fi; "
"printf ' %s=%s' \"$bin\" \"$present\"; "
"done; printf '\\n'"
)
def split_remote_output(output: str) -> tuple[str, str, str, str]:
ports_marker = "\n--PORTS--\n"
spool_marker = "\n--SPOOL--\n"
release_marker = "\n--RELEASE--\n"
if ports_marker not in output:
return output, "", "", ""
service_output, rest = output.split(ports_marker, 1)
if spool_marker not in rest:
return service_output, rest, "", ""
port_output, spool_output = rest.split(spool_marker, 1)
if release_marker not in spool_output:
return service_output, port_output, spool_output, ""
spool_output, release_output = spool_output.split(release_marker, 1)
return service_output, port_output, spool_output, release_output
def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
output = ssh(args.host, args.user, remote_command(DEFAULT_SERVICES), args.timeout)
service_output, port_output, spool_output, release_output = split_remote_output(output)
checks = (
service_checks(service_output, DEFAULT_SERVICES)
+ port_checks(port_output, DEFAULT_PORTS)
+ [spool_check(spool_output)]
+ [release_check(release_output)]
)
status = "fail" if any(check.status == "fail" for check in checks) else "pass"
return status, checks
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--timeout", type=float, default=20.0)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
status, checks = run(args)
except Exception as exc:
status = "fail"
checks = [Check("systemd.ssh", "fail", str(exc))]
print(json.dumps({
"status": status,
"host": args.host,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

View File

@@ -0,0 +1,62 @@
import unittest
from tools import go_kafka_prod_smoke as smoke
class GoKafkaProdSmokeTest(unittest.TestCase):
def test_parse_topic_list_marks_required_topics_present(self):
topics = smoke.parse_topic_list("""
vehicle.raw.go.gb32960.v1
vehicle.raw.go.jt808.v1
vehicle.raw.go.yutong-mqtt.v1
vehicle.event.go.unified.v1
""")
checks = smoke.topic_checks(topics, smoke.DEFAULT_TOPICS)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_topic_checks_fail_when_required_topic_missing(self):
topics = {"vehicle.raw.go.gb32960.v1"}
checks = smoke.topic_checks(topics, ["vehicle.raw.go.gb32960.v1", "vehicle.event.go.unified.v1"])
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["topic.vehicle.event.go.unified.v1"].status, "fail")
def test_parse_consumer_group_describe_reads_lag_rows(self):
rows = smoke.parse_consumer_group_describe("""
GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG
go-realtime-api vehicle.event.go.unified.v1 0 15992 15992 0
go-realtime-api vehicle.event.go.unified.v1 1 2036 2036 3
""")
self.assertEqual(rows[0].group, "go-realtime-api")
self.assertEqual(rows[1].topic, "vehicle.event.go.unified.v1")
self.assertEqual(rows[1].lag, 3)
def test_lag_checks_fail_when_lag_exceeds_threshold(self):
rows = [
smoke.ConsumerLag("go-history-writer", "vehicle.raw.go.jt808.v1", 0, 100, 100, 0),
smoke.ConsumerLag("go-history-writer", "vehicle.raw.go.jt808.v1", 1, 100, 110, 10),
]
check = smoke.group_lag_check("go-history-writer", rows, max_lag=5)
self.assertEqual(check.status, "fail")
self.assertIn("max_lag=10", check.message)
def test_lag_checks_pass_when_group_has_rows_and_lag_is_small(self):
rows = [
smoke.ConsumerLag("go-stat-writer", "vehicle.raw.go.gb32960.v1", 0, 100, 100, 0),
smoke.ConsumerLag("go-stat-writer", "vehicle.raw.go.jt808.v1", 1, 100, 101, 1),
]
check = smoke.group_lag_check("go-stat-writer", rows, max_lag=5)
self.assertEqual(check.status, "pass")
self.assertIn("max_lag=1", check.message)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,63 @@
import unittest
from tools import go_native_deploy as deploy
class GoNativeDeployTest(unittest.TestCase):
def test_build_commands_cover_four_production_binaries(self):
commands = deploy.build_commands("/repo/go/vehicle-gateway", "/tmp/out")
self.assertEqual([command.output_name for command in commands], [
"gateway",
"history-writer",
"stat-writer",
"realtime-api",
])
for command in commands:
self.assertEqual(command.argv[:3], ["go", "build", "-trimpath"])
self.assertIn("GOOS=linux", command.env)
self.assertIn("GOARCH=amd64", command.env)
self.assertIn("CGO_ENABLED=0", command.env)
def test_remote_switch_command_creates_release_and_restarts_all_services(self):
command = deploy.remote_switch_command("409f55b", "/opt/lingniu-go-native", "/tmp/lingniu-go-native-409f55b.tar.gz")
self.assertIn("/opt/lingniu-go-native/releases/409f55b", command)
self.assertIn("ln -sfn", command)
self.assertIn("systemctl restart lingniu-go-gateway", command)
self.assertIn("systemctl restart lingniu-go-history-writer", command)
self.assertIn("systemctl restart lingniu-go-stat-writer", command)
self.assertIn("systemctl restart lingniu-go-realtime-api", command)
def test_acceptance_command_uses_deployed_hosts_and_date(self):
command = deploy.acceptance_command(
app_host="115.29.187.205",
kafka_host="114.55.58.251",
date="2026-07-02",
timeout=20,
)
self.assertEqual(command[:2], ["python3", "tools/go_prod_acceptance.py"])
self.assertIn("--app-host", command)
self.assertIn("115.29.187.205", command)
self.assertIn("--kafka-host", command)
self.assertIn("114.55.58.251", command)
self.assertIn("--date", command)
self.assertIn("2026-07-02", command)
def test_parse_spool_status_reads_file_and_recent_counts(self):
status = deploy.parse_spool_status("files=42 recent=3")
self.assertEqual(status.files, 42)
self.assertEqual(status.recent, 3)
def test_spool_status_command_targets_gateway_spool_directory(self):
command = deploy.spool_status_command("/opt/lingniu-go-native")
self.assertIn("/opt/lingniu-go-native/spool/gateway", command)
self.assertIn("files=", command)
self.assertIn("recent=", command)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,358 @@
import datetime as dt
import unittest
from tools import go_native_prod_smoke as smoke
class GoNativeProdSmokeTest(unittest.TestCase):
def test_shanghai_day_window_uses_plus_eight_bounds(self):
now = dt.datetime(2026, 7, 2, 1, 35, tzinfo=smoke.SHANGHAI)
date_from, date_to = smoke.shanghai_day_window(now)
self.assertEqual(date_from, "2026-07-02T00:00:00+08:00")
self.assertEqual(date_to, "2026-07-03T00:00:00+08:00")
def test_api_url_encodes_plus_eight_date_range(self):
url = smoke.api_url(
"http://example.test/base/",
"/api/history/raw-frames",
{
"protocol": "JT808",
"dateFrom": "2026-07-02T00:00:00+08:00",
"dateTo": "2026-07-03T00:00:00+08:00",
"limit": 1,
},
)
self.assertEqual(
url,
"http://example.test/base/api/history/raw-frames?"
"protocol=JT808&dateFrom=2026-07-02T00%3A00%3A00%2B08%3A00"
"&dateTo=2026-07-03T00%3A00%3A00%2B08%3A00&limit=1",
)
def test_explicit_today_date_still_enforces_raw_freshness(self):
now = dt.datetime(2026, 7, 2, 9, 30, tzinfo=smoke.SHANGHAI)
window = smoke.resolve_check_window("2026-07-02", 15.0, now)
self.assertEqual(window.date_from, "2026-07-02T00:00:00+08:00")
self.assertEqual(window.date_to, "2026-07-03T00:00:00+08:00")
self.assertEqual(window.stat_date, "2026-07-02")
self.assertEqual(window.max_raw_age_minutes, 15.0)
def test_explicit_historical_date_disables_raw_freshness(self):
now = dt.datetime(2026, 7, 2, 9, 30, tzinfo=smoke.SHANGHAI)
window = smoke.resolve_check_window("2026-07-01", 15.0, now)
self.assertEqual(window.date_from, "2026-07-01T00:00:00+08:00")
self.assertEqual(window.date_to, "2026-07-02T00:00:00+08:00")
self.assertEqual(window.stat_date, "2026-07-01")
self.assertIsNone(window.max_raw_age_minutes)
def test_total_check_passes_when_total_reaches_minimum(self):
check = smoke.check_total(
"jt808.raw",
{"total": 2, "items": [{"vehicle_key": "JT808:013307811170"}]},
minimum=1,
)
self.assertEqual(check.status, "pass")
self.assertEqual(check.count, 2)
self.assertIn("JT808:013307811170", check.message)
def test_total_check_fails_when_total_is_below_minimum(self):
check = smoke.check_total("gb32960.raw", {"total": 0, "items": []}, minimum=1)
self.assertEqual(check.status, "fail")
self.assertEqual(check.count, 0)
self.assertIn("expected >= 1", check.message)
def test_overall_status_fails_if_any_check_fails(self):
checks = [
smoke.Check("a", "pass", 1, 1, "ok"),
smoke.Check("b", "fail", 0, 1, "bad"),
]
self.assertEqual(smoke.overall_status(checks), "fail")
def test_build_checks_cover_three_raw_protocols_and_two_daily_metric_protocols(self):
specs = smoke.build_check_specs(
date_from="2026-07-02T00:00:00+08:00",
date_to="2026-07-03T00:00:00+08:00",
stat_date="2026-07-02",
min_raw=1,
min_history=1,
min_stat=1,
)
names = [spec.name for spec in specs]
self.assertIn("gb32960.raw", names)
self.assertIn("jt808.raw", names)
self.assertIn("yutong_mqtt.raw", names)
self.assertIn("gb32960.locations", names)
self.assertIn("gb32960.mileage_points", names)
self.assertIn("jt808.locations", names)
self.assertIn("jt808.mileage_points", names)
self.assertIn("yutong_mqtt.locations", names)
self.assertIn("yutong_mqtt.mileage_points", names)
self.assertIn("gb32960.daily_mileage", names)
self.assertIn("jt808.daily_total_mileage", names)
def test_raw_checks_order_by_received_at_for_ingest_freshness(self):
specs = smoke.build_check_specs(
date_from="2026-07-02T00:00:00+08:00",
date_to="2026-07-03T00:00:00+08:00",
stat_date="2026-07-02",
min_raw=1,
min_history=1,
min_stat=1,
max_raw_age_minutes=15,
)
raw_specs = [spec for spec in specs if spec.name.endswith(".raw")]
self.assertTrue(raw_specs)
for spec in raw_specs:
self.assertEqual(spec.params.get("orderBy"), "receivedAt")
self.assertEqual(spec.params.get("includeTotal"), "false")
self.assertNotIn("dateFrom", spec.params)
self.assertNotIn("dateTo", spec.params)
def test_vehicle_identifier_prefers_vin_over_vehicle_key(self):
identifier = smoke.vehicle_identifier({
"vin": "LB9A32A20R0LS1343",
"vehicle_key": "GB32960:ignored",
})
self.assertEqual(identifier, "LB9A32A20R0LS1343")
def test_vehicle_identifier_falls_back_to_vehicle_key(self):
identifier = smoke.vehicle_identifier({
"vin": "",
"vehicle_key": "JT808:013307811170",
})
self.assertEqual(identifier, "JT808:013307811170")
def test_realtime_specs_are_built_from_raw_samples_with_vin(self):
payloads = {
"gb32960.raw": {"items": [{"vin": "LB9A32A20R0LS1343", "vehicle_key": "LB9A32A20R0LS1343"}]},
"jt808.raw": {"items": [{"vin": "", "vehicle_key": "JT808:013307811170"}]},
"yutong_mqtt.raw": {"items": [{"vin": "LMRKH9AC6R1004108"}]},
}
specs = smoke.build_realtime_specs(payloads)
self.assertEqual(
[(spec.name, spec.path) for spec in specs],
[
("gb32960.realtime", "/api/realtime/vehicles/LB9A32A20R0LS1343"),
("gb32960.realtime_online", "/api/realtime/vehicles/LB9A32A20R0LS1343/online"),
("gb32960.realtime_protocol", "/api/realtime/vehicles/LB9A32A20R0LS1343/protocols/GB32960"),
("jt808.realtime", "/api/realtime/vehicles/JT808%3A013307811170"),
("jt808.realtime_online", "/api/realtime/vehicles/JT808%3A013307811170/online"),
("jt808.realtime_protocol", "/api/realtime/vehicles/JT808%3A013307811170/protocols/JT808"),
("yutong_mqtt.realtime", "/api/realtime/vehicles/LMRKH9AC6R1004108"),
("yutong_mqtt.realtime_online", "/api/realtime/vehicles/LMRKH9AC6R1004108/online"),
("yutong_mqtt.realtime_protocol", "/api/realtime/vehicles/LMRKH9AC6R1004108/protocols/YUTONG_MQTT"),
],
)
def test_realtime_check_requires_online_true_when_field_exists(self):
check = smoke.check_realtime(
"gb32960.realtime_online",
{"vin": "LB9A32A20R0LS1343", "online": True, "protocols": ["GB32960"]},
)
self.assertEqual(check.status, "pass")
self.assertEqual(check.count, 1)
def test_realtime_check_fails_when_online_false(self):
check = smoke.check_realtime(
"gb32960.realtime_online",
{"vin": "LB9A32A20R0LS1343", "online": False},
)
self.assertEqual(check.status, "fail")
self.assertIn("online=false", check.message)
def test_realtime_check_fails_when_snapshot_is_stale(self):
now = dt.datetime(2026, 7, 2, 0, 30, 0, tzinfo=dt.timezone.utc)
check = smoke.check_realtime(
"jt808.realtime",
{"vehicle_key": "JT808:013307811170", "updated_at_ms": 1782950400000},
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "fail")
self.assertIn("updated_age_minutes=30.0", check.message)
def test_realtime_check_passes_when_snapshot_is_fresh(self):
now = dt.datetime(2026, 7, 2, 0, 30, 0, tzinfo=dt.timezone.utc)
check = smoke.check_realtime(
"jt808.realtime",
{"vehicle_key": "JT808:013307811170", "updated_at_ms": 1782951600000},
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("updated_age_minutes=10.0", check.message)
def test_parse_tdengine_utc_timestamp_as_aware_utc(self):
parsed = smoke.parse_tdengine_utc_timestamp("2026-07-01 17:36:02")
self.assertEqual(parsed, dt.datetime(2026, 7, 1, 17, 36, 2, tzinfo=dt.timezone.utc))
def test_total_check_fails_when_latest_sample_is_stale(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{"total": 1, "items": [{"ts": "2026-07-01 17:00:00"}]},
minimum=1,
max_age_minutes=30,
now=now,
)
self.assertEqual(check.status, "fail")
self.assertIn("latest_age_minutes=60.0", check.message)
def test_total_check_uses_received_at_for_raw_freshness_when_present(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:00:00",
"received_at": "2026-07-01 17:58:00",
}],
},
minimum=1,
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("latest_age_minutes=2.0", check.message)
def test_total_check_passes_when_latest_sample_is_fresh(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00"}]},
minimum=1,
max_age_minutes=30,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("latest_age_minutes=15.0", check.message)
def test_raw_check_requires_structured_parsed_json(self):
check = smoke.check_total(
"jt808.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:45:00",
"parsed_json": "",
}],
},
minimum=1,
require_parsed_json=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("missing parsed_json", check.message)
def test_raw_check_accepts_structured_parsed_json(self):
check = smoke.check_total(
"gb32960.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:45:00",
"parsed_json": "{\"header\":{\"vin\":\"LTEST\"}}",
}],
},
minimum=1,
require_parsed_json=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("parsed_json=ok", check.message)
def test_history_check_requires_frame_id_backlink(self):
check = smoke.check_total(
"jt808.locations",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00", "frame_id": ""}]},
minimum=1,
require_frame_id=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("missing frame_id", check.message)
def test_history_check_accepts_frame_id_backlink(self):
check = smoke.check_total(
"jt808.mileage_points",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00", "frame_id": "go_abc"}]},
minimum=1,
require_frame_id=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("frame_id=ok", check.message)
def test_daily_mileage_formula_requires_latest_minus_first(self):
check = smoke.check_total(
"jt808.daily_mileage",
{
"total": 1,
"items": [{
"metric_key": "daily_mileage_km",
"metric_value": 40.0,
"first_total_mileage_km": 4434.9,
"latest_total_mileage_km": 4481.0,
}],
},
minimum=1,
require_metric_formula=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("metric_formula mismatch", check.message)
def test_daily_total_formula_requires_latest_total(self):
check = smoke.check_total(
"jt808.daily_total_mileage",
{
"total": 1,
"items": [{
"metric_key": "daily_total_mileage_km",
"metric_value": 4481.0,
"first_total_mileage_km": 4434.9,
"latest_total_mileage_km": 4481.0,
}],
},
minimum=1,
require_metric_formula=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("metric_formula=ok", check.message)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,53 @@
import argparse
import sys
import unittest
from tools import go_prod_acceptance as acceptance
class GoProdAcceptanceTest(unittest.TestCase):
def test_build_commands_runs_systemd_kafka_and_http_smokes(self):
args = argparse.Namespace(
app_host="115.29.187.205",
kafka_host="114.55.58.251",
ssh_user="root",
base_url="http://115.29.187.205:20210",
date="2026-07-02",
timeout=8.0,
max_lag=100,
)
commands = acceptance.build_commands(args)
self.assertEqual(len(commands), 3)
self.assertEqual(commands[0].name, "systemd")
self.assertEqual(commands[0].argv[:2], [sys.executable, "tools/go_systemd_prod_smoke.py"])
self.assertIn("--host", commands[0].argv)
self.assertIn("115.29.187.205", commands[0].argv)
self.assertEqual(commands[1].name, "kafka")
self.assertIn("tools/go_kafka_prod_smoke.py", commands[1].argv)
self.assertIn("--max-lag", commands[1].argv)
self.assertEqual(commands[2].name, "http")
self.assertIn("tools/go_native_prod_smoke.py", commands[2].argv)
self.assertIn("--base-url", commands[2].argv)
def test_overall_status_fails_when_any_child_fails(self):
results = [
acceptance.ChildResult("systemd", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("kafka", "fail", 1, {"status": "fail"}),
]
self.assertEqual(acceptance.overall_status(results), "fail")
def test_overall_status_passes_when_all_children_pass(self):
results = [
acceptance.ChildResult("systemd", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("kafka", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("http", "pass", 0, {"status": "pass"}),
]
self.assertEqual(acceptance.overall_status(results), "pass")
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,101 @@
import unittest
from tools import go_systemd_prod_smoke as smoke
class GoSystemdProdSmokeTest(unittest.TestCase):
def test_service_checks_require_all_go_units_active(self):
output = """
lingniu-go-gateway.service active enabled
lingniu-go-history-writer.service active enabled
lingniu-go-stat-writer.service active enabled
lingniu-go-realtime-api.service active enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_service_checks_fail_when_unit_is_missing_or_inactive(self):
output = """
lingniu-go-gateway.service active enabled
lingniu-go-history-writer.service failed enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["service.lingniu-go-history-writer.service"].status, "fail")
self.assertEqual(by_name["service.lingniu-go-stat-writer.service"].status, "fail")
def test_service_checks_fail_when_unit_is_not_enabled(self):
output = """
lingniu-go-gateway.service active disabled
lingniu-go-history-writer.service active enabled
lingniu-go-stat-writer.service active enabled
lingniu-go-realtime-api.service active enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["service.lingniu-go-gateway.service"].status, "fail")
self.assertIn("enabled=disabled", by_name["service.lingniu-go-gateway.service"].message)
def test_port_checks_require_expected_go_processes(self):
output = """
LISTEN 0 4096 *:32960 *:* users:(("gateway",pid=10,fd=3))
LISTEN 0 4096 *:808 *:* users:(("gateway",pid=10,fd=4))
LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
"""
checks = smoke.port_checks(output, smoke.DEFAULT_PORTS)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_port_checks_fail_when_java_owns_ingest_port(self):
output = """
LISTEN 0 4096 *:808 *:* users:(("java",pid=20,fd=4))
LISTEN 0 4096 *:32960 *:* users:(("gateway",pid=10,fd=3))
LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
"""
checks = smoke.port_checks(output, smoke.DEFAULT_PORTS)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["port.808"].status, "fail")
self.assertIn("java", by_name["port.808"].message)
def test_spool_check_passes_when_no_pending_files(self):
check = smoke.spool_check("files=0 bytes=208896 recent=0")
self.assertEqual(check.status, "pass")
self.assertIn("files=0", check.message)
def test_spool_check_fails_when_pending_or_recent_files_exist(self):
for output in ["files=2 bytes=300 recent=0", "files=0 bytes=208896 recent=1"]:
check = smoke.spool_check(output)
self.assertEqual(check.status, "fail")
def test_release_check_passes_when_current_release_has_all_binaries(self):
check = smoke.release_check(
"current=/opt/lingniu-go-native/releases/8def635 "
"gateway=1 history-writer=1 stat-writer=1 realtime-api=1"
)
self.assertEqual(check.status, "pass")
self.assertIn("8def635", check.message)
def test_release_check_fails_when_binary_is_missing(self):
check = smoke.release_check(
"current=/opt/lingniu-go-native/releases/8def635 "
"gateway=1 history-writer=0 stat-writer=1 realtime-api=1"
)
self.assertEqual(check.status, "fail")
self.assertIn("history-writer=0", check.message)
if __name__ == "__main__":
unittest.main()