44 Commits

Author SHA1 Message Date
lingniu
cbb6f3b741 perf: skip raw freshness count scan 2026-07-02 09:30:15 +08:00
lingniu
42de422935 test: check raw freshness without event date filter 2026-07-02 09:27:22 +08:00
lingniu
318486b1e9 fix: verify raw freshness by received time 2026-07-02 09:16:45 +08:00
lingniu
0a024f9a76 chore: wait for gateway spool before acceptance 2026-07-02 09:11:50 +08:00
lingniu
ea9d9f5d35 chore: add go native ecs deploy script 2026-07-02 08:50:19 +08:00
lingniu
409f55b5ae test: verify go native release layout 2026-07-02 08:47:05 +08:00
lingniu
8def635bf5 test: verify realtime snapshots are fresh 2026-07-02 08:43:27 +08:00
lingniu
c211447747 test: verify daily metric formulas 2026-07-02 08:38:32 +08:00
lingniu
1741aacaaa test: verify history rows link raw frames 2026-07-02 08:35:34 +08:00
lingniu
27f4ad3468 test: verify raw parsed json in production smoke 2026-07-02 02:00:50 +08:00
lingniu
601150b364 test: verify gateway spool is drained 2026-07-02 01:57:53 +08:00
lingniu
5b7f386c5a test: verify go services are enabled 2026-07-02 01:55:39 +08:00
lingniu
f4d9c9ef5a test: add go production acceptance smoke 2026-07-02 01:53:43 +08:00
lingniu
96514898eb test: add go systemd production smoke 2026-07-02 01:51:41 +08:00
lingniu
0ef88e1a4c test: add kafka production smoke 2026-07-02 01:49:02 +08:00
lingniu
0fd4d319f9 docs: record go kafka production topics 2026-07-02 01:44:27 +08:00
lingniu
4e8434c511 test: verify mqtt history tables in production smoke 2026-07-02 01:41:08 +08:00
lingniu
576644eb65 test: verify gb32960 history tables in production smoke 2026-07-02 01:39:45 +08:00
lingniu
9bd48c5781 test: require fresh raw data in production smoke 2026-07-02 01:38:16 +08:00
lingniu
2214abe9c5 test: include realtime in production smoke 2026-07-02 01:36:14 +08:00
lingniu
1477b997c9 test: add go native production smoke 2026-07-02 01:34:28 +08:00
lingniu
4ea8d18972 docs: document go native ecs deployment 2026-07-02 01:31:58 +08:00
lingniu
947e645090 fix: query history with utc tdengine filters 2026-07-02 01:28:45 +08:00
lingniu
78b805e3a2 fix: normalize timezone history filters 2026-07-02 01:24:51 +08:00
lingniu
a7b24f4c0b feat: expose realtime snapshot protocols 2026-07-02 01:23:03 +08:00
lingniu
eb124d929b fix: accept datetime-local history filters 2026-07-02 01:19:19 +08:00
lingniu
4711287655 fix: return empty arrays for query APIs 2026-07-02 01:16:48 +08:00
lingniu
2c224396c9 fix: return real pagination totals 2026-07-02 01:14:06 +08:00
lingniu
4f2cded760 fix: treat yutong mqtt mileage as meters 2026-07-02 01:08:59 +08:00
lingniu
7ad1a25e8e fix: normalize yutong mqtt mileage units 2026-07-02 01:04:14 +08:00
lingniu
3ee1d6f52b fix: stabilize yutong mqtt ingestion 2026-07-02 01:01:33 +08:00
lingniu
6d1d0aa85e feat: write protocol acknowledgements from go gateway 2026-07-02 00:54:31 +08:00
lingniu
055373c405 fix: support extended gb32960 platform login 2026-07-02 00:47:01 +08:00
lingniu
e37dd39c89 fix: enrich jt808 location additional parsing 2026-07-02 00:36:51 +08:00
lingniu
dd73fdbb5b feat: add vehicle mileage history query 2026-07-02 00:29:49 +08:00
lingniu
fb9ab8525a feat: add vehicle location history query 2026-07-02 00:27:20 +08:00
lingniu
cc89e0d537 fix: filter raw frames by vehicle key 2026-07-02 00:24:53 +08:00
lingniu
f3ecf430cc fix: cache realtime snapshots by vehicle key 2026-07-02 00:21:50 +08:00
lingniu
bcf7cdae2e fix: key daily metrics by vehicle identity 2026-07-02 00:18:35 +08:00
lingniu
9045b871d6 fix: align jt808 parser with protocol variants 2026-07-02 00:14:02 +08:00
lingniu
b0150a5503 docs: record yutong mqtt eof diagnosis 2026-07-02 00:04:23 +08:00
lingniu
8f3b337fd6 docs: add native go deployment runbook 2026-07-02 00:02:21 +08:00
lingniu
594ab2d9c7 feat: support yutong mqtt tls config 2026-07-01 23:50:30 +08:00
lingniu
229ffcf61f docs: record go production port verification 2026-07-01 23:46:33 +08:00
45 changed files with 5120 additions and 236 deletions

View File

@@ -44,11 +44,21 @@ services:
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt} YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt}
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-} YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-} YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-} IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-}
VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding} VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding}
ports: ports:
- "${GO_GB32960_TCP_PORT:-32960}:32960" - "${GO_GB32960_TCP_PORT:-32960}:32960"
- "${GO_JT808_TCP_PORT:-808}:808" - "${GO_JT808_TCP_PORT:-808}:808"
volumes:
- "${KAFKA_SPOOL_HOST_DIR:-/opt/lingniu-go/spool/gateway}:${KAFKA_SPOOL_DIR:-/data/spool/gateway}"
- "${YUTONG_MQTT_CERT_HOST_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:${YUTONG_MQTT_CERT_CONTAINER_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:ro"
go-history-writer: go-history-writer:
<<: *restart-policy <<: *restart-policy

66
deploy/systemd/README.md Normal file
View File

@@ -0,0 +1,66 @@
# Go Vehicle Gateway Native Deployment
本目录用于 ECS 原生部署。当前 goal 后续不再使用 Docker/Portainer 作为 Go 接入链路的部署方式。
## Runtime Layout
```text
/opt/lingniu-go-native/
current -> /opt/lingniu-go-native/releases/<git-short-sha>
releases/<git-short-sha>/
gateway
history-writer
stat-writer
realtime-api
env/
gateway.env
history-writer.env
stat-writer.env
realtime-api.env
spool/gateway/
```
## Services
| systemd unit | Binary | Purpose |
|---|---|---|
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP `32960`、JT808 TCP `808`、宇通 MQTT 接入,写 Kafka RAW/unified |
| `lingniu-go-history-writer.service` | `history-writer` | 消费 RAW topic写 TDengine `raw_frames` 和核心时序表 |
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 RAW topic写 MySQL `vehicle_daily_metric` |
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费 unified topic写 Redis并提供 realtime/raw/stat 查询 API |
## Build
在开发机或 CI 上构建 Linux amd64 二进制:
```bash
cd go/vehicle-gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/gateway ./cmd/gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/history-writer ./cmd/history-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/stat-writer ./cmd/stat-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/realtime-api ./cmd/realtime-api
```
## Cutover Notes
1. 先上传二进制到新 release 目录并更新 `current` symlink。
2. 从现有生产 env 生成四个 systemd 专用 env 文件;不要把密钥写入仓库。
3. 停止旧 Docker Go 容器或 Java 容器,释放 `808``32960``20210`
4. 执行:
```bash
systemctl daemon-reload
systemctl enable lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
```
## Verification
```bash
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=JT808&limit=1'
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=GB32960&limit=1'
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=YUTONG_MQTT&limit=1'
```

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Vehicle Gateway
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/gateway.env
ExecStart=/opt/lingniu-go-native/current/gateway
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go History Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/history-writer.env
ExecStart=/opt/lingniu-go-native/current/history-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Realtime API
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/realtime-api.env
ExecStart=/opt/lingniu-go-native/current/realtime-api
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Stat Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/stat-writer.env
ExecStart=/opt/lingniu-go-native/current/stat-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,227 @@
# 当前 ECS Go 原生部署说明
更新时间2026-07-02 01:30 CST
本文档记录当前 `lingniu-vehicle-ingest` 的生产运行面。当前 goal 的接入链路已经切到 Go 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。密钥只维护在 ECS 环境文件或受控凭据中,不写入 Git。
## 部署范围
| systemd 服务 | 二进制 | 说明 | 对外端口 |
| --- | --- | --- | --- |
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP、JT808 TCP、宇通 MQTT 接入,解析后写 Kafka RAW 和统一事件 | TCP `32960`、TCP `808` |
| `lingniu-go-history-writer.service` | `history-writer` | 消费 Kafka RAW写 TDengine RAW、位置点、里程点核心表 | 无 HTTP |
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 Kafka RAW按总里程差值法写 MySQL 每日指标 | 无 HTTP |
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费统一事件写 Redis并提供实时、历史、统计查询 API | HTTP `20210` |
信达 Push 已废弃,不参与当前 Go 生产链路。旧 Java/Docker 服务不应占用 `808``32960``20210`
## 应用 ECS
| 项 | 值 |
| --- | --- |
| 公网 IP | `115.29.187.205` |
| 登录用户 | `root` |
| 部署目录 | `/opt/lingniu-go-native` |
| 当前 release | `/opt/lingniu-go-native/current` |
| 环境文件目录 | `/opt/lingniu-go-native/env` |
| systemd unit 目录 | `/etc/systemd/system` |
| 对外服务 | `32960``808``20210` |
运行目录结构:
```text
/opt/lingniu-go-native/
current -> /opt/lingniu-go-native/releases/<git-short-sha>
releases/<git-short-sha>/
gateway
history-writer
stat-writer
realtime-api
env/
gateway.env
history-writer.env
stat-writer.env
realtime-api.env
spool/gateway/
```
## 中间件
| 组件 | 内网地址 | 公网地址 | 用途 |
| --- | --- | --- | --- |
| Kafka | `172.17.111.56:9092` | `114.55.58.251:9092` | RAW 和统一事件消息总线 |
| TDengine | `172.17.111.57:6041` | `115.29.185.82:6041` | RAW、位置、里程点时序热存储 |
| MySQL RDS | `rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306` | `rm-bp179zbv481rnw3e2no.mysql.rds.aliyuncs.com:3306` | 身份映射、JT808 注册、每日指标 |
| Redis RDS | `r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379` | 无 | 准实时车辆状态缓存 |
生产应用之间访问中间件优先使用内网地址。RDS 白名单需要允许应用 ECS 私网访问。
## Kafka Topic
| Topic | 生产者 | 消费者 | 内容 |
| --- | --- | --- | --- |
| `vehicle.raw.go.gb32960.v1` | `gateway` | `history-writer``stat-writer` | GB32960 完整 RAW 记录 |
| `vehicle.raw.go.jt808.v1` | `gateway` | `history-writer``stat-writer` | JT808 完整 RAW 记录 |
| `vehicle.raw.go.yutong-mqtt.v1` | `gateway` | `history-writer` | 宇通 MQTT 完整 RAW 记录 |
| `vehicle.event.go.unified.v1` | `gateway` | `realtime-api` | 三类协议统一事件,用于 Redis 实时状态 |
`gateway` 配置了 Kafka retry生产 env 中启用 `KAFKA_SPOOL_DIR` 时,本地 spool 可在 Kafka 短暂不可用后回放。
Kafka 消费组:
| Consumer Group | Topic | 当前验证 |
| --- | --- | --- |
| `go-history-writer` | 三个 `vehicle.raw.go.*` topic | 2026-07-02 验证 lag 为 `0` |
| `go-stat-writer` | `vehicle.raw.go.gb32960.v1``vehicle.raw.go.jt808.v1` | 2026-07-02 验证 lag 为 `0` |
| `go-realtime-api` | `vehicle.event.go.unified.v1` | 2026-07-02 验证 lag 为 `0` |
可重复 smoke
```bash
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
```
该脚本通过 SSH 登录 Kafka ECS检查 Go 生产 topic 是否存在,并校验 `go-history-writer``go-stat-writer``go-realtime-api` 的消费组 lag。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
## TDengine 数据层
默认数据库:`lingniu_vehicle_ts`
| 表 | 类型 | 说明 |
| --- | --- | --- |
| `raw_frames` | stable | 完整 RAW 帧,包含 `raw_hex``parsed_json``fields_json``source_endpoint` 和协议/车辆 tags |
| `vehicle_locations` | stable | 最小化位置历史,保留 `frame_id` 回链 RAW |
| `vehicle_mileage_points` | stable | 最小化总里程点,供历史查询和统计复核 |
设计原则:
- 完整结构化 payload 只落在 `raw_frames.parsed_json` / `fields_json`
- `vehicle_locations``vehicle_mileage_points` 只保留核心查询字段,不重复保存完整 JSON。
- `telemetry_fields` 不由当前服务维护,字段配置解析由独立子服务处理。
- API 传入东八区时间时,服务会转换为 TDengine 当前存储使用的 UTC 字面量再查询。
## MySQL 数据层
默认业务库:`lingniu_vehicle_data`
| 表 | 维护方 | 说明 |
| --- | --- | --- |
| `vehicle_identity_binding` | 人工/外部主数据 | VIN 映射主表,用 `phone``device_id``plate` 降级定位 VIN |
| `jt808_registration` | `gateway` 自动写入 | 仅 JT808 注册和鉴权记录,以 `phone` 作为主键,记录设备、车牌、厂家、鉴权、来源端点 |
| `vehicle_daily_metric` | `stat-writer` 自动写入 | 每日指标表,保存 `daily_mileage_km``daily_total_mileage_km` |
每日里程算法:
```text
daily_mileage_km = 当日最大 total_mileage_km - 当日最小 total_mileage_km
daily_total_mileage_km = 当日最大 total_mileage_km
```
统计按 `Asia/Shanghai` 自然日归档,当前支持 GB32960、JT808、宇通 MQTT 中能解析出 `total_mileage_km` 的数据。JT808 的总里程来自位置附加信息 `0x01`,单位按协议转换为 km宇通 MQTT 的 `TOTAL_MILEAGE` 原始单位为米,入库前转换为 km。
## Redis 实时层
`realtime-api` 消费 `vehicle.event.go.unified.v1`,按 VIN/vehicle key 维护准实时快照。主要能力:
| API | 说明 |
| --- | --- |
| `/api/realtime/vehicles/{vin}` | 查询车辆合并实时快照 |
| `/api/realtime/vehicles/{vin}/online` | 查询车辆是否在线 |
| `/api/realtime/vehicles/{vin}/protocols/{protocol}` | 查询指定协议的实时快照 |
Redis 只作为实时缓存;历史和统计以 TDengine/MySQL 为准。
## API 入口
公网基础地址:
```text
http://115.29.187.205:20210
```
常用查询:
```bash
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=YUTONG_MQTT&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/locations?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/mileage-points?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/stats/daily-metrics?dateFrom=2026-07-02&dateTo=2026-07-02&limit=20'
```
分页参数统一使用 `limit``offset`
部署后推荐直接运行 Go 原生生产 smoke
```bash
python3 tools/go_prod_acceptance.py --date 2026-07-02
```
该聚合脚本会串行执行 systemd/端口检查、Kafka topic/consumer lag 检查、HTTP 数据链路检查,任一子检查失败时整体退出码为非 0。需要 SSH 可登录应用 ECS 和 Kafka ECS。
也可以单独运行子检查:
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
```
`go_systemd_prod_smoke.py` 通过 SSH 检查四个 Go systemd 服务是否 `active``enabled`,并确认 `808``32960``gateway` 监听、`20210``realtime-api` 监听,同时要求 `/opt/lingniu-go-native/current` 指向有效 release、四个生产二进制均可执行、`/opt/lingniu-go-native/spool/gateway` 没有待回放文件,避免旧 Java/Docker 进程占用生产端口、部署目录错乱或 Kafka spool 静默堆积。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
`go_native_prod_smoke.py` 通过 `20210` HTTP API 验证 GB32960、JT808、宇通 MQTT 的 RAW 查询及结构化 `parsed_json`,三类协议的位置和里程点查询及 `frame_id` RAW 回链GB32960/JT808 的 `daily_mileage_km``daily_total_mileage_km` 统计公式,以及三类协议的 Redis realtime snapshot、online、protocol 查询。实时 snapshot/protocol 查询会校验 `updated_at_ms` 新鲜度;默认检查今天东八区数据时,还会要求三类 RAW 最新样本不超过 15 分钟,避免旧数据误判为接收正常;任一检查不达标时退出码为非 0。
## 部署命令
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收:
```bash
python3 tools/go_native_deploy.py --date 2026-07-02
```
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout``--spool-poll-interval` 调整。
脚本执行的发布动作等价于:
```bash
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/history-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/stat-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/realtime-api
scp release.tar.gz root@115.29.187.205:/tmp/
ln -sfn /opt/lingniu-go-native/releases/<git-short-sha> /opt/lingniu-go-native/current
systemctl daemon-reload
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
```
只更新查询 API 时,可以仅替换 `/opt/lingniu-go-native/current/realtime-api` 并重启:
```bash
systemctl restart lingniu-go-realtime-api.service
```
## 运行检查
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-history-writer --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-stat-writer --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-realtime-api --since '5 minutes ago' --no-pager
```
2026-07-02 01:30 CST 已验证:
- 四个 Go systemd 服务均为 `active``enabled`
- `/opt/lingniu-go-native/current` 指向有效 release四个生产二进制均存在且可执行。
- `gateway` 正在监听 `32960``808`
- `realtime-api` 正在监听 `20210`
- gateway Kafka spool 当前无待回放文件。
- GB32960、JT808、YUTONG_MQTT 的东八区 RAW 查询均可命中生产数据,且最新 RAW 样本包含结构化 `parsed_json`
- GB32960、JT808、YUTONG_MQTT 的最新 RAW 样本均在 15 分钟内。
- GB32960、JT808、YUTONG_MQTT 的 `vehicle_locations``vehicle_mileage_points` 查询均可命中生产数据,且样本包含 `frame_id` 回链 RAW。
- `vehicle_daily_metric` 可查到 `daily_mileage_km``daily_total_mileage_km`,且公式满足 `daily_mileage_km = latest_total_mileage_km - first_total_mileage_km``daily_total_mileage_km = latest_total_mileage_km`
- Redis realtime 可查到 GB32960、JT808、YUTONG_MQTT 的在线状态和协议快照,且 snapshot/protocol 快照最近刷新。

View File

@@ -657,3 +657,232 @@ http://115.29.187.205:20210/api/realtime/vehicles/LKLG7C4E3NA774736/online
``` ```
说明:本次发布重启窗口产生的 spool 正在回放下降,且最近 1 分钟没有新增gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。 说明:本次发布重启窗口产生的 spool 正在回放下降,且最近 1 分钟没有新增gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。
## 2026-07-01 23:50 生产端口接管与 RAW 查询 API 复验
部署版本:
- Git commit`5937132`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5937132-20260701234220`
生产端口状态:
```text
go-vehicle-gateway 0.0.0.0:808->808/tcp
go-vehicle-gateway 0.0.0.0:32960->32960/tcp
```
旧 Java 容器状态:
```text
gb32960-ingest-app restart=no status=exited
jt808-ingest-app restart=no status=exited
yutong-mqtt-app restart=no status=exited
vehicle-history-app restart=no status=exited
vehicle-analytics-app restart=no status=exited
vehicle-state-app restart=no status=exited
telemetry-field-parser-app restart=no status=exited
```
生产连接确认:
- JT808 端口 `808` 已收到外部连接:`222.66.200.68``115.231.168.135`
- GB32960 端口 `32960` 已收到外部连接:`8.134.95.166``117.160.0.65`
- ECS 当前没有 `8089` 监听。
RAW 查询 API
```text
GET /api/history/raw-frames
```
生产可访问样例:
```text
http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
```
接口复验结果:
```text
JT808_RAW:
total=1
protocol=JT808
vin=LKLG7C4E3NA774736
phone=013079963379
message_id=512
message_id_hex=0x0200
parse_status=OK
raw_hex_len=94
parsed_json_len=651
fields_json_len=186
GB32960_RAW:
total=1
protocol=GB32960
vin=LB9A32A21R0LS1707
message_id=2
message_id_hex=0x0002
parse_status=OK
raw_hex_len=1548
parsed_json_len=3698
fields_json_len=563
```
同轮回归验证:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
online=true
protocols=["JT808"]
GET /api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2026-07-01&limit=1
total=1
protocol=GB32960
metric_key=daily_mileage_km
calculation_method=TOTAL_MILEAGE_DIFF
```
说明:
- RAW 查询已返回 `raw_hex``parsed_json``fields_json`,可以用于查看完整解析结果。
- 日统计和实时查询在生产端口接管后仍可用。
## 2026-07-02 00:00 原生 systemd 部署与宇通 MQTT 复验
根据本次 goal 的新要求Go 接入链路后续不再使用 Docker 部署。本轮已中断 Docker 镜像构建路径,改为 ECS 原生 Linux 二进制 + systemd。
部署版本:
- Git commit`594ab2d`
- 部署目录:`/opt/lingniu-go-native`
- 当前 release`/opt/lingniu-go-native/releases/594ab2d`
- 当前指针:`/opt/lingniu-go-native/current -> /opt/lingniu-go-native/releases/594ab2d`
systemd 服务:
```text
lingniu-go-gateway.service active
lingniu-go-history-writer.service active
lingniu-go-stat-writer.service active
lingniu-go-realtime-api.service active
```
端口确认:
```text
[::]:32960 gateway
[::]:808 gateway
[::]:20210 realtime-api
```
Go Docker 容器已停止:
```text
go-vehicle-gateway Exited
go-history-writer Exited
go-stat-writer Exited
go-realtime-api Exited
```
宇通 MQTT
```text
yutong mqtt client started
mqtt subscribed broker=ssl://cpxlm.axxc.cn:38883 topic=/ytforward/shln/+ qos=1
```
说明MQTT broker 存在短连接 `EOF` 后自动重连现象,但订阅后已收到真实数据并落 RAW。
RAW 查询复验:
```text
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=3
total=3
protocol=YUTONG_MQTT
vin=LMRKH9AC1R1004131
vehicle_key=LMRKH9AC1R1004131
parse_status=OK
parsed_json_len=1104
fields_json_len=116
```
同轮回归:
```text
GET /api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
total=1
parse_status=OK
GET /api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
total=1
parse_status=OK
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
online=true
protocols=["JT808"]
```
本轮落地文件:
- `deploy/systemd/README.md`
- `deploy/systemd/lingniu-go-gateway.service`
- `deploy/systemd/lingniu-go-history-writer.service`
- `deploy/systemd/lingniu-go-stat-writer.service`
- `deploy/systemd/lingniu-go-realtime-api.service`
## 2026-07-02 00:04 宇通 MQTT EOF 诊断
现象:
Go 原生部署后,`lingniu-go-gateway` 日志中出现多次:
```text
mqtt connection lost error=EOF
mqtt connection lost error=write: broken pipe
mqtt subscribed topic=/ytforward/shln/+ qos=1
```
10 分钟窗口统计:
```text
mqtt subscribed = 33
mqtt connection lost = 32
```
对照旧 Java `yutong-mqtt-app` 历史日志,停止前也存在同类行为:
```text
mqtt endpoint [yutong] received topic=/ytforward/shln/1 bytes=574
mqtt endpoint [yutong] connection lost
Caused by: java.io.EOFException
mqtt endpoint [yutong] re-subscribed topic=/ytforward/shln/+ qos=1
mqtt endpoint [yutong] received topic=/ytforward/shln/3 bytes=586
```
判断:
- EOF/短连接不是 Go TLS 支持或 systemd 原生部署新引入的问题。
- 旧 Java 和新 Go 都表现为订阅、收到数据、broker 断开、自动重连。
- 当前链路可持续收到宇通 MQTT 数据并写入 TDengine RAW。
复验:
```text
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=1
total=1
protocol=YUTONG_MQTT
vin=LMRKH9ACXR1004094
parse_status=OK
parsed_json_len=1080
fields_json_len=115
```
运维建议:
- 单条 EOF 不作为不可用判断。
-`YUTONG_MQTT` RAW 入库增长、最近成功 `mqtt subscribed` 时间、服务 `systemctl is-active` 作为可用性判断。
- 如果 EOF 频率继续升高且 RAW 不再增长,再联系宇通侧确认 broker 长连接策略、同一 clientId 并发限制或网络出口策略。

View File

@@ -18,32 +18,40 @@ go build ./cmd/stat-writer
go build ./cmd/realtime-api go build ./cmd/realtime-api
``` ```
## 镜像构建 ## 生产原生部署验证
```bash ```bash
cd go/vehicle-gateway python3 tools/go_prod_acceptance.py --date 2026-07-02
docker build -t vehicle-gateway-go:local .
``` ```
同一个镜像包含四个二进制 该命令会聚合执行
- `/app/gateway` - `go_systemd_prod_smoke.py`:应用 ECS systemd 服务 active/enabled、端口归属和 gateway spool
- `/app/history-writer` - `go_kafka_prod_smoke.py`Kafka topic 和消费组 lag
- `/app/stat-writer` - `go_native_prod_smoke.py`HTTP API、RAW `parsed_json`、历史核心表 `frame_id` 回链、每日指标公式、Redis 实时新鲜度、TDengine/MySQL/Redis 数据链路
- `/app/realtime-api`
## Portainer 部署 如需单独检查 systemd
Go 旁路 stack 文件:
```bash ```bash
deploy/portainer/docker-compose-go.yml python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
``` ```
必填变量 脚本通过 SSH 检查
- `lingniu-go-gateway.service`
- `lingniu-go-history-writer.service`
- `lingniu-go-stat-writer.service`
- `lingniu-go-realtime-api.service`
- 上述服务均为 `active``enabled`
- `808``32960``gateway` 监听
- `20210``realtime-api` 监听
- `/opt/lingniu-go-native/spool/gateway` 无待回放文件
## 生产环境变量
当前 goal 的生产面使用 ECS 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。以下环境变量维护在 `/opt/lingniu-go-native/env/*.env`
```bash ```bash
LINGNIU_GO_IMAGE_VERSION=<image-version>
KAFKA_BROKERS=172.17.111.56:9092 KAFKA_BROKERS=172.17.111.56:9092
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
@@ -68,23 +76,44 @@ YUTONG_MQTT_PASSWORD=<password>
## ECS 进程检查 ## ECS 进程检查
```bash ```bash
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' \ systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
| egrep 'go-vehicle-gateway|go-history-writer|go-stat-writer|go-realtime-api|NAMES'
ss -lntp | egrep ':(808|32960|20210)\b' ss -lntp | egrep ':(808|32960|20210)\b'
``` ```
## Kafka 验证 ## Kafka 验证
优先使用 smoke 脚本做可重复检查:
```bash
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
```
脚本会检查 `vehicle.raw.go.gb32960.v1``vehicle.raw.go.jt808.v1``vehicle.raw.go.yutong-mqtt.v1``vehicle.event.go.unified.v1` 是否存在,并汇总 `go-history-writer``go-stat-writer``go-realtime-api` 的消费 lag。运行环境需要 SSH 免密或已建立可用的 SSH 认证方式。
手工核对命令:
```bash ```bash
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \ kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.jt808.v1 --max-messages 1 --timeout-ms 10000 --topic vehicle.raw.go.jt808.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \ kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.gb32960.v1 --max-messages 1 --timeout-ms 10000 --topic vehicle.raw.go.gb32960.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \ kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.event.unified.v1 --max-messages 1 --timeout-ms 10000 --topic vehicle.raw.go.yutong-mqtt.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.event.go.unified.v1 --max-messages 1 --timeout-ms 10000
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-history-writer
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-stat-writer
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-realtime-api
``` ```
## TDengine 验证 ## TDengine 验证

View File

@@ -46,12 +46,14 @@ func main() {
Addr: env("GB32960_TCP_ADDR", ":32960"), Addr: env("GB32960_TCP_ADDR", ":32960"),
Extract: gb32960.ExtractFrames, Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame, Parse: gb32960.ParseFrame,
Respond: gb32960.AutoResponse,
}, },
{ {
Protocol: envelope.ProtocolJT808, Protocol: envelope.ProtocolJT808,
Addr: env("JT808_TCP_ADDR", ":808"), Addr: env("JT808_TCP_ADDR", ":808"),
Extract: jt808.ExtractFrames, Extract: jt808.ExtractFrames,
Parse: jt808.ParseFrame, Parse: jt808.ParseFrame,
Respond: jt808.NewAutoResponder(env("JT808_REGISTER_AUTH_CODE", "g7gps")).Respond,
}, },
} }
@@ -81,16 +83,23 @@ func main() {
} }
if envBool("YUTONG_MQTT_ENABLED", false) { if envBool("YUTONG_MQTT_ENABLED", false) {
client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{ client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{
EndpointName: env("YUTONG_MQTT_ENDPOINT", "yutong"), EndpointName: env("YUTONG_MQTT_ENDPOINT", env("YUTONG_MQTT_ENDPOINT_NAME", "yutong")),
Broker: env("YUTONG_MQTT_URI", ""), Broker: env("YUTONG_MQTT_URI", ""),
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"), ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
Username: env("YUTONG_MQTT_USERNAME", ""), Username: env("YUTONG_MQTT_USERNAME", ""),
Password: env("YUTONG_MQTT_PASSWORD", ""), Password: env("YUTONG_MQTT_PASSWORD", ""),
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))), Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)), QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
Sink: sink, CleanSession: envBool("YUTONG_MQTT_CLEAN_SESSION", false),
Resolver: resolver, KeepAlive: time.Duration(envInt("YUTONG_MQTT_KEEP_ALIVE_SECONDS", 20)) * time.Second,
Logger: logger, ConnectTimeout: time.Duration(envInt("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", 10)) * time.Second,
TLSCACertPath: env("YUTONG_MQTT_TLS_CA_PEM", ""),
TLSClientCertPath: env("YUTONG_MQTT_TLS_CLIENT_PEM", ""),
TLSClientKeyPath: env("YUTONG_MQTT_TLS_CLIENT_KEY", ""),
TLSHostnameVerification: envBool("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", true),
Sink: sink,
Resolver: resolver,
Logger: logger,
}) })
if err != nil { if err != nil {
logger.Error("build yutong mqtt client failed", "error", err) logger.Error("build yutong mqtt client failed", "error", err)

View File

@@ -92,14 +92,19 @@ func main() {
closeHistory = func() { _ = db.Close() } closeHistory = func() { _ = db.Close() }
database := env("TDENGINE_DATABASE", history.DefaultDatabase) database := env("TDENGINE_DATABASE", history.DefaultDatabase)
mux.Handle("/api/history/raw-frames", history.NewRawFrameHandler(history.NewRawFrameRepository(db, database))) mux.Handle("/api/history/raw-frames", history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)))
logger.Info("history raw frame query enabled", "driver", driver, "database", database) mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database)))
mux.Handle("/api/history/mileage-points", history.NewMileagePointHandler(history.NewMileagePointRepository(db, database)))
logger.Info("history query enabled", "driver", driver, "database", database)
} else { } else {
mux.HandleFunc("/api/history/raw-frames", func(w http.ResponseWriter, _ *http.Request) { historyUnavailable := func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable) w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"}) _ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"})
}) }
logger.Warn("TDENGINE_DSN is empty; raw frame query api disabled") mux.HandleFunc("/api/history/raw-frames", historyUnavailable)
mux.HandleFunc("/api/history/locations", historyUnavailable)
mux.HandleFunc("/api/history/mileage-points", historyUnavailable)
logger.Warn("TDENGINE_DSN is empty; history query api disabled")
} }
defer closeHistory() defer closeHistory()

View File

@@ -2,9 +2,13 @@ package gateway
import ( import (
"context" "context"
"crypto/tls"
"crypto/x509"
"encoding/hex" "encoding/hex"
"errors" "errors"
"fmt"
"log/slog" "log/slog"
"os"
"strings" "strings"
"time" "time"
@@ -17,16 +21,23 @@ import (
) )
type MQTTClientConfig struct { type MQTTClientConfig struct {
EndpointName string EndpointName string
Broker string Broker string
ClientID string ClientID string
Username string Username string
Password string Password string
Topics []string Topics []string
QoS byte QoS byte
Sink eventbus.Sink CleanSession bool
Resolver identity.Resolver KeepAlive time.Duration
Logger *slog.Logger ConnectTimeout time.Duration
TLSCACertPath string
TLSClientCertPath string
TLSClientKeyPath string
TLSHostnameVerification bool
Sink eventbus.Sink
Resolver identity.Resolver
Logger *slog.Logger
} }
type MQTTClient struct { type MQTTClient struct {
@@ -60,17 +71,47 @@ func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
} }
func (c *MQTTClient) Start(ctx context.Context) error { func (c *MQTTClient) Start(ctx context.Context) error {
opts, err := c.buildOptions(ctx)
if err != nil {
return err
}
c.client = mqtt.NewClient(opts)
token := c.client.Connect()
if token.Wait() && token.Error() != nil {
return token.Error()
}
go func() {
<-ctx.Done()
if c.client != nil && c.client.IsConnected() {
c.client.Disconnect(250)
}
}()
return nil
}
func (c *MQTTClient) buildOptions(ctx context.Context) (*mqtt.ClientOptions, error) {
keepAlive := c.cfg.KeepAlive
if keepAlive <= 0 {
keepAlive = 20 * time.Second
}
connectTimeout := c.cfg.ConnectTimeout
if connectTimeout <= 0 {
connectTimeout = 10 * time.Second
}
opts := mqtt.NewClientOptions(). opts := mqtt.NewClientOptions().
AddBroker(c.cfg.Broker). AddBroker(c.cfg.Broker).
SetClientID(c.cfg.ClientID). SetClientID(c.cfg.ClientID).
SetUsername(c.cfg.Username). SetUsername(c.cfg.Username).
SetPassword(c.cfg.Password). SetPassword(c.cfg.Password).
SetCleanSession(false). SetCleanSession(c.cfg.CleanSession).
SetAutoReconnect(true). SetAutoReconnect(true).
SetConnectRetry(true). SetConnectRetry(true).
SetConnectRetryInterval(5 * time.Second). SetConnectRetryInterval(5 * time.Second).
SetKeepAlive(20 * time.Second). SetResumeSubs(true).
SetConnectTimeout(10 * time.Second) SetOrderMatters(false).
SetKeepAlive(keepAlive).
SetConnectTimeout(connectTimeout)
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) { opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
c.handleMessage(ctx, message.Topic(), message.Payload()) c.handleMessage(ctx, message.Topic(), message.Payload())
@@ -89,18 +130,49 @@ func (c *MQTTClient) Start(ctx context.Context) error {
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err) c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
} }
c.client = mqtt.NewClient(opts) tlsConfig, err := c.buildTLSConfig()
token := c.client.Connect() if err != nil {
if token.Wait() && token.Error() != nil { return nil, err
return token.Error()
} }
go func() { if tlsConfig != nil {
<-ctx.Done() opts.SetTLSConfig(tlsConfig)
if c.client != nil && c.client.IsConnected() { }
c.client.Disconnect(250) return opts, nil
}
func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
caPath := strings.TrimSpace(c.cfg.TLSCACertPath)
certPath := strings.TrimSpace(c.cfg.TLSClientCertPath)
keyPath := strings.TrimSpace(c.cfg.TLSClientKeyPath)
if caPath == "" && certPath == "" && keyPath == "" {
return nil, nil
}
config := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: !c.cfg.TLSHostnameVerification,
}
if caPath != "" {
caPEM, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("read mqtt ca certificate: %w", err)
} }
}() roots := x509.NewCertPool()
return nil if !roots.AppendCertsFromPEM(caPEM) {
return nil, fmt.Errorf("parse mqtt ca certificate %s", caPath)
}
config.RootCAs = roots
}
if certPath != "" || keyPath != "" {
if certPath == "" || keyPath == "" {
return nil, errors.New("mqtt client certificate and key must be configured together")
}
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, fmt.Errorf("load mqtt client certificate: %w", err)
}
config.Certificates = []tls.Certificate{cert}
}
return config, nil
} }
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) { func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {

View File

@@ -2,8 +2,17 @@ package gateway
import ( import (
"context" "context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"log/slog" "log/slog"
"math/big"
"os"
"path/filepath"
"testing" "testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
) )
@@ -59,3 +68,116 @@ func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus) t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
} }
} }
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
dir := t.TempDir()
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "ssl://mqtt.example.test:8883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
QoS: 1,
Sink: &recordingSink{},
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
TLSCACertPath: caPath,
TLSClientCertPath: certPath,
TLSClientKeyPath: keyPath,
TLSHostnameVerification: false,
CleanSession: true,
KeepAlive: 20 * time.Second,
ConnectTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
opts, err := client.buildOptions(context.Background())
if err != nil {
t.Fatalf("buildOptions() error = %v", err)
}
if opts.TLSConfig == nil {
t.Fatal("TLSConfig is nil")
}
if opts.TLSConfig.RootCAs == nil {
t.Fatal("RootCAs is nil")
}
if len(opts.TLSConfig.Certificates) != 1 {
t.Fatalf("client certificates = %d, want 1", len(opts.TLSConfig.Certificates))
}
if !opts.TLSConfig.InsecureSkipVerify {
t.Fatal("InsecureSkipVerify should be true when hostname verification is disabled")
}
if !opts.CleanSession {
t.Fatal("CleanSession should be true")
}
if opts.Order {
t.Fatal("OrderMatters should be false so MQTT network handling is not blocked by Kafka/DB work")
}
if !opts.ResumeSubs {
t.Fatal("ResumeSubs should be true to avoid subscribe failures during reconnect churn")
}
if got := opts.KeepAlive; got != 20 {
t.Fatalf("KeepAlive = %d, want 20", got)
}
if got := opts.ConnectTimeout; got != 10*time.Second {
t.Fatalf("ConnectTimeout = %v, want 10s", got)
}
}
func writeTestTLSMaterial(t *testing.T, dir string) (string, string, string) {
t.Helper()
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate ca key: %v", err)
}
caTemplate := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "test-ca"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
BasicConstraintsValid: true,
IsCA: true,
}
caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
if err != nil {
t.Fatalf("create ca cert: %v", err)
}
clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate client key: %v", err)
}
clientTemplate := &x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{CommonName: "test-client"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey)
if err != nil {
t.Fatalf("create client cert: %v", err)
}
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "client.pem")
keyPath := filepath.Join(dir, "client-key.pem")
writePEM(t, caPath, "CERTIFICATE", caDER)
writePEM(t, certPath, "CERTIFICATE", clientDER)
keyDER := x509.MarshalPKCS1PrivateKey(clientKey)
writePEM(t, keyPath, "RSA PRIVATE KEY", keyDER)
return caPath, certPath, keyPath
}
func writePEM(t *testing.T, path, typ string, der []byte) {
t.Helper()
file, err := os.Create(path)
if err != nil {
t.Fatalf("create %s: %v", path, err)
}
defer file.Close()
if err := pem.Encode(file, &pem.Block{Type: typ, Bytes: der}); err != nil {
t.Fatalf("write pem %s: %v", path, err)
}
}

View File

@@ -21,11 +21,14 @@ type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
type TCPProtocol struct { type TCPProtocol struct {
Protocol envelope.Protocol Protocol envelope.Protocol
Addr string Addr string
Extract FrameExtractor Extract FrameExtractor
Parse FrameParser Parse FrameParser
Respond FrameResponder
} }
type TCPServer struct { type TCPServer struct {
@@ -153,7 +156,7 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
} }
pending = remainder pending = remainder
for _, frame := range frames { for _, frame := range frames {
s.handleFrame(ctx, frame, source) s.handleFrame(ctx, conn, frame, source)
} }
} }
if err != nil { if err != nil {
@@ -174,7 +177,7 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
} }
} }
func (s *TCPServer) handleFrame(ctx context.Context, raw []byte, source string) { func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string) {
receivedAtMS := time.Now().UnixMilli() receivedAtMS := time.Now().UnixMilli()
env, err := s.protocol.Parse(raw, receivedAtMS, source) env, err := s.protocol.Parse(raw, receivedAtMS, source)
if err != nil { if err != nil {
@@ -213,6 +216,21 @@ func (s *TCPServer) handleFrame(ctx context.Context, raw []byte, source string)
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return return
} }
if s.protocol.Respond == nil {
return
}
response, ok, err := s.protocol.Respond(raw, env)
if err != nil {
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if !ok || len(response) == 0 {
return
}
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := conn.Write(response); err != nil {
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
}
} }
func (p TCPProtocol) String() string { func (p TCPProtocol) String() string {

View File

@@ -3,6 +3,7 @@ package gateway
import ( import (
"context" "context"
"encoding/hex" "encoding/hex"
"io"
"log/slog" "log/slog"
"net" "net"
"testing" "testing"
@@ -73,6 +74,37 @@ func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
} }
} }
func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
frame := buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil)
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolGB32960,
Addr: ":0",
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
return []byte("ACK"), true, nil
},
}, sink)
client, done := runPipe(t, server)
if _, err := client.Write(frame); err != nil {
t.Fatalf("client.Write() error = %v", err)
}
buf := make([]byte, 3)
if _, err := io.ReadFull(client, buf); err != nil {
t.Fatalf("read response error = %v", err)
}
if string(buf) != "ACK" {
t.Fatalf("response = %q", string(buf))
}
_ = client.Close()
<-done
}
func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer { func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer {
t.Helper() t.Helper()
server, err := NewTCPServer(TCPServerConfig{ server, err := NewTCPServer(TCPServerConfig{

View File

@@ -17,15 +17,18 @@ type Queryer interface {
} }
type RawFrameQuery struct { type RawFrameQuery struct {
Protocol string Protocol string
VIN string VehicleKey string
Phone string VIN string
DeviceID string Phone string
MessageID string DeviceID string
DateFrom string MessageID string
DateTo string OrderBy string
Limit int IncludeTotal bool
Offset int DateFrom string
DateTo string
Limit int
Offset int
} }
type RawFrameRow struct { type RawFrameRow struct {
@@ -51,6 +54,66 @@ type RawFrameRow struct {
DeviceID string `json:"device_id,omitempty"` DeviceID string `json:"device_id,omitempty"`
} }
type LocationQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
DateFrom string
DateTo string
Limit int
Offset int
}
type LocationRow struct {
TS string `json:"ts"`
EventID string `json:"event_id"`
FrameID string `json:"frame_id"`
ReceivedAt string `json:"received_at"`
Longitude float64 `json:"longitude"`
Latitude float64 `json:"latitude"`
AltitudeM *float64 `json:"altitude_m,omitempty"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
DirectionDeg *int64 `json:"direction_deg,omitempty"`
AlarmFlag *int64 `json:"alarm_flag,omitempty"`
StatusFlag *int64 `json:"status_flag,omitempty"`
TotalMileageKM *float64 `json:"total_mileage_km,omitempty"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type MileagePointQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
DateFrom string
DateTo string
Limit int
Offset int
}
type MileagePointRow struct {
TS string `json:"ts"`
EventID string `json:"event_id"`
FrameID string `json:"frame_id"`
ReceivedAt string `json:"received_at"`
TotalMileageKM float64 `json:"total_mileage_km"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type RawFrameRepository struct { type RawFrameRepository struct {
db Queryer db Queryer
database string database string
@@ -67,6 +130,38 @@ func NewRawFrameRepository(db Queryer, database string) *RawFrameRepository {
return &RawFrameRepository{db: db, database: database} return &RawFrameRepository{db: db, database: database}
} }
type LocationRepository struct {
db Queryer
database string
}
type MileagePointRepository struct {
db Queryer
database string
}
func NewLocationRepository(db Queryer, database string) *LocationRepository {
if db == nil {
panic("location query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &LocationRepository{db: db, database: database}
}
func NewMileagePointRepository(db Queryer, database string) *MileagePointRepository {
if db == nil {
panic("mileage point query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &MileagePointRepository{db: db, database: database}
}
func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) { func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) {
query = normalizeRawFrameQuery(query) query = normalizeRawFrameQuery(query)
sqlText, args := buildRawFrameSQL(r.tableName(), query) sqlText, args := buildRawFrameSQL(r.tableName(), query)
@@ -76,7 +171,7 @@ func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]
} }
defer rows.Close() defer rows.Close()
var out []RawFrameRow out := make([]RawFrameRow, 0)
for rows.Next() { for rows.Next() {
var row RawFrameRow var row RawFrameRow
var ts scanDateTime var ts scanDateTime
@@ -114,6 +209,137 @@ func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]
return out, rows.Err() return out, rows.Err()
} }
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
query = normalizeRawFrameQuery(query)
sqlText, args := buildRawFrameCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]LocationRow, error) {
query = normalizeLocationQuery(query)
sqlText, args := buildLocationSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]LocationRow, 0)
for rows.Next() {
var row LocationRow
var ts scanDateTime
var receivedAt scanDateTime
var altitude sql.NullFloat64
var speed sql.NullFloat64
var direction sql.NullInt64
var alarm sql.NullInt64
var status sql.NullInt64
var mileage sql.NullFloat64
if err := rows.Scan(
&ts,
&row.EventID,
&row.FrameID,
&receivedAt,
&row.Longitude,
&row.Latitude,
&altitude,
&speed,
&direction,
&alarm,
&status,
&mileage,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.ReceivedAt = receivedAt.String
row.AltitudeM = nullableFloat(altitude)
row.SpeedKMH = nullableFloat(speed)
row.DirectionDeg = nullableInt(direction)
row.AlarmFlag = nullableInt(alarm)
row.StatusFlag = nullableInt(status)
row.TotalMileageKM = nullableFloat(mileage)
out = append(out, row)
}
return out, rows.Err()
}
func (r *LocationRepository) Count(ctx context.Context, query LocationQuery) (int64, error) {
query = normalizeLocationQuery(query)
sqlText, args := buildLocationCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func (r *MileagePointRepository) Query(ctx context.Context, query MileagePointQuery) ([]MileagePointRow, error) {
query = normalizeMileagePointQuery(query)
sqlText, args := buildMileagePointSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]MileagePointRow, 0)
for rows.Next() {
var row MileagePointRow
var ts scanDateTime
var receivedAt scanDateTime
var speed sql.NullFloat64
var longitude sql.NullFloat64
var latitude sql.NullFloat64
if err := rows.Scan(
&ts,
&row.EventID,
&row.FrameID,
&receivedAt,
&row.TotalMileageKM,
&speed,
&longitude,
&latitude,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.ReceivedAt = receivedAt.String
row.SpeedKMH = nullableFloat(speed)
row.Longitude = nullableFloat(longitude)
row.Latitude = nullableFloat(latitude)
out = append(out, row)
}
return out, rows.Err()
}
func (r *MileagePointRepository) Count(ctx context.Context, query MileagePointQuery) (int64, error) {
query = normalizeMileagePointQuery(query)
sqlText, args := buildMileagePointCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func countRows(ctx context.Context, db Queryer, sqlText string, args ...any) (int64, error) {
rows, err := db.QueryContext(ctx, sqlText, args...)
if err != nil {
return 0, err
}
defer rows.Close()
var total int64
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, rows.Err()
}
func (r *RawFrameRepository) tableName() string { func (r *RawFrameRepository) tableName() string {
if r.database == "" { if r.database == "" {
return "raw_frames" return "raw_frames"
@@ -121,14 +347,58 @@ func (r *RawFrameRepository) tableName() string {
return r.database + ".raw_frames" return r.database + ".raw_frames"
} }
func (r *LocationRepository) tableName() string {
if r.database == "" {
return "vehicle_locations"
}
return r.database + ".vehicle_locations"
}
func (r *MileagePointRepository) tableName() string {
if r.database == "" {
return "vehicle_mileage_points"
}
return r.database + ".vehicle_mileage_points"
}
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery { func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN) query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone) query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID) query.DeviceID = strings.TrimSpace(query.DeviceID)
query.MessageID = strings.TrimSpace(query.MessageID) query.MessageID = strings.TrimSpace(query.MessageID)
query.DateFrom = strings.TrimSpace(query.DateFrom) query.OrderBy = normalizeRawFrameOrderBy(query.OrderBy)
query.DateTo = strings.TrimSpace(query.DateTo) query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func normalizeLocationQuery(query LocationQuery) LocationQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func normalizeMileagePointQuery(query MileagePointQuery) MileagePointQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 { if query.Limit <= 0 {
query.Limit = 20 query.Limit = 20
} }
@@ -136,6 +406,60 @@ func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
} }
func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) { func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
where := rawFrameWhere(query)
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY " + rawFrameOrderColumn(query.OrderBy) + " DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := rawFrameWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
where := locationWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildLocationCountSQL(table string, query LocationQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := locationWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func buildMileagePointSQL(table string, query MileagePointQuery) (string, []any) {
where := mileagePointWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildMileagePointCountSQL(table string, query MileagePointQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := mileagePointWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func rawFrameWhere(query RawFrameQuery) []string {
var where []string var where []string
add := func(clause string) { add := func(clause string) {
where = append(where, clause) where = append(where, clause)
@@ -143,6 +467,9 @@ func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
if query.Protocol != "" { if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'") add("protocol = '" + quote(query.Protocol) + "'")
} }
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" { if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'") add("vin = '" + quote(query.VIN) + "'")
} }
@@ -158,23 +485,101 @@ func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
} }
} }
if query.DateFrom != "" { if query.DateFrom != "" {
add("ts >= '" + quote(query.DateFrom) + "'") add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
} }
if query.DateTo != "" { if query.DateTo != "" {
add("ts <= '" + quote(query.DateTo) + "'") add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
} }
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table return where
if len(where) > 0 { }
sqlText += " WHERE " + strings.Join(where, " AND ")
func normalizeRawFrameOrderBy(value string) string {
value = strings.ToLower(strings.TrimSpace(value))
switch value {
case "receivedat", "received_at":
return "receivedAt"
default:
return "ts"
} }
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset) }
return sqlText, nil
func rawFrameOrderColumn(value string) string {
if normalizeRawFrameOrderBy(value) == "receivedAt" {
return "received_at"
}
return "ts"
}
func locationWhere(query LocationQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
func mileagePointWhere(query MileagePointQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
} }
type RawFrameHandler struct { type RawFrameHandler struct {
repository *RawFrameRepository repository *RawFrameRepository
} }
type LocationHandler struct {
repository *LocationRepository
}
type MileagePointHandler struct {
repository *MileagePointRepository
}
func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler { func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler {
if repository == nil { if repository == nil {
panic("raw frame repository must not be nil") panic("raw frame repository must not be nil")
@@ -182,6 +587,20 @@ func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler {
return &RawFrameHandler{repository: repository} return &RawFrameHandler{repository: repository}
} }
func NewLocationHandler(repository *LocationRepository) *LocationHandler {
if repository == nil {
panic("location repository must not be nil")
}
return &LocationHandler{repository: repository}
}
func NewMileagePointHandler(repository *MileagePointRepository) *MileagePointHandler {
if repository == nil {
panic("mileage point repository must not be nil")
}
return &MileagePointHandler{repository: repository}
}
func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet { if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed") writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
@@ -196,6 +615,50 @@ func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeHistoryError(w, http.StatusBadRequest, err.Error()) writeHistoryError(w, http.StatusBadRequest, err.Error())
return return
} }
var total int64
if query.IncludeTotal {
total, err = h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
if !query.IncludeTotal {
total = int64(len(rows))
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/locations" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseLocationQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query) rows, err := h.repository.Query(r.Context(), query)
if err != nil { if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error()) writeHistoryError(w, http.StatusInternalServerError, err.Error())
@@ -204,7 +667,40 @@ func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{ _ = json.NewEncoder(w).Encode(map[string]any{
"items": rows, "items": rows,
"total": len(rows), "total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func (h *MileagePointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/mileage-points" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseMileagePointQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit, "limit": query.Limit,
"offset": query.Offset, "offset": query.Offset,
}) })
@@ -221,15 +717,18 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
return RawFrameQuery{}, err return RawFrameQuery{}, err
} }
query := RawFrameQuery{ query := RawFrameQuery{
Protocol: values.Get("protocol"), Protocol: values.Get("protocol"),
VIN: values.Get("vin"), VehicleKey: values.Get("vehicleKey"),
Phone: values.Get("phone"), VIN: values.Get("vin"),
DeviceID: values.Get("deviceId"), Phone: values.Get("phone"),
MessageID: values.Get("messageId"), DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"), MessageID: values.Get("messageId"),
DateTo: values.Get("dateTo"), OrderBy: values.Get("orderBy"),
Limit: limit, IncludeTotal: values.Get("includeTotal") != "false",
Offset: offset, DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
} }
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) { if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss") return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
@@ -242,6 +741,60 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
return normalizeRawFrameQuery(query), nil return normalizeRawFrameQuery(query), nil
} }
func parseMileagePointQuery(r *http.Request) (MileagePointQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return MileagePointQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return MileagePointQuery{}, err
}
query := MileagePointQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return MileagePointQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
return normalizeMileagePointQuery(query), nil
}
func parseLocationQuery(r *http.Request) (LocationQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return LocationQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return LocationQuery{}, err
}
query := LocationQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
return normalizeLocationQuery(query), nil
}
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) { func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
raw = strings.TrimSpace(raw) raw = strings.TrimSpace(raw)
if raw == "" { if raw == "" {
@@ -268,7 +821,7 @@ func parseMessageID(value string) (int64, bool) {
} }
func validDateTime(value string) bool { func validDateTime(value string) bool {
value = strings.TrimSpace(value) value = normalizeDateTimeLiteral(value)
if value == "" { if value == "" {
return true return true
} }
@@ -280,6 +833,23 @@ func validDateTime(value string) bool {
return false return false
} }
func normalizeDateTimeLiteral(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} {
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
}
}
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
}
return value
}
func safeIdentifier(value string) bool { func safeIdentifier(value string) bool {
if value == "" { if value == "" {
return false return false
@@ -317,6 +887,20 @@ func formatSQLTime(value any, layout string) string {
} }
} }
func nullableFloat(value sql.NullFloat64) *float64 {
if !value.Valid {
return nil
}
return &value.Float64
}
func nullableInt(value sql.NullInt64) *int64 {
if !value.Valid {
return nil
}
return &value.Int64
}
func writeHistoryError(w http.ResponseWriter, status int, message string) { func writeHistoryError(w http.ResponseWriter, status int, message string) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status) w.WriteHeader(status)

View File

@@ -63,6 +63,8 @@ func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(38))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames"). mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes", "ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
@@ -84,7 +86,192 @@ func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
} }
body := response.Body.String() body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"message_id_hex":"0x0002"`, `"total":1`} { for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"message_id_hex":"0x0002"`, `"total":38`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(11))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "go_frame", "event-3", 0x0200, "2026-07-02 00:18:22", "2026-07-02 00:22:43",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"total_mileage_km":8792.8}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"phone":"013307811350"`, `"total":11`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=NOT_EXISTS&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
t.Fatalf("expected empty items array, got: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestLocationHandlerReturnsLocationsByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(17))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
"JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
8792.8, 8.0, 121.07764, 30.585928,
"JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewMileagePointHandler(NewMileagePointRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/mileage-points?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"total_mileage_km":8792.8`, `"speed_kmh":8`, `"total":19`} {
if !strings.Contains(body, want) { if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body) t.Fatalf("response missing %s: %s", want, body)
} }
@@ -106,6 +293,31 @@ func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
} }
} }
func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) {
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?dateFrom=2026-07-01T00:00:00&dateTo=2026-07-02T00:00:00", nil)
query, err := parseRawFrameQuery(request)
if err != nil {
t.Fatalf("parseRawFrameQuery() error = %v", err)
}
if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" {
t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo)
}
}
func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) {
for raw, want := range map[string]string{
"2026-07-01T00:00:00": "2026-06-30 16:00:00",
"2026-07-01 00:00:00": "2026-06-30 16:00:00",
"2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00",
"2026-06-30T16:00:00Z": "2026-06-30 16:00:00",
} {
if got := normalizeDateTimeLiteral(raw); got != want {
t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want)
}
}
}
func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) { func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
for raw, want := range map[string]int64{ for raw, want := range map[string]int64{
"512": 512, "512": 512,
@@ -119,23 +331,22 @@ func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
} }
} }
func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) { func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{ sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{
Protocol: "JT808", Protocol: "JT808",
VIN: "VIN'1", VehicleKey: "JT808:013307811350",
MessageID: "0x0200", DateFrom: "2026-07-02 00:00:00",
DateFrom: "2026-07-01 00:00:00", DateTo: "2026-07-02 23:59:59",
DateTo: "2026-07-01 23:59:59", Limit: 20,
Limit: 20, Offset: 5,
Offset: 5,
}) })
if len(args) != 0 { if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args) t.Fatalf("expected no query args for TDengine, got %#v", args)
} }
for _, want := range []string{ for _, want := range []string{
"protocol = 'JT808'", "protocol = 'JT808'",
"vin = 'VIN''1'", "vehicle_key = 'JT808:013307811350'",
"message_id = 512", "ts >= '2026-07-01 16:00:00'",
"LIMIT 20 OFFSET 5", "LIMIT 20 OFFSET 5",
} { } {
if !strings.Contains(sqlText, want) { if !strings.Contains(sqlText, want) {
@@ -143,3 +354,70 @@ func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
} }
} }
} }
func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildLocationSQL("lingniu_vehicle_ts.vehicle_locations", LocationQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
DateFrom: "2026-07-02 00:00:00",
DateTo: "2026-07-02 23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"ts >= '2026-07-01 16:00:00'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
VIN: "VIN'1",
MessageID: "0x0200",
DateFrom: "2026-07-01T00:00:00",
DateTo: "2026-07-01T23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"vin = 'VIN''1'",
"message_id = 512",
"ts >= '2026-06-30 16:00:00'",
"ts <= '2026-07-01 15:59:59'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildRawFrameSQLCanOrderByReceivedAt(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
Protocol: "JT808",
OrderBy: "receivedAt",
Limit: 1,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
if !strings.Contains(sqlText, "ORDER BY received_at DESC LIMIT 1 OFFSET 0") {
t.Fatalf("sql should order by received_at: %s", sqlText)
}
}

View File

@@ -133,7 +133,7 @@ func rawValues(env envelope.FrameEnvelope) []any {
messageIDInt(env.MessageID), messageIDInt(env.MessageID),
eventTime, eventTime,
received, received,
rawSize(env.RawHex), rawSizeBytes(env),
env.RawHex, env.RawHex,
env.RawText, env.RawText,
jsonString(env.Parsed), jsonString(env.Parsed),
@@ -200,12 +200,15 @@ func messageIDInt(value string) int64 {
return parsed return parsed
} }
func rawSize(rawHex string) int { func rawSizeBytes(env envelope.FrameEnvelope) int {
rawHex = strings.TrimSpace(rawHex) rawHex := strings.TrimSpace(env.RawHex)
if len(rawHex)%2 != 0 { if len(rawHex)%2 != 0 {
return len(rawHex) / 2 return len(rawHex) / 2
} }
return len(rawHex) / 2 if rawHex != "" {
return len(rawHex) / 2
}
return len([]byte(env.RawText))
} }
func jsonString(value any) string { func jsonString(value any) string {

View File

@@ -75,6 +75,18 @@ func TestWriterSkipsSparseDerivedRows(t *testing.T) {
} }
} }
func TestRawSizeBytesUsesRawTextWhenHexIsEmpty(t *testing.T) {
env := envelope.FrameEnvelope{RawText: `{"code":"0F80","data":{"speed":12}}`}
if got, want := rawSizeBytes(env), len([]byte(env.RawText)); got != want {
t.Fatalf("raw text size = %d, want %d", got, want)
}
env.RawHex = "7E0200"
if got, want := rawSizeBytes(env), 3; got != want {
t.Fatalf("raw hex size = %d, want %d", got, want)
}
}
func sampleEnvelope() envelope.FrameEnvelope { func sampleEnvelope() envelope.FrameEnvelope {
return envelope.FrameEnvelope{ return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808, Protocol: envelope.ProtocolJT808,

View File

@@ -38,7 +38,7 @@ func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error)
return frames, append([]byte(nil), remaining...), nil return frames, append([]byte(nil), remaining...), nil
} }
bodyLen := int(binary.BigEndian.Uint16(remaining[22:24])) bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
frameLen := headerLen + bodyLen + 1 frameLen := frameLength(remaining, bodyLen)
if len(remaining) < frameLen { if len(remaining) < frameLen {
return frames, append([]byte(nil), remaining...), nil return frames, append([]byte(nil), remaining...), nil
} }
@@ -59,7 +59,8 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2])) return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
} }
bodyLen := int(binary.BigEndian.Uint16(raw[22:24])) bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
if len(raw) != headerLen+bodyLen+1 { actualBodyLen := len(raw) - headerLen - 1
if len(raw) != headerLen+bodyLen+1 && !isExtendedPlatformLogin(raw[2], bodyLen, actualBodyLen) {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1) return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
} }
if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want { if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
@@ -69,15 +70,16 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
command := raw[2] command := raw[2]
responseFlag := raw[3] responseFlag := raw[3]
vin := strings.TrimRight(string(raw[4:21]), "\x00 ") vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
body := raw[headerLen : headerLen+bodyLen] body := raw[headerLen : headerLen+actualBodyLen]
parsed := map[string]any{ parsed := map[string]any{
"header": map[string]any{ "header": map[string]any{
"version": version, "version": version,
"command": fmt.Sprintf("0x%02X", command), "command": fmt.Sprintf("0x%02X", command),
"response_flag": fmt.Sprintf("0x%02X", responseFlag), "response_flag": fmt.Sprintf("0x%02X", responseFlag),
"vin": vin, "vin": vin,
"encrypt": raw[21], "encrypt": raw[21],
"body_length": bodyLen, "body_length": bodyLen,
"actual_body_length": actualBodyLen,
}, },
} }
fields := map[string]any{} fields := map[string]any{}
@@ -91,6 +93,16 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
parsed["device_time"] = eventTime.Format(time.RFC3339) parsed["device_time"] = eventTime.Format(time.RFC3339)
} }
parsed["data_units"] = units parsed["data_units"] = units
} else if command == 0x05 {
login := parsePlatformLogin(body)
parsed["platform_login"] = login
if deviceTime, ok := login["login_time"].(string); ok && deviceTime != "" {
fields["device_time"] = deviceTime
parsed["device_time"] = deviceTime
}
if account, ok := login["username"].(string); ok && account != "" {
fields["platform_account"] = account
}
} }
env := envelope.FrameEnvelope{ env := envelope.FrameEnvelope{
@@ -113,7 +125,7 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
if len(body) < 6 { if len(body) < 6 {
return time.Time{}, nil return time.Time{}, nil
} }
eventTime := parseBCDTime(body[:6]) eventTime := parseGBTime(body[:6])
cursor := 6 cursor := 6
var units []map[string]any var units []map[string]any
for cursor < len(body) { for cursor < len(body) {
@@ -303,6 +315,54 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
return eventTime, units return eventTime, units
} }
func frameLength(frameStart []byte, bodyLen int) int {
standardLen := headerLen + bodyLen + 1
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
return standardLen
}
// Some field platform-login frames declare the standard 41-byte body but
// carry a 32-byte password, making the actual body 53 bytes. Only consume
// the extended form when its own BCC verifies so the splitter does not eat
// bytes from the next frame.
extendedLen := headerLen + 53 + 1
if len(frameStart) < extendedLen {
return standardLen
}
extended := frameStart[:extendedLen]
if bcc(extended[2:len(extended)-1]) == extended[len(extended)-1] {
return extendedLen
}
return standardLen
}
func isExtendedPlatformLogin(command byte, declaredBodyLen int, actualBodyLen int) bool {
return command == 0x05 && declaredBodyLen == 41 && actualBodyLen == 53
}
func parsePlatformLogin(body []byte) map[string]any {
login := map[string]any{
"raw_body_length": len(body),
}
if len(body) < 9 {
login["error"] = "truncated"
return login
}
loginTime := parseGBTime(body[:6])
if !loginTime.IsZero() {
login["login_time"] = loginTime.Format(time.RFC3339)
}
login["serial_no"] = binary.BigEndian.Uint16(body[6:8])
if len(body) >= 21 {
login["username"] = trimASCII(body[8:20])
}
if len(body) > 21 {
passwordEnd := len(body) - 1
login["password"] = trimASCII(body[20:passwordEnd])
login["encrypt_rule"] = body[passwordEnd]
}
return login
}
func parseVehicleData(data []byte) map[string]any { func parseVehicleData(data []byte) map[string]any {
return map[string]any{ return map[string]any{
"vehicle_status": int(data[0]), "vehicle_status": int(data[0]),
@@ -751,24 +811,24 @@ func version(a byte, b byte) (string, bool) {
} }
} }
func parseBCDTime(data []byte) time.Time { func parseGBTime(data []byte) time.Time {
if len(data) != 6 { if len(data) != 6 {
return time.Time{} return time.Time{}
} }
year := 2000 + bcdByte(data[0]) year := 2000 + int(data[0])
month := time.Month(bcdByte(data[1])) month := time.Month(data[1])
day := bcdByte(data[2]) day := int(data[2])
hour := bcdByte(data[3]) hour := int(data[3])
minute := bcdByte(data[4]) minute := int(data[4])
second := bcdByte(data[5]) second := int(data[5])
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 { if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
return time.Time{} return time.Time{}
} }
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600)) return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
} }
func bcdByte(value byte) int { func trimASCII(data []byte) string {
return int(value>>4)*10 + int(value&0x0f) return strings.TrimRight(string(data), "\x00 ")
} }
func bcc(data []byte) byte { func bcc(data []byte) byte {

View File

@@ -12,7 +12,7 @@ import (
const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA" const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) { func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57}) first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39})
second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil) second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
stream := append([]byte{0x00, 0x01}, first...) stream := append([]byte{0x00, 0x01}, first...)
stream = append(stream, second[:len(second)-3]...) stream = append(stream, second[:len(second)-3]...)
@@ -49,8 +49,81 @@ func TestParseFrameRejectsBadBCC(t *testing.T) {
} }
} }
func TestExtractFramesSupportsExtendedPlatformLogin(t *testing.T) {
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x1f, 0x00, 0x01}
body = append(body, fixedASCII("Hyundai", 12)...)
body = append(body, fixedASCII("f2e3445d7cda409fb4f278f6fb890734", 32)...)
body = append(body, 0x01)
frame := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
frames, remainder, err := ExtractFrames(append(frame, []byte{0x23, 0x23, 0x05}...))
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 3 {
t.Fatalf("unexpected split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
if len(frames[0]) != headerLen+53+1 {
t.Fatalf("extended frame len = %d", len(frames[0]))
}
env, err := ParseFrame(frames[0], 1782745114999, "8.134.95.166:39376")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x05" || env.ParseStatus != envelope.ParseOK {
t.Fatalf("unexpected env: %#v", env)
}
header := env.Parsed["header"].(map[string]any)
if header["body_length"] != 41 || header["actual_body_length"] != 53 {
t.Fatalf("unexpected length header: %#v", header)
}
login := env.Parsed["platform_login"].(map[string]any)
if login["username"] != "Hyundai" || login["password"] != "f2e3445d7cda409fb4f278f6fb890734" {
t.Fatalf("unexpected platform login: %#v", login)
}
if login["login_time"] != "2026-07-02T00:38:31+08:00" {
t.Fatalf("unexpected login time: %#v", login)
}
if env.Fields["platform_account"] != "Hyundai" {
t.Fatalf("platform account not exposed: %#v", env.Fields)
}
}
func TestAutoResponseEchoesRawVINAndOriginalTime(t *testing.T) {
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x0f, 0x00, 0x01}
body = append(body, fixedASCII("Hyundai", 12)...)
body = append(body, fixedASCII("ack-test-password", 32)...)
body = append(body, 0x01)
request := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
env, err := ParseFrame(request, 1782914969584, "8.134.95.166:39376")
if err != nil {
t.Fatal(err)
}
response, ok, err := AutoResponse(request, env)
if err != nil {
t.Fatalf("AutoResponse() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
if response[2] != 0x05 || response[3] != 0x01 {
t.Fatalf("unexpected response header: %x", response[:4])
}
if hex.EncodeToString(response[4:21]) != "0000000000000000000000000000000000" {
t.Fatalf("raw vin not echoed: %x", response[4:21])
}
if hex.EncodeToString(response[24:30]) != "1a070200260f" {
t.Fatalf("timestamp body not preserved: %x", response[24:30])
}
if got, want := bcc(response[2:len(response)-1]), response[len(response)-1]; got != want {
t.Fatalf("response bcc got=0x%02x want=0x%02x", got, want)
}
}
func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) { func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
body := []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57} body := []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39}
body = append(body, 0x01) body = append(body, 0x01)
body = append(body, body = append(body,
0x01, // vehicle status 0x01, // vehicle status
@@ -163,13 +236,23 @@ func assertIntField(t *testing.T, env envelope.FrameEnvelope, key string, want i
} }
func buildFrame(command byte, response byte, vin string, body []byte) []byte { func buildFrame(command byte, response byte, vin string, body []byte) []byte {
return buildFrameWithDeclaredLength(command, response, vin, body, len(body))
}
func buildFrameWithDeclaredLength(command byte, response byte, vin string, body []byte, declaredBodyLen int) []byte {
frame := []byte{'#', '#', command, response} frame := []byte{'#', '#', command, response}
vinBytes := []byte(vin) vinBytes := []byte(vin)
if len(vinBytes) < 17 { if len(vinBytes) < 17 {
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...) vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
} }
frame = append(frame, vinBytes[:17]...) frame = append(frame, vinBytes[:17]...)
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body))) frame = append(frame, 0x01, byte(declaredBodyLen>>8), byte(declaredBodyLen))
frame = append(frame, body...) frame = append(frame, body...)
return append(frame, bcc(frame[2:])) return append(frame, bcc(frame[2:]))
} }
func fixedASCII(value string, size int) []byte {
out := make([]byte, size)
copy(out, []byte(value))
return out
}

View File

@@ -0,0 +1,80 @@
package gb32960
import (
"encoding/binary"
"errors"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrResponseFrameTooShort = errors.New("gb32960 response frame too short")
const (
responseSuccess = byte(0x01)
responseCommand = byte(0xfe)
encryptNone = byte(0x01)
)
// AutoResponse builds the protocol ACK for accepted upstream GB/T 32960 frames.
// The ACK is written only after the caller has durably published the frame.
func AutoResponse(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if env.ParseStatus == envelope.ParseBadFrame {
return nil, false, nil
}
if len(raw) < headerLen+1 {
return nil, false, ErrResponseFrameTooShort
}
if raw[3] != responseCommand {
return nil, false, nil
}
command := raw[2]
if !shouldRespond(command) {
return nil, false, nil
}
body := []byte(nil)
if timestamp := responseTime(raw, command); !timestamp.IsZero() {
body = encodeGBTime(timestamp)
}
return buildResponse(raw[0], command, responseSuccess, raw[4:21], body), true, nil
}
func shouldRespond(command byte) bool {
switch command {
case 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08:
return true
default:
return false
}
}
func responseTime(raw []byte, command byte) time.Time {
bodyLen := len(raw) - headerLen - 1
if bodyLen >= 6 {
return parseGBTime(raw[headerLen : headerLen+6])
}
return time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600))
}
func encodeGBTime(t time.Time) []byte {
local := t.In(time.FixedZone("Asia/Shanghai", 8*3600))
return []byte{
byte(local.Year() - 2000),
byte(local.Month()),
byte(local.Day()),
byte(local.Hour()),
byte(local.Minute()),
byte(local.Second()),
}
}
func buildResponse(start byte, command byte, responseFlag byte, vin17 []byte, body []byte) []byte {
out := make([]byte, 0, headerLen+len(body)+1)
out = append(out, start, start, command, responseFlag)
out = append(out, vin17[:17]...)
out = append(out, encryptNone, 0, 0)
binary.BigEndian.PutUint16(out[22:24], uint16(len(body)))
out = append(out, body...)
return append(out, bcc(out[2:]))
}

View File

@@ -10,8 +10,8 @@ import (
"time" "time"
"unicode/utf8" "unicode/utf8"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/encoding/simplifiedchinese"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
) )
var ( var (
@@ -97,16 +97,19 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
phoneEnd = 15 phoneEnd = 15
serialStart = 15 serialStart = 15
} }
if props&0x2000 != 0 { subpackage := props&0x2000 != 0
if subpackage {
headerLen += 4 headerLen += 4
}
if len(raw) < headerLen+int(bodySize)+1 {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
}
if subpackage {
packageInfo = map[string]any{ packageInfo = map[string]any{
"total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]), "total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]),
"index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]), "index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]),
} }
} }
if len(raw) < headerLen+int(bodySize)+1 {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
}
phoneBCD := raw[phoneStart:phoneEnd] phoneBCD := raw[phoneStart:phoneEnd]
phone := bcdString(phoneBCD) phone := bcdString(phoneBCD)
@@ -126,7 +129,7 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
"phone": phone, "phone": phone,
"phone_trim_zero": phoneTrimZero, "phone_trim_zero": phoneTrimZero,
"sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]), "sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
"subpackage": props&0x2000 != 0, "subpackage": subpackage,
"package_info": packageInfo, "package_info": packageInfo,
"encryption_flags": (props >> 10) & 0x07, "encryption_flags": (props >> 10) & 0x07,
}, },
@@ -155,7 +158,7 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
} }
} else if messageID == 0x0100 { } else if messageID == 0x0100 {
registration, err := parseRegistration(body) registration, err := parseRegistration(body, versioned)
if err != nil { if err != nil {
return envelope.FrameEnvelope{}, err return envelope.FrameEnvelope{}, err
} }
@@ -167,7 +170,7 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
fields["plate"] = plate fields["plate"] = plate
} }
} else if messageID == 0x0102 { } else if messageID == 0x0102 {
authentication := parseAuthentication(body) authentication := parseAuthentication(body, versioned)
parsed["authentication"] = authentication parsed["authentication"] = authentication
if token, ok := authentication["token"].(string); ok { if token, ok := authentication["token"].(string); ok {
fields["auth_token"] = token fields["auth_token"] = token
@@ -195,37 +198,107 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
return env, nil return env, nil
} }
func parseRegistration(body []byte) (map[string]any, error) { func parseRegistration(body []byte, versioned bool) (map[string]any, error) {
if len(body) < 37 { schemaVersion := "2011_2013"
manufacturerSize := 5
deviceTypeSize := 20
deviceIDSize := 7
if versioned {
schemaVersion = "2019"
manufacturerSize = 11
deviceTypeSize = 30
deviceIDSize = 30
}
minLen := 2 + 2 + manufacturerSize + deviceTypeSize + deviceIDSize + 1
if len(body) < minLen {
return nil, fmt.Errorf("%w: registration body len=%d", ErrFrameTooShort, len(body)) return nil, fmt.Errorf("%w: registration body len=%d", ErrFrameTooShort, len(body))
} }
manufacturerStart := 4
deviceTypeStart := manufacturerStart + manufacturerSize
deviceIDStart := deviceTypeStart + deviceTypeSize
plateColorStart := deviceIDStart + deviceIDSize
registration := map[string]any{ registration := map[string]any{
"province": binary.BigEndian.Uint16(body[0:2]), "schema_version": schemaVersion,
"city": binary.BigEndian.Uint16(body[2:4]), "province": binary.BigEndian.Uint16(body[0:2]),
"manufacturer": fixedText(body[4:9]), "city": binary.BigEndian.Uint16(body[2:4]),
"device_type": fixedText(body[9:29]), "manufacturer": fixedText(body[manufacturerStart:deviceTypeStart]),
"device_id": fixedText(body[29:36]), "device_type": fixedText(body[deviceTypeStart:deviceIDStart]),
"plate_color": body[36], "device_id": fixedText(body[deviceIDStart:plateColorStart]),
"plate": freeText(body[37:]), "plate_color": body[plateColorStart],
"plate": freeText(body[plateColorStart+1:]),
} }
return registration, nil return registration, nil
} }
func parseAuthentication(body []byte) map[string]any { func parseAuthentication(body []byte, versioned bool) map[string]any {
token := freeText(body) if versioned {
out := map[string]any{"token": token} if out, ok := parseVersionedAuthentication(body); ok {
return out return out
}
if out, ok := parseDelimitedVersionedAuthentication(body); ok {
return out
}
}
return map[string]any{"token": freeText(body)}
}
func parseVersionedAuthentication(body []byte) (map[string]any, bool) {
if len(body) < 17 {
return nil, false
}
tokenLen := int(body[0])
imeiStart := 1 + tokenLen
softwareLenStart := imeiStart + 15
if tokenLen <= 0 || len(body) < softwareLenStart+1 {
return nil, false
}
softwareLen := int(body[softwareLenStart])
softwareStart := softwareLenStart + 1
if len(body) < softwareStart+softwareLen {
return nil, false
}
return map[string]any{
"schema_version": "2019",
"token": freeText(body[1:imeiStart]),
"imei": fixedText(body[imeiStart:softwareLenStart]),
"software_version": freeText(body[softwareStart : softwareStart+softwareLen]),
}, true
}
func parseDelimitedVersionedAuthentication(body []byte) (map[string]any, bool) {
parts := bytes.Split(body, []byte{0xff, 0xff})
if len(parts) < 2 {
return nil, false
}
out := map[string]any{
"schema_version": "2019_delimited",
"token": freeText(parts[0]),
"imei": freeText(parts[1]),
}
if len(parts) > 2 {
out["software_version"] = freeText(parts[2])
}
return out, true
} }
func parseLocation(body []byte) (Location, error) { func parseLocation(body []byte) (Location, error) {
if len(body) < 28 { if len(body) < 28 {
return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body)) return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
} }
status := binary.BigEndian.Uint32(body[4:8])
latitude := float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000
longitude := float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000
if status&(1<<2) != 0 {
latitude = -latitude
}
if status&(1<<3) != 0 {
longitude = -longitude
}
location := Location{ location := Location{
AlarmFlag: binary.BigEndian.Uint32(body[0:4]), AlarmFlag: binary.BigEndian.Uint32(body[0:4]),
StatusFlag: binary.BigEndian.Uint32(body[4:8]), StatusFlag: status,
Latitude: float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000, Latitude: latitude,
Longitude: float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000, Longitude: longitude,
AltitudeM: binary.BigEndian.Uint16(body[16:18]), AltitudeM: binary.BigEndian.Uint16(body[16:18]),
SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10, SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10,
DirectionDeg: binary.BigEndian.Uint16(body[20:22]), DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
@@ -256,14 +329,105 @@ func parseAdditional(data []byte, location *Location) []AdditionalItem {
Length: size, Length: size,
ValueHex: strings.ToUpper(hex.EncodeToString(value)), ValueHex: strings.ToUpper(hex.EncodeToString(value)),
} }
// JT/T 808-2011 table 27: id 0x01 is GPS total mileage, DWORD, 0.1 km. switch {
if id == 0x01 && size == 4 { case id == 0x01 && size == 4:
mileage := float64(binary.BigEndian.Uint32(value)) / 10 rawMileage := binary.BigEndian.Uint32(value)
mileage := float64(rawMileage) / 10
location.TotalMileageKM = &mileage location.TotalMileageKM = &mileage
item.Parsed = map[string]any{ item.Parsed = map[string]any{
"name": envelope.FieldTotalMileageKM, "name": envelope.FieldTotalMileageKM,
"value": mileage, "raw_value_tenth_km": rawMileage,
"unit": "km", "value": mileage,
"unit": "km",
}
case id == 0x02 && size == 2:
rawFuel := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "fuel_l",
"raw_value_tenth_l": rawFuel,
"value": float64(rawFuel) / 10,
"unit": "L",
}
case id == 0x03 && size == 2:
rawSpeed := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "recorder_speed_kmh",
"raw_value_tenth_kmh": rawSpeed,
"value": float64(rawSpeed) / 10,
"unit": "km/h",
}
case id == 0x04 && size == 2:
item.Parsed = map[string]any{
"name": "manual_alarm_event_id",
"value": binary.BigEndian.Uint16(value),
}
case id == 0x05 && size == 30:
item.Parsed = map[string]any{
"name": "tire_pressure",
"values": parseTirePressure(value),
}
case id == 0x06 && size == 2:
item.Parsed = map[string]any{
"name": "carriage_temperature_c",
"value": int16(binary.BigEndian.Uint16(value)),
"unit": "C",
}
case id == 0x11 && (size == 1 || size == 5):
parsed := map[string]any{
"name": "overspeed_alarm",
"location_type": value[0],
}
if size == 5 {
parsed["area_id"] = binary.BigEndian.Uint32(value[1:5])
}
item.Parsed = parsed
case id == 0x12 && size == 6:
item.Parsed = map[string]any{
"name": "area_route_alarm",
"location_type": value[0],
"area_id": binary.BigEndian.Uint32(value[1:5]),
"direction": value[5],
}
case id == 0x13 && size == 7:
item.Parsed = map[string]any{
"name": "road_section_driving_time_alarm",
"road_section_id": binary.BigEndian.Uint32(value[0:4]),
"driving_time_seconds": binary.BigEndian.Uint16(value[4:6]),
"result": value[6],
"result_description": roadSectionDrivingTimeResult(value[6]),
"driving_time_result_source": "jt808_0x0200_additional_0x13",
}
case id == 0x25 && size == 4:
rawStatus := binary.BigEndian.Uint32(value)
item.Parsed = map[string]any{
"name": "extended_vehicle_signal_status",
"value": rawStatus,
"bits": parseExtendedVehicleSignal(rawStatus),
}
case id == 0x2a && size == 2:
rawStatus := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "io_status",
"value": rawStatus,
"bits": parseIOStatus(rawStatus),
}
case id == 0x2b && size == 4:
analog := binary.BigEndian.Uint32(value)
item.Parsed = map[string]any{
"name": "analog",
"ad0": analog & 0xffff,
"ad1": analog >> 16,
"value": analog,
}
case id == 0x30 && size == 1:
item.Parsed = map[string]any{
"name": "network_signal_strength",
"value": value[0],
}
case id == 0x31 && size == 1:
item.Parsed = map[string]any{
"name": "gnss_satellite_count",
"value": value[0],
} }
} }
out = append(out, item) out = append(out, item)
@@ -308,6 +472,55 @@ func additionalToMaps(items []AdditionalItem) []map[string]any {
return out return out
} }
func parseTirePressure(data []byte) []map[string]any {
out := make([]map[string]any, 0, len(data))
for index, value := range data {
out = append(out, map[string]any{
"position": index,
"value": value,
"valid": value != 0xff,
})
}
return out
}
func parseExtendedVehicleSignal(value uint32) map[string]bool {
return map[string]bool{
"low_beam": value&0x0001 != 0,
"high_beam": value&0x0002 != 0,
"right_turn": value&0x0004 != 0,
"left_turn": value&0x0008 != 0,
"brake": value&0x0010 != 0,
"reverse_gear": value&0x0020 != 0,
"fog_light": value&0x0040 != 0,
"clearance_light": value&0x0080 != 0,
"horn": value&0x0100 != 0,
"air_conditioner": value&0x0200 != 0,
"neutral": value&0x0400 != 0,
"retarder": value&0x0800 != 0,
"abs": value&0x1000 != 0,
"heater": value&0x2000 != 0,
"clutch": value&0x4000 != 0,
}
}
func parseIOStatus(value uint16) map[string]bool {
return map[string]bool{
"deep_sleep": value&0x0001 != 0,
"sleep": value&0x0002 != 0,
}
}
func roadSectionDrivingTimeResult(value byte) string {
if value == 0 {
return "insufficient"
}
if value == 1 {
return "too_long"
}
return "unknown"
}
func bcdString(data []byte) string { func bcdString(data []byte) string {
out := make([]byte, 0, len(data)*2) out := make([]byte, 0, len(data)*2)
for _, value := range data { for _, value := range data {

View File

@@ -10,6 +10,7 @@ import (
const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E" const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E" const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E"
const fullAdditionalLocationFrame = "7e020000800123456789017fff000004000000080006eeb6ad02633df701380003006320070719235901040000000b02020016030200210402002c051e3737370000000000000000000000000000000000000000000000000000001105420000004212064d0000004d4d1307000000580058582504000000632a02000a2b040000001430011e31012806020001927e"
func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) { func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000") stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
@@ -63,6 +64,13 @@ func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" { if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
t.Fatalf("unexpected first additional item: %#v", additional[0]) t.Fatalf("unexpected first additional item: %#v", additional[0])
} }
parsed, ok := additional[0]["parsed"].(map[string]any)
if !ok {
t.Fatalf("mileage parsed field missing: %#v", additional[0])
}
if parsed["raw_value_tenth_km"] != uint32(102412) || parsed["value"] != 10241.2 {
t.Fatalf("unexpected mileage parsed value: %#v", parsed)
}
} }
func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) { func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
@@ -115,6 +123,89 @@ func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
} }
} }
func TestParseFrameParsesCommonLocationAdditionalItems(t *testing.T) {
stream, err := hex.DecodeString(fullAdditionalLocationFrame)
if err != nil {
t.Fatal(err)
}
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782914967713, "127.0.0.1:808")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Phone != "012345678901" {
t.Fatalf("phone = %q", env.Phone)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.1)
location := env.Parsed["location"].(map[string]any)
additional := location["additional"].([]map[string]any)
byID := map[string]map[string]any{}
for _, item := range additional {
byID[item["id"].(string)] = item["parsed"].(map[string]any)
}
if byID["0x01"]["raw_value_tenth_km"] != uint32(11) || byID["0x01"]["value"] != 1.1 {
t.Fatalf("mileage parsed = %#v", byID["0x01"])
}
if byID["0x05"]["name"] != "tire_pressure" {
t.Fatalf("tire pressure missing: %#v", byID["0x05"])
}
tireValues := byID["0x05"]["values"].([]map[string]any)
if tireValues[0]["value"] != byte(0x37) || tireValues[0]["valid"] != true {
t.Fatalf("tire pressure first value = %#v", tireValues[0])
}
if byID["0x11"]["location_type"] != byte(0x42) || byID["0x11"]["area_id"] != uint32(0x42) {
t.Fatalf("overspeed parsed = %#v", byID["0x11"])
}
if byID["0x12"]["location_type"] != byte(0x4d) || byID["0x12"]["area_id"] != uint32(0x4d) || byID["0x12"]["direction"] != byte(0x4d) {
t.Fatalf("area route parsed = %#v", byID["0x12"])
}
if byID["0x13"]["road_section_id"] != uint32(0x58) || byID["0x13"]["driving_time_seconds"] != uint16(0x58) {
t.Fatalf("road section parsed = %#v", byID["0x13"])
}
ioBits := byID["0x2A"]["bits"].(map[string]bool)
if ioBits["deep_sleep"] || !ioBits["sleep"] {
t.Fatalf("io bits = %#v", ioBits)
}
}
func TestParseFrameRejectsTruncatedSubpackageHeader(t *testing.T) {
payload := []byte{0x02, 0x00, 0x20, 0x00}
payload = append(payload, encodeBCD("013307795425", 6)...)
payload = binary.BigEndian.AppendUint16(payload, 1)
payload = append(payload, checksum(payload))
_, err := ParseFrame(payload, 1782914967713, "115.231.168.135:43625")
if err == nil {
t.Fatal("expected truncated subpackage header to fail")
}
}
func TestParseLocationAppliesSouthWestStatusBits(t *testing.T) {
body := make([]byte, 28)
binary.BigEndian.PutUint32(body[4:8], (1<<2)|(1<<3))
binary.BigEndian.PutUint32(body[8:12], 12345678)
binary.BigEndian.PutUint32(body[12:16], 98765432)
location, err := parseLocation(body)
if err != nil {
t.Fatalf("parseLocation() error = %v", err)
}
if location.Latitude != -12.345678 {
t.Fatalf("latitude = %v", location.Latitude)
}
if location.Longitude != -98.765432 {
t.Fatalf("longitude = %v", location.Longitude)
}
}
func TestParseFrameParsesRegistrationIdentity(t *testing.T) { func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
body := make([]byte, 0, 46) body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16) body = binary.BigEndian.AppendUint16(body, 16)
@@ -153,6 +244,38 @@ func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
} }
} }
func TestParseFrameParsesJT8082019RegistrationIdentity(t *testing.T) {
body := make([]byte, 0, 84)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTONG00001", 11)
body = appendFixedASCII(body, "ZK6105CHEVNPG4-2019", 30)
body = appendFixedASCII(body, "DEV201900000000000000000000001", 30)
body = append(body, 2)
body = append(body, []byte("TEST2019")...)
env, err := ParseFrame(buildVersionedFrame(0x0100, "14894135060", 11, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Phone != "14894135060" {
t.Fatalf("phone = %q", env.Phone)
}
if env.DeviceID != "DEV201900000000000000000000001" {
t.Fatalf("device id = %q", env.DeviceID)
}
if env.Plate != "TEST2019" {
t.Fatalf("plate = %q", env.Plate)
}
registration, ok := env.Parsed["registration"].(map[string]any)
if !ok {
t.Fatalf("registration missing: %#v", env.Parsed)
}
if registration["schema_version"] != "2019" || registration["manufacturer"] != "YUTONG00001" {
t.Fatalf("registration fields = %#v", registration)
}
}
func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) { func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
body := make([]byte, 0, 46) body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16) body = binary.BigEndian.AppendUint16(body, 16)
@@ -172,6 +295,26 @@ func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
} }
} }
func TestParseFrameParsesJT8082019Authentication(t *testing.T) {
body := []byte{5}
body = append(body, []byte("TOKEN")...)
body = appendFixedASCII(body, "123456789012345", 15)
body = append(body, 4)
body = append(body, []byte("v1.2")...)
env, err := ParseFrame(buildVersionedFrame(0x0102, "14894135060", 12, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
auth, ok := env.Parsed["authentication"].(map[string]any)
if !ok {
t.Fatalf("authentication missing: %#v", env.Parsed)
}
if auth["token"] != "TOKEN" || auth["imei"] != "123456789012345" || auth["software_version"] != "v1.2" {
t.Fatalf("authentication = %#v", auth)
}
}
func TestParseFrameParsesAuthenticationToken(t *testing.T) { func TestParseFrameParsesAuthenticationToken(t *testing.T) {
env, err := ParseFrame(buildFrame(0x0102, "064646848757", 8, []byte("g7gps")), 1782918600000, "115.231.168.135:43625") env, err := ParseFrame(buildFrame(0x0102, "064646848757", 8, []byte("g7gps")), 1782918600000, "115.231.168.135:43625")
if err != nil { if err != nil {
@@ -189,6 +332,86 @@ func TestParseFrameParsesAuthenticationToken(t *testing.T) {
} }
} }
func TestAutoResponderBuildsRegisterAck(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTNG", 5)
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
body = appendFixedASCII(body, "DEV0001", 7)
body = append(body, 2)
body = append(body, []byte("TEST123")...)
request := buildFrame(0x0100, "013079963379", 9, body)
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatal(err)
}
response, ok, err := NewAutoResponder("g7gps").Respond(request, env)
if err != nil {
t.Fatalf("Respond() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
frames, remainder, err := ExtractFrames(response)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
payload := frames[0]
if binary.BigEndian.Uint16(payload[0:2]) != 0x8100 {
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
}
if hex.EncodeToString(payload[4:10]) != "013079963379" {
t.Fatalf("phone bcd = %x", payload[4:10])
}
if binary.BigEndian.Uint16(payload[12:14]) != 9 || payload[14] != 0 || string(payload[15:20]) != "g7gps" {
t.Fatalf("unexpected register ack body: %x", payload[12:len(payload)-1])
}
}
func TestAutoResponderBuildsVersionedGeneralAck(t *testing.T) {
request := buildVersionedFrame(0x0200, "14894135060", 12, make([]byte, 28))
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatal(err)
}
response, ok, err := NewAutoResponder("").Respond(request, env)
if err != nil {
t.Fatalf("Respond() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
frames, remainder, err := ExtractFrames(response)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
payload := frames[0]
if binary.BigEndian.Uint16(payload[0:2]) != 0x8001 {
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
}
if binary.BigEndian.Uint16(payload[2:4])&0x4000 == 0 || payload[4] != 1 {
t.Fatalf("versioned header not preserved: %x", payload[:5])
}
if hex.EncodeToString(payload[5:15]) != "00000000014894135060" {
t.Fatalf("phone bcd = %x", payload[5:15])
}
bodyStart := 17
if binary.BigEndian.Uint16(payload[bodyStart:bodyStart+2]) != 12 ||
binary.BigEndian.Uint16(payload[bodyStart+2:bodyStart+4]) != 0x0200 ||
payload[bodyStart+4] != 0 {
t.Fatalf("unexpected general ack body: %x", payload[bodyStart:len(payload)-1])
}
}
func TestExtractFramesHandlesEscapedPayload(t *testing.T) { func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d} payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...) frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
@@ -217,6 +440,18 @@ func buildFrame(messageID uint16, phone string, sequence uint16, body []byte) []
return payload return payload
} }
func buildVersionedFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
payload := make([]byte, 0, 17+len(body)+1)
payload = binary.BigEndian.AppendUint16(payload, messageID)
payload = binary.BigEndian.AppendUint16(payload, 0x4000|uint16(len(body)))
payload = append(payload, 1)
payload = append(payload, encodeBCD(phone, 10)...)
payload = binary.BigEndian.AppendUint16(payload, sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return payload
}
func appendFixedASCII(out []byte, value string, size int) []byte { func appendFixedASCII(out []byte, value string, size int) []byte {
start := len(out) start := len(out)
out = append(out, make([]byte, size)...) out = append(out, make([]byte, size)...)

View File

@@ -0,0 +1,130 @@
package jt808
import (
"encoding/binary"
"errors"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrResponseHeaderTooShort = errors.New("jt808 response header too short")
const (
msgTerminalGeneralResponse = uint16(0x0001)
msgTerminalRegister = uint16(0x0100)
msgPlatformGeneralResponse = uint16(0x8001)
msgPlatformRegisterAck = uint16(0x8100)
)
type AutoResponder struct {
authCode string
}
func NewAutoResponder(authCode string) AutoResponder {
if authCode == "" {
authCode = "g7gps"
}
return AutoResponder{authCode: authCode}
}
func (r AutoResponder) Respond(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if env.ParseStatus == envelope.ParseBadFrame {
return nil, false, nil
}
header, err := parseResponseHeader(raw)
if err != nil {
return nil, false, err
}
if header.messageID == msgTerminalGeneralResponse {
return nil, false, nil
}
if header.messageID == msgTerminalRegister {
body := make([]byte, 3, 3+len(r.authCode))
binary.BigEndian.PutUint16(body[0:2], header.sequence)
body[2] = 0
body = append(body, []byte(r.authCode)...)
return encodeResponse(msgPlatformRegisterAck, header, body), true, nil
}
body := make([]byte, 5)
binary.BigEndian.PutUint16(body[0:2], header.sequence)
binary.BigEndian.PutUint16(body[2:4], header.messageID)
body[4] = 0
return encodeResponse(msgPlatformGeneralResponse, header, body), true, nil
}
type responseHeader struct {
messageID uint16
versioned bool
protocolVersion byte
phoneBCD []byte
sequence uint16
}
func parseResponseHeader(raw []byte) (responseHeader, error) {
if len(raw) < 12 {
return responseHeader{}, ErrResponseHeaderTooShort
}
props := binary.BigEndian.Uint16(raw[2:4])
versioned := props&0x4000 != 0
phoneStart := 4
phoneLen := 6
protocolVersion := byte(0)
if versioned {
if len(raw) < 17 {
return responseHeader{}, ErrResponseHeaderTooShort
}
protocolVersion = raw[4]
phoneStart = 5
phoneLen = 10
}
sequenceStart := phoneStart + phoneLen
if len(raw) < sequenceStart+2 {
return responseHeader{}, ErrResponseHeaderTooShort
}
phone := append([]byte(nil), raw[phoneStart:sequenceStart]...)
return responseHeader{
messageID: binary.BigEndian.Uint16(raw[0:2]),
versioned: versioned,
protocolVersion: protocolVersion,
phoneBCD: phone,
sequence: binary.BigEndian.Uint16(raw[sequenceStart : sequenceStart+2]),
}, nil
}
func encodeResponse(messageID uint16, header responseHeader, body []byte) []byte {
props := uint16(len(body)) & 0x03ff
if header.versioned {
props |= 0x4000
}
payload := make([]byte, 0, 4+1+len(header.phoneBCD)+2+len(body)+1)
payload = appendU16(payload, messageID)
payload = appendU16(payload, props)
if header.versioned {
payload = append(payload, header.protocolVersion)
}
payload = append(payload, header.phoneBCD...)
payload = appendU16(payload, header.sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return frame(payload)
}
func appendU16(out []byte, value uint16) []byte {
return append(out, byte(value>>8), byte(value))
}
func frame(payload []byte) []byte {
out := []byte{0x7e}
for _, value := range payload {
switch value {
case 0x7e:
out = append(out, 0x7d, 0x02)
case 0x7d:
out = append(out, 0x7d, 0x01)
default:
out = append(out, value)
}
}
out = append(out, 0x7e)
return out
}

View File

@@ -69,7 +69,7 @@ func fieldsFromData(data map[string]any) map[string]any {
fields[envelope.FieldSpeedKMH] = speed fields[envelope.FieldSpeedKMH] = speed
} }
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok { if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
fields[envelope.FieldTotalMileageKM] = mileage fields[envelope.FieldTotalMileageKM] = normalizeTotalMileageKM(mileage)
} }
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok { if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
fields[envelope.FieldSOCPercent] = soc fields[envelope.FieldSOCPercent] = soc
@@ -166,6 +166,14 @@ func parseTimeMS(raw string, fallback int64) int64 {
if raw == "" { if raw == "" {
return fallback return fallback
} }
for _, layout := range []string{
"2006-01-02 15:04:05.000",
"2006-01-02 15:04:05",
} {
if parsed, err := time.ParseInLocation(layout, raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
return parsed.UnixMilli()
}
}
if len(raw) == 14 { if len(raw) == 14 {
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil { if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
return parsed.UnixMilli() return parsed.UnixMilli()
@@ -187,3 +195,7 @@ func looksLikeVIN(value string) bool {
value = strings.TrimSpace(value) value = strings.TrimSpace(value)
return len(value) == 17 return len(value) == 17
} }
func normalizeTotalMileageKM(value float64) float64 {
return value / 1000
}

View File

@@ -36,7 +36,7 @@ func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
t.Fatalf("plate = %q", env.Plate) t.Fatalf("plate = %q", env.Plate)
} }
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3) assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123456.7) assertFloatField(t, env, envelope.FieldTotalMileageKM, 123.4567)
assertFloatField(t, env, envelope.FieldSOCPercent, 70) assertFloatField(t, env, envelope.FieldSOCPercent, 70)
assertFloatField(t, env, envelope.FieldLongitude, 116.397128) assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
assertFloatField(t, env, envelope.FieldLatitude, 39.916527) assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
@@ -64,6 +64,35 @@ func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3) assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
} }
func TestParseMessageNormalizesProductionMileageMeters(t *testing.T) {
payload := []byte(`{
"device":"LMRKH9AC3R1004101",
"time":"2026-07-02 01:02:04.085",
"data":{"TOTAL_MILEAGE":56905000,"METER_SPEED":19}
}`)
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 56905)
if env.EventTimeMS != 1782925324085 {
t.Fatalf("event time = %d, want 1782925324085", env.EventTimeMS)
}
}
func TestParseMessageTreatsSmallYutongMileageAsMeters(t *testing.T) {
payload := []byte(`{
"device":"LMRKH9AC3R1004101",
"time":"2026-07-02 01:02:04.085",
"data":{"TOTAL_MILEAGE":1024,"METER_SPEED":19}
}`)
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.024)
}
func TestParseMessageRejectsMalformedJSON(t *testing.T) { func TestParseMessageRejectsMalformedJSON(t *testing.T) {
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999) _, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
if !errors.Is(err, ErrInvalidJSON) { if !errors.Is(err, ErrInvalidJSON) {

View File

@@ -7,18 +7,21 @@ import (
) )
type Snapshot struct { type Snapshot struct {
VIN string `json:"vin"` VehicleKey string `json:"vehicle_key"`
Protocol envelope.Protocol `json:"protocol,omitempty"` VIN string `json:"vin"`
EventID string `json:"event_id,omitempty"` Protocol envelope.Protocol `json:"protocol,omitempty"`
EventTimeMS int64 `json:"event_time_ms"` Protocols []envelope.Protocol `json:"protocols,omitempty"`
ReceivedAtMS int64 `json:"received_at_ms"` EventID string `json:"event_id,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"` EventTimeMS int64 `json:"event_time_ms"`
Fields map[string]any `json:"fields,omitempty"` ReceivedAtMS int64 `json:"received_at_ms"`
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"` SourceEndpoint string `json:"source_endpoint,omitempty"`
UpdatedAtMS int64 `json:"updated_at_ms"` Fields map[string]any `json:"fields,omitempty"`
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
UpdatedAtMS int64 `json:"updated_at_ms"`
} }
type OnlineStatus struct { type OnlineStatus struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"` VIN string `json:"vin"`
Online bool `json:"online"` Online bool `json:"online"`
LastSeenMS int64 `json:"last_seen_ms"` LastSeenMS int64 `json:"last_seen_ms"`

View File

@@ -28,7 +28,8 @@ func NewRepository(client *redis.Client, cfg Config) *Repository {
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error { func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
vin := strings.TrimSpace(env.VIN) vin := strings.TrimSpace(env.VIN)
if vin == "" { vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil return nil
} }
nowMS := time.Now().UnixMilli() nowMS := time.Now().UnixMilli()
@@ -37,6 +38,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
eventMS = env.ReceivedAtMS eventMS = env.ReceivedAtMS
} }
protocolSnapshot := Snapshot{ protocolSnapshot := Snapshot{
VehicleKey: vehicleKey,
VIN: vin, VIN: vin,
Protocol: env.Protocol, Protocol: env.Protocol,
EventID: env.StableEventID(), EventID: env.StableEventID(),
@@ -47,16 +49,23 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
FieldTimesMS: fieldTimes(env.Fields, eventMS), FieldTimesMS: fieldTimes(env.Fields, eventMS),
UpdatedAtMS: nowMS, UpdatedAtMS: nowMS,
} }
if err := r.setJSON(ctx, protocolKey(vin, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil { if err := r.setJSON(ctx, protocolKey(vehicleKey, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
return err return err
} }
merged, err := r.GetMerged(ctx, vin) protocols, err := r.addProtocol(ctx, vehicleKey, env.Protocol)
if err != nil {
return err
}
merged, err := r.GetMerged(ctx, vehicleKey)
if err != nil && !errors.Is(err, redis.Nil) { if err != nil && !errors.Is(err, redis.Nil) {
return err return err
} }
if merged.VIN == "" { if merged.VehicleKey == "" {
merged = Snapshot{VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}} merged = Snapshot{VehicleKey: vehicleKey, VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
} else if merged.VIN == "" && vin != "" {
merged.VIN = vin
} }
mergeFields(&merged, env.Fields, eventMS) mergeFields(&merged, env.Fields, eventMS)
if eventMS >= merged.EventTimeMS { if eventMS >= merged.EventTimeMS {
@@ -65,49 +74,48 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
merged.ReceivedAtMS = env.ReceivedAtMS merged.ReceivedAtMS = env.ReceivedAtMS
merged.SourceEndpoint = env.SourceEndpoint merged.SourceEndpoint = env.SourceEndpoint
} }
merged.Protocols = protocols
merged.UpdatedAtMS = nowMS merged.UpdatedAtMS = nowMS
if err := r.setJSON(ctx, mergedKey(vin), merged, r.cfg.ttl()); err != nil { if err := r.setJSON(ctx, mergedKey(vehicleKey), merged, r.cfg.ttl()); err != nil {
return err return err
} }
protocols, err := r.addProtocol(ctx, vin, env.Protocol)
if err != nil {
return err
}
online := OnlineStatus{ online := OnlineStatus{
VehicleKey: vehicleKey,
VIN: vin, VIN: vin,
Online: true, Online: true,
LastSeenMS: env.ReceivedAtMS, LastSeenMS: env.ReceivedAtMS,
Protocols: protocols, Protocols: protocols,
TTLSeconds: int64(r.cfg.ttl().Seconds()), TTLSeconds: int64(r.cfg.ttl().Seconds()),
} }
if err := r.setJSON(ctx, onlineKey(vin), online, r.cfg.ttl()); err != nil { if err := r.setJSON(ctx, onlineKey(vehicleKey), online, r.cfg.ttl()); err != nil {
return err return err
} }
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vin}).Err() return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vehicleKey}).Err()
} }
func (r *Repository) GetMerged(ctx context.Context, vin string) (Snapshot, error) { func (r *Repository) GetMerged(ctx context.Context, vehicleKey string) (Snapshot, error) {
return r.getSnapshot(ctx, mergedKey(vin)) return r.getSnapshot(ctx, mergedKey(vehicleKey))
} }
func (r *Repository) GetProtocol(ctx context.Context, vin string, protocol envelope.Protocol) (Snapshot, error) { func (r *Repository) GetProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) (Snapshot, error) {
return r.getSnapshot(ctx, protocolKey(vin, protocol)) return r.getSnapshot(ctx, protocolKey(vehicleKey, protocol))
} }
func (r *Repository) IsOnline(ctx context.Context, vin string) (OnlineStatus, error) { func (r *Repository) IsOnline(ctx context.Context, vehicleKey string) (OnlineStatus, error) {
vehicleKey = strings.TrimSpace(vehicleKey)
var status OnlineStatus var status OnlineStatus
payload, err := r.client.Get(ctx, onlineKey(vin)).Bytes() payload, err := r.client.Get(ctx, onlineKey(vehicleKey)).Bytes()
if err != nil { if err != nil {
if errors.Is(err, redis.Nil) { if errors.Is(err, redis.Nil) {
return OnlineStatus{VIN: vin, Online: false}, nil return OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}, nil
} }
return status, err return status, err
} }
if err := json.Unmarshal(payload, &status); err != nil { if err := json.Unmarshal(payload, &status); err != nil {
return status, err return status, err
} }
ttl := r.client.TTL(ctx, onlineKey(vin)).Val() ttl := r.client.TTL(ctx, onlineKey(vehicleKey)).Val()
status.Online = ttl > 0 status.Online = ttl > 0
status.TTLSeconds = int64(ttl.Seconds()) status.TTLSeconds = int64(ttl.Seconds())
return status, nil return status, nil
@@ -130,8 +138,8 @@ func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl tim
return r.client.Set(ctx, key, payload, ttl).Err() return r.client.Set(ctx, key, payload, ttl).Err()
} }
func (r *Repository) addProtocol(ctx context.Context, vin string, protocol envelope.Protocol) ([]envelope.Protocol, error) { func (r *Repository) addProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
key := protocolsKey(vin) key := protocolsKey(vehicleKey)
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil { if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
return nil, err return nil, err
} }
@@ -207,18 +215,18 @@ func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
return out return out
} }
func mergedKey(vin string) string { func mergedKey(vehicleKey string) string {
return "vehicle:latest:" + strings.TrimSpace(vin) return "vehicle:latest:" + strings.TrimSpace(vehicleKey)
} }
func protocolKey(vin string, protocol envelope.Protocol) string { func protocolKey(vehicleKey string, protocol envelope.Protocol) string {
return "vehicle:latest:" + strings.TrimSpace(vin) + ":" + string(protocol) return "vehicle:latest:" + strings.TrimSpace(vehicleKey) + ":" + string(protocol)
} }
func onlineKey(vin string) string { func onlineKey(vehicleKey string) string {
return "vehicle:online:" + strings.TrimSpace(vin) return "vehicle:online:" + strings.TrimSpace(vehicleKey)
} }
func protocolsKey(vin string) string { func protocolsKey(vehicleKey string) string {
return "vehicle:protocols:" + strings.TrimSpace(vin) return "vehicle:protocols:" + strings.TrimSpace(vehicleKey)
} }

View File

@@ -54,6 +54,9 @@ func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
if merged.Fields[envelope.FieldLatitude] != 30.5 { if merged.Fields[envelope.FieldLatitude] != 30.5 {
t.Fatalf("new latitude missing: %#v", merged.Fields) t.Fatalf("new latitude missing: %#v", merged.Fields)
} }
if got := protocolNames(merged.Protocols); strings.Join(got, ",") != "GB32960,JT808" {
t.Fatalf("merged protocols = %#v", got)
}
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808) protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
if err != nil { if err != nil {
t.Fatalf("GetProtocol() error = %v", err) t.Fatalf("GetProtocol() error = %v", err)
@@ -94,6 +97,44 @@ func TestRepositoryOnlineStatus(t *testing.T) {
} }
} }
func TestRepositoryUpdatesPhoneOnlyVehicleKey(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811170",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10003.7,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
vehicleKey := "JT808:013307811170"
merged, err := repo.GetMerged(ctx, vehicleKey)
if err != nil {
t.Fatalf("GetMerged() error = %v", err)
}
if merged.VehicleKey != vehicleKey || merged.VIN != "" {
t.Fatalf("unexpected identity: %#v", merged)
}
if merged.Fields[envelope.FieldTotalMileageKM] != 10003.7 {
t.Fatalf("merged fields = %#v", merged.Fields)
}
status, err := repo.IsOnline(ctx, vehicleKey)
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if !status.Online || status.VehicleKey != vehicleKey {
t.Fatalf("online status = %#v", status)
}
}
func TestRepositoryDoesNotOverwritePositiveMileageWithZero(t *testing.T) { func TestRepositoryDoesNotOverwritePositiveMileageWithZero(t *testing.T) {
repo, closeFn := newTestRepository(t) repo, closeFn := newTestRepository(t)
defer closeFn() defer closeFn()
@@ -159,6 +200,30 @@ func TestHandlerReturnsMergedSnapshot(t *testing.T) {
} }
} }
func TestHandlerReturnsPhoneOnlyVehicleKeySnapshot(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811170",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/JT808:013307811170", nil)
rec := httptest.NewRecorder()
NewHandler(repo).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
if !stringsContains(rec.Body.String(), `"vehicle_key":"JT808:013307811170"`) {
t.Fatalf("unexpected body: %s", rec.Body.String())
}
}
func newTestRepository(t *testing.T) (*Repository, func()) { func newTestRepository(t *testing.T) (*Repository, func()) {
t.Helper() t.Helper()
server, err := miniredis.Run() server, err := miniredis.Run()
@@ -175,3 +240,11 @@ func newTestRepository(t *testing.T) (*Repository, func()) {
func stringsContains(value string, pattern string) bool { func stringsContains(value string, pattern string) bool {
return strings.Contains(value, pattern) return strings.Contains(value, pattern)
} }
func protocolNames(protocols []envelope.Protocol) []string {
names := make([]string, 0, len(protocols))
for _, protocol := range protocols {
names = append(names, string(protocol))
}
return names
}

View File

@@ -26,6 +26,7 @@ type Writer struct {
} }
type MetricSample struct { type MetricSample struct {
VehicleKey string
VIN string VIN string
Protocol envelope.Protocol Protocol envelope.Protocol
StatDate string StatDate string
@@ -45,8 +46,13 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
} }
func (w *Writer) EnsureSchema(ctx context.Context) error { func (w *Writer) EnsureSchema(ctx context.Context) error {
_, err := w.exec.ExecContext(ctx, DailyMetricTableSQL) if _, err := w.exec.ExecContext(ctx, DailyMetricTableSQL); err != nil {
return err return err
}
if db, ok := w.exec.(metadataDB); ok {
return migrateDailyMetricTable(ctx, db)
}
return nil
} }
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error { func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -56,6 +62,7 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
} }
for _, sample := range samples { for _, sample := range samples {
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL, if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL,
sample.VehicleKey,
sample.VIN, sample.VIN,
sample.StatDate, sample.StatDate,
string(sample.Protocol), string(sample.Protocol),
@@ -71,7 +78,8 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) { func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
vin := strings.TrimSpace(env.VIN) vin := strings.TrimSpace(env.VIN)
if vin == "" { vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil, nil return nil, nil
} }
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM) totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
@@ -94,6 +102,7 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02") statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
return []MetricSample{ return []MetricSample{
{ {
VehicleKey: vehicleKey,
VIN: vin, VIN: vin,
Protocol: env.Protocol, Protocol: env.Protocol,
StatDate: statDate, StatDate: statDate,
@@ -102,6 +111,7 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
TotalMileageKM: totalMileage, TotalMileageKM: totalMileage,
}, },
{ {
VehicleKey: vehicleKey,
VIN: vin, VIN: vin,
Protocol: env.Protocol, Protocol: env.Protocol,
StatDate: statDate, StatDate: statDate,
@@ -114,10 +124,11 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
const upsertDailyMetricSQL = ` const upsertDailyMetricSQL = `
INSERT INTO vehicle_daily_metric INSERT INTO vehicle_daily_metric
(vin, stat_date, protocol, metric_key, metric_value, metric_unit, (vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit,
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method) first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF') VALUES (?, ?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
ON DUPLICATE KEY UPDATE ON DUPLICATE KEY UPDATE
vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
first_total_mileage_km = CASE first_total_mileage_km = CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0 WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km) THEN VALUES(first_total_mileage_km)
@@ -158,6 +169,82 @@ ON DUPLICATE KEY UPDATE
updated_at = CURRENT_TIMESTAMP updated_at = CURRENT_TIMESTAMP
` `
type metadataDB interface {
Execer
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func migrateDailyMetricTable(ctx context.Context, db metadataDB) error {
columnExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND column_name = 'vehicle_key'`)
if err != nil {
return err
}
if !columnExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD COLUMN vehicle_key VARCHAR(96) NOT NULL DEFAULT '' AFTER id`); err != nil {
return err
}
}
if _, err := db.ExecContext(ctx, `UPDATE vehicle_daily_metric SET vehicle_key = vin WHERE vehicle_key = ''`); err != nil {
return err
}
vehicleIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric_vehicle'`)
if err != nil {
return err
}
if !vehicleIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key)`); err != nil {
return err
}
}
oldIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric'`)
if err != nil {
return err
}
if oldIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric DROP INDEX uk_daily_metric`); err != nil {
return err
}
}
vinIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'idx_vin'`)
if err != nil {
return err
}
if !vinIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD KEY idx_vin (vin)`); err != nil {
return err
}
}
return nil
}
func informationSchemaExists(ctx context.Context, db metadataDB, query string) (bool, error) {
var count int
if err := db.QueryRowContext(ctx, query).Scan(&count); err != nil {
return false, err
}
return count > 0, nil
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) { func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
if env.Fields == nil { if env.Fields == nil {
return 0, false return 0, false

View File

@@ -31,6 +31,9 @@ func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 { if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
t.Fatalf("unexpected daily mileage sample: %#v", samples[0]) t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
} }
if samples[0].VehicleKey != "LNBVIN00000000001" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 { if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 {
t.Fatalf("unexpected daily total sample: %#v", samples[1]) t.Fatalf("unexpected daily total sample: %#v", samples[1])
} }
@@ -39,7 +42,31 @@ func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
} }
} }
func TestSamplesFromEnvelopeSkipsMissingVINOrMileage(t *testing.T) { func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811254",
EventTimeMS: time.Date(2026, 7, 2, 0, 3, 0, 0, loc).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: 10985.7,
},
}, loc)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 2 {
t.Fatalf("sample count = %d", len(samples))
}
if samples[0].VehicleKey != "JT808:013307811254" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[0].VIN != "" {
t.Fatalf("vin should stay empty for unresolved JT808 identity, got %q", samples[0].VIN)
}
}
func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{ samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808, Protocol: envelope.ProtocolJT808,
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2}, Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
@@ -48,7 +75,7 @@ func TestSamplesFromEnvelopeSkipsMissingVINOrMileage(t *testing.T) {
t.Fatalf("SamplesFromEnvelope() error = %v", err) t.Fatalf("SamplesFromEnvelope() error = %v", err)
} }
if len(samples) != 0 { if len(samples) != 0 {
t.Fatalf("expected no samples without vin, got %#v", samples) t.Fatalf("expected no samples without vehicle key, got %#v", samples)
} }
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{ samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
@@ -103,6 +130,9 @@ func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") { if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") {
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query) t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
} }
if !strings.Contains(exec.calls[0].query, "vehicle_key") {
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
}
if len(exec.calls) != 3 { if len(exec.calls) != 3 {
t.Fatalf("exec calls = %d", len(exec.calls)) t.Fatalf("exec calls = %d", len(exec.calls))
} }
@@ -112,6 +142,9 @@ func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") { if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query) t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
} }
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
t.Fatalf("first upsert arg should be vehicle_key, got %#v", got)
}
} }
type execCall struct { type execCall struct {

View File

@@ -17,16 +17,18 @@ type Queryer interface {
} }
type MetricQuery struct { type MetricQuery struct {
VIN string VehicleKey string
Protocol string VIN string
MetricKey string Protocol string
DateFrom string MetricKey string
DateTo string DateFrom string
Limit int DateTo string
Offset int Limit int
Offset int
} }
type MetricRow struct { type MetricRow struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"` VIN string `json:"vin"`
StatDate string `json:"stat_date"` StatDate string `json:"stat_date"`
Protocol string `json:"protocol"` Protocol string `json:"protocol"`
@@ -61,7 +63,7 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
} }
defer rows.Close() defer rows.Close()
var out []MetricRow out := make([]MetricRow, 0)
for rows.Next() { for rows.Next() {
var row MetricRow var row MetricRow
var statDate scanDate var statDate scanDate
@@ -70,6 +72,7 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
var first sql.NullFloat64 var first sql.NullFloat64
var latest sql.NullFloat64 var latest sql.NullFloat64
if err := rows.Scan( if err := rows.Scan(
&row.VehicleKey,
&row.VIN, &row.VIN,
&statDate, &statDate,
&row.Protocol, &row.Protocol,
@@ -99,7 +102,26 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
return out, rows.Err() return out, rows.Err()
} }
func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, error) {
query = normalizeMetricQuery(query)
sqlText, args := buildMetricCountSQL(query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return 0, err
}
defer rows.Close()
var total int64
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, rows.Err()
}
func normalizeMetricQuery(query MetricQuery) MetricQuery { func normalizeMetricQuery(query MetricQuery) MetricQuery {
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN) query.VIN = strings.TrimSpace(query.VIN)
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.MetricKey = strings.TrimSpace(query.MetricKey) query.MetricKey = strings.TrimSpace(query.MetricKey)
@@ -112,6 +134,26 @@ func normalizeMetricQuery(query MetricQuery) MetricQuery {
} }
func buildMetricSQL(query MetricQuery) (string, []any) { func buildMetricSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
args = append(args, query.Limit, query.Offset)
return sqlText, args
}
func buildMetricCountSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT COUNT(*) FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, args
}
func buildMetricWhere(query MetricQuery) ([]string, []any) {
var where []string var where []string
var args []any var args []any
add := func(clause string, value any) { add := func(clause string, value any) {
@@ -121,6 +163,9 @@ func buildMetricSQL(query MetricQuery) (string, []any) {
if query.VIN != "" { if query.VIN != "" {
add("vin = ?", query.VIN) add("vin = ?", query.VIN)
} }
if query.VehicleKey != "" {
add("vehicle_key = ?", query.VehicleKey)
}
if query.Protocol != "" { if query.Protocol != "" {
add("protocol = ?", query.Protocol) add("protocol = ?", query.Protocol)
} }
@@ -133,13 +178,7 @@ func buildMetricSQL(query MetricQuery) (string, []any) {
if query.DateTo != "" { if query.DateTo != "" {
add("stat_date <= ?", query.DateTo) add("stat_date <= ?", query.DateTo)
} }
sqlText := `SELECT vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric` return where, args
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY stat_date DESC, vin ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
args = append(args, query.Limit, query.Offset)
return sqlText, args
} }
type MetricHandler struct { type MetricHandler struct {
@@ -167,6 +206,11 @@ func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeMetricError(w, http.StatusBadRequest, err.Error()) writeMetricError(w, http.StatusBadRequest, err.Error())
return return
} }
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeMetricError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query) rows, err := h.repository.Query(r.Context(), query)
if err != nil { if err != nil {
writeMetricError(w, http.StatusInternalServerError, err.Error()) writeMetricError(w, http.StatusInternalServerError, err.Error())
@@ -175,7 +219,7 @@ func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{ _ = json.NewEncoder(w).Encode(map[string]any{
"items": rows, "items": rows,
"total": len(rows), "total": total,
"limit": query.Limit, "limit": query.Limit,
"offset": query.Offset, "offset": query.Offset,
}) })
@@ -192,13 +236,14 @@ func parseMetricQuery(r *http.Request) (MetricQuery, error) {
return MetricQuery{}, err return MetricQuery{}, err
} }
query := MetricQuery{ query := MetricQuery{
VIN: values.Get("vin"), VehicleKey: values.Get("vehicleKey"),
Protocol: values.Get("protocol"), VIN: values.Get("vin"),
MetricKey: values.Get("metricKey"), Protocol: values.Get("protocol"),
DateFrom: values.Get("dateFrom"), MetricKey: values.Get("metricKey"),
DateTo: values.Get("dateTo"), DateFrom: values.Get("dateFrom"),
Limit: limit, DateTo: values.Get("dateTo"),
Offset: offset, Limit: limit,
Offset: offset,
} }
if !validDate(query.DateFrom) || !validDate(query.DateTo) { if !validDate(query.DateFrom) || !validDate(query.DateTo) {
return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD") return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD")

View File

@@ -18,14 +18,14 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0). WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit", "vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method", "first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at", "created_at", "updated_at",
}).AddRow( }).AddRow(
"LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km", "LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km",
12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)), 12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
)) ))
@@ -46,6 +46,9 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 { if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 {
t.Fatalf("unexpected row: %#v", rows[0]) t.Fatalf("unexpected row: %#v", rows[0])
} }
if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
t.Fatalf("vehicle key = %q", rows[0].VehicleKey)
}
if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" { if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" {
t.Fatalf("unexpected time formatting: %#v", rows[0]) t.Fatalf("unexpected time formatting: %#v", rows[0])
} }
@@ -60,14 +63,17 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0). WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit", "vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method", "first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at", "created_at", "updated_at",
}).AddRow( }).AddRow(
"LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km", "LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km",
53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25", 53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25",
)) ))
@@ -81,7 +87,7 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
} }
body := response.Body.String() body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":1`} { for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":42`} {
if !strings.Contains(body, want) { if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body) t.Fatalf("response missing %s: %s", want, body)
} }
@@ -91,6 +97,81 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
} }
} }
func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("JT808:013307811254", "JT808").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("JT808:013307811254", "JT808", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"JT808:013307811254", "", "2026-07-02", "JT808", MetricDailyTotalMileageKM, 10985.7, "km",
10985.7, 10985.7, 1, "TOTAL_MILEAGE_DIFF", "2026-07-02 00:03:00", "2026-07-02 00:03:00",
))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vehicleKey=JT808:013307811254&protocol=JT808", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"metric_value":10985.7`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("YUTONG_MQTT").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("YUTONG_MQTT", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?protocol=YUTONG_MQTT", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
t.Fatalf("expected empty items array, got: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerRejectsInvalidPagination(t *testing.T) { func TestMetricHandlerRejectsInvalidPagination(t *testing.T) {
handler := NewMetricHandler(NewMetricRepository(&sql.DB{})) handler := NewMetricHandler(NewMetricRepository(&sql.DB{}))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?limit=2001", nil) request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?limit=2001", nil)

View File

@@ -2,7 +2,8 @@ package stats
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric ( const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
id BIGINT PRIMARY KEY AUTO_INCREMENT, id BIGINT PRIMARY KEY AUTO_INCREMENT,
vin VARCHAR(32) NOT NULL, vehicle_key VARCHAR(96) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '',
stat_date DATE NOT NULL, stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL, protocol VARCHAR(32) NOT NULL,
metric_key VARCHAR(64) NOT NULL, metric_key VARCHAR(64) NOT NULL,
@@ -14,7 +15,8 @@ const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
calculation_method VARCHAR(64) NOT NULL, calculation_method VARCHAR(64) NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key), UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key),
KEY idx_vin (vin),
KEY idx_stat_date (stat_date), KEY idx_stat_date (stat_date),
KEY idx_protocol_metric (protocol, metric_key) KEY idx_protocol_metric (protocol, metric_key)
)` )`

162
tools/go_kafka_prod_smoke.py Executable file
View File

@@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""Kafka topic and consumer lag smoke checks for the Go native production stack."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
DEFAULT_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_BOOTSTRAP = "127.0.0.1:9092"
DEFAULT_KAFKA_BIN = "/opt/kafka/current/bin"
DEFAULT_TOPICS = [
"vehicle.raw.go.gb32960.v1",
"vehicle.raw.go.jt808.v1",
"vehicle.raw.go.yutong-mqtt.v1",
"vehicle.event.go.unified.v1",
]
DEFAULT_GROUPS = [
"go-history-writer",
"go-stat-writer",
"go-realtime-api",
]
@dataclass(frozen=True)
class Check:
name: str
status: str
message: str
@dataclass(frozen=True)
class ConsumerLag:
group: str
topic: str
partition: int
current_offset: int | None
log_end_offset: int
lag: int | None
def parse_topic_list(output: str) -> set[str]:
return {line.strip() for line in output.splitlines() if line.strip()}
def topic_checks(existing: set[str], required: list[str]) -> list[Check]:
checks: list[Check] = []
for topic in required:
if topic in existing:
checks.append(Check("topic." + topic, "pass", "present"))
else:
checks.append(Check("topic." + topic, "fail", "missing"))
return checks
def parse_consumer_group_describe(output: str) -> list[ConsumerLag]:
rows: list[ConsumerLag] = []
for line in output.splitlines():
parts = line.split()
if len(parts) < 6 or parts[0] == "GROUP":
continue
group, topic = parts[0], parts[1]
try:
partition = int(parts[2])
current = parse_optional_int(parts[3])
log_end = int(parts[4])
lag = parse_optional_int(parts[5])
except ValueError:
continue
rows.append(ConsumerLag(group, topic, partition, current, log_end, lag))
return rows
def parse_optional_int(value: str) -> int | None:
value = value.strip()
if value == "-":
return None
return int(value)
def group_lag_check(group: str, rows: list[ConsumerLag], max_lag: int) -> Check:
group_rows = [row for row in rows if row.group == group]
if not group_rows:
return Check("group." + group, "fail", "no rows")
concrete_lags = [row.lag for row in group_rows if row.lag is not None]
if not concrete_lags:
return Check("group." + group, "fail", "no concrete lag rows")
max_seen = max(concrete_lags)
topics = sorted({row.topic for row in group_rows})
status = "pass" if max_seen <= max_lag else "fail"
return Check(
"group." + group,
status,
"max_lag=" + str(max_seen) + "; threshold=" + str(max_lag) + "; topics=" + ",".join(topics),
)
def ssh(host: str, user: str, command: str, timeout: float) -> str:
target = user + "@" + host if user else host
completed = subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", target, command],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
return completed.stdout
def remote_topic_command(kafka_bin: str, bootstrap: str) -> str:
return kafka_bin.rstrip("/") + "/kafka-topics.sh --bootstrap-server " + bootstrap + " --list"
def remote_group_command(kafka_bin: str, bootstrap: str, group: str) -> str:
return kafka_bin.rstrip("/") + "/kafka-consumer-groups.sh --bootstrap-server " + bootstrap + " --describe --group " + group
def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
topic_output = ssh(args.host, args.user, remote_topic_command(args.kafka_bin, args.bootstrap), args.timeout)
checks = topic_checks(parse_topic_list(topic_output), DEFAULT_TOPICS)
for group in DEFAULT_GROUPS:
group_output = ssh(args.host, args.user, remote_group_command(args.kafka_bin, args.bootstrap, group), args.timeout)
checks.append(group_lag_check(group, parse_consumer_group_describe(group_output), args.max_lag))
status = "fail" if any(check.status == "fail" for check in checks) else "pass"
return status, checks
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--bootstrap", default=DEFAULT_BOOTSTRAP)
parser.add_argument("--kafka-bin", default=DEFAULT_KAFKA_BIN)
parser.add_argument("--max-lag", type=int, default=100)
parser.add_argument("--timeout", type=float, default=20.0)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
status, checks = run(args)
except Exception as exc:
status = "fail"
checks = [Check("kafka.ssh", "fail", str(exc))]
print(json.dumps({
"status": status,
"host": args.host,
"bootstrap": args.bootstrap,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

253
tools/go_native_deploy.py Normal file
View File

@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Build and deploy the Go native vehicle gateway release to ECS."""
from __future__ import annotations
import argparse
import os
import pathlib
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
from dataclasses import dataclass
DEFAULT_APP_HOST = "115.29.187.205"
DEFAULT_KAFKA_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_RELEASE_ROOT = "/opt/lingniu-go-native"
DEFAULT_GO_DIR = "go/vehicle-gateway"
BINARIES = [
("gateway", "./cmd/gateway"),
("history-writer", "./cmd/history-writer"),
("stat-writer", "./cmd/stat-writer"),
("realtime-api", "./cmd/realtime-api"),
]
SERVICES = [
"lingniu-go-gateway",
"lingniu-go-history-writer",
"lingniu-go-stat-writer",
"lingniu-go-realtime-api",
]
@dataclass(frozen=True)
class BuildCommand:
output_name: str
argv: list[str]
env: list[str]
cwd: str
@dataclass(frozen=True)
class SpoolStatus:
files: int
recent: int
def git_short_sha(cwd: str) -> str:
return run_capture(["git", "rev-parse", "--short=7", "HEAD"], cwd=cwd).strip()
def build_commands(go_dir: str, output_dir: str) -> list[BuildCommand]:
commands: list[BuildCommand] = []
env = ["CGO_ENABLED=0", "GOOS=linux", "GOARCH=amd64"]
for output_name, package in BINARIES:
commands.append(BuildCommand(
output_name=output_name,
argv=[
"go",
"build",
"-trimpath",
"-ldflags=-s -w",
"-o",
str(pathlib.Path(output_dir) / output_name),
package,
],
env=env,
cwd=go_dir,
))
return commands
def build_release(go_dir: str, output_dir: str) -> None:
for command in build_commands(go_dir, output_dir):
env = os.environ.copy()
for item in command.env:
key, value = item.split("=", 1)
env[key] = value
run(command.argv, cwd=command.cwd, env=env)
def create_archive(source_dir: str, archive_path: str) -> None:
with tarfile.open(archive_path, "w:gz") as archive:
for binary, _ in BINARIES:
archive.add(pathlib.Path(source_dir) / binary, arcname=binary)
def remote_switch_command(release: str, release_root: str, remote_archive: str) -> str:
if not safe_release_name(release):
raise ValueError("release must contain only letters, numbers, dot, underscore, or dash")
release_dir = release_root.rstrip("/") + "/releases/" + release
root = release_root.rstrip("/")
commands = [
"set -euo pipefail",
"mkdir -p " + shell_quote(release_dir),
"tar -xzf " + shell_quote(remote_archive) + " -C " + shell_quote(release_dir),
"chmod +x " + shell_quote(release_dir) + "/*",
"ln -sfn " + shell_quote(release_dir) + " " + shell_quote(root + "/current"),
"systemctl daemon-reload",
]
commands.extend("systemctl restart " + service for service in SERVICES)
commands.append("rm -f " + shell_quote(remote_archive))
return " && ".join(commands)
def acceptance_command(app_host: str, kafka_host: str, date: str, timeout: float) -> list[str]:
command = [
"python3",
"tools/go_prod_acceptance.py",
"--app-host",
app_host,
"--kafka-host",
kafka_host,
"--timeout",
str(int(timeout) if timeout == int(timeout) else timeout),
]
if date:
command.extend(["--date", date])
return command
def deploy(args: argparse.Namespace) -> None:
repo = pathlib.Path(args.repo).resolve()
go_dir = str(repo / args.go_dir)
release = args.release or git_short_sha(str(repo))
with tempfile.TemporaryDirectory(prefix="lingniu-go-native-") as tmp:
output_dir = pathlib.Path(tmp) / "out"
output_dir.mkdir()
archive_path = pathlib.Path(tmp) / ("lingniu-go-native-" + release + ".tar.gz")
build_release(go_dir, str(output_dir))
create_archive(str(output_dir), str(archive_path))
remote_archive = "/tmp/" + archive_path.name
target = ssh_target(args.user, args.app_host)
run(["scp", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", str(archive_path), target + ":" + remote_archive])
run([
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
target,
remote_switch_command(release, args.release_root, remote_archive),
])
wait_for_spool_drain(target, args.release_root, args.spool_drain_timeout, args.spool_poll_interval)
if not args.skip_acceptance:
run(acceptance_command(args.app_host, args.kafka_host, args.date, args.timeout), cwd=str(repo))
def wait_for_spool_drain(target: str, release_root: str, timeout: float, poll_interval: float) -> None:
deadline = time.monotonic() + timeout
last = SpoolStatus(files=-1, recent=-1)
while time.monotonic() <= deadline:
output = run_capture([
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
target,
spool_status_command(release_root),
])
last = parse_spool_status(output)
print(f"spool files={last.files} recent={last.recent}", flush=True)
if last.files == 0:
return
time.sleep(max(1.0, poll_interval))
raise RuntimeError(f"gateway spool did not drain: files={last.files} recent={last.recent}")
def spool_status_command(release_root: str) -> str:
spool_dir = release_root.rstrip("/") + "/spool/gateway"
return (
"spool=" + shell_quote(spool_dir) + "; "
"files=$(find \"$spool\" -type f 2>/dev/null | wc -l); "
"recent=$(find \"$spool\" -type f -mmin -5 2>/dev/null | wc -l); "
"printf 'files=%s recent=%s\\n' \"$files\" \"$recent\""
)
def parse_spool_status(output: str) -> SpoolStatus:
values: dict[str, int] = {}
for part in output.split():
if "=" not in part:
continue
key, value = part.split("=", 1)
try:
values[key] = int(value)
except ValueError:
continue
return SpoolStatus(files=values.get("files", -1), recent=values.get("recent", -1))
def ssh_target(user: str, host: str) -> str:
return user + "@" + host if user else host
def shell_quote(value: str) -> str:
return "'" + value.replace("'", "'\"'\"'") + "'"
def safe_release_name(value: str) -> bool:
value = value.strip()
if not value:
return False
return all(r.isalnum() or r in "._-" for r in value)
def run(argv: list[str], cwd: str | None = None, env: dict[str, str] | None = None) -> None:
subprocess.run(argv, cwd=cwd, env=env, check=True)
def run_capture(argv: list[str], cwd: str | None = None) -> str:
return subprocess.run(argv, cwd=cwd, check=True, stdout=subprocess.PIPE, text=True).stdout
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", default=".")
parser.add_argument("--go-dir", default=DEFAULT_GO_DIR)
parser.add_argument("--app-host", default=DEFAULT_APP_HOST)
parser.add_argument("--kafka-host", default=DEFAULT_KAFKA_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--release-root", default=DEFAULT_RELEASE_ROOT)
parser.add_argument("--release", default="")
parser.add_argument("--date", default="")
parser.add_argument("--timeout", type=float, default=20.0)
parser.add_argument("--spool-drain-timeout", type=float, default=900.0)
parser.add_argument("--spool-poll-interval", type=float, default=10.0)
parser.add_argument("--skip-acceptance", action="store_true")
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
deploy(args)
except subprocess.CalledProcessError as exc:
print("command failed: " + " ".join(exc.cmd), file=sys.stderr)
return exc.returncode or 1
except (OSError, shutil.Error) as exc:
print("deploy failed: " + str(exc), file=sys.stderr)
return 1
except RuntimeError as exc:
print("deploy failed: " + str(exc), file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

544
tools/go_native_prod_smoke.py Executable file
View File

@@ -0,0 +1,544 @@
#!/usr/bin/env python3
"""HTTP smoke checks for the Go native production vehicle ingest stack."""
from __future__ import annotations
import argparse
import datetime as dt
import json
import sys
import urllib.parse
import urllib.request
from dataclasses import asdict, dataclass
from typing import Any
SHANGHAI = dt.timezone(dt.timedelta(hours=8))
DEFAULT_BASE_URL = "http://115.29.187.205:20210"
DEFAULT_REALTIME_MAX_AGE_MINUTES = 15.0
@dataclass(frozen=True)
class CheckSpec:
name: str
path: str
params: dict[str, Any]
minimum: int
kind: str = "total"
max_age_minutes: float | None = None
require_parsed_json: bool = False
require_frame_id: bool = False
require_metric_formula: bool = False
@dataclass(frozen=True)
class Check:
name: str
status: str
count: int
minimum: int
message: str
@dataclass(frozen=True)
class CheckWindow:
date_from: str
date_to: str
stat_date: str
max_raw_age_minutes: float | None
def shanghai_day_window(now: dt.datetime | None = None) -> tuple[str, str]:
now = now or dt.datetime.now(tz=SHANGHAI)
local = now.astimezone(SHANGHAI)
start = dt.datetime(local.year, local.month, local.day, tzinfo=SHANGHAI)
end = start + dt.timedelta(days=1)
return start.isoformat(timespec="seconds"), end.isoformat(timespec="seconds")
def resolve_check_window(
raw_date: str | None,
max_raw_age_minutes: float,
now: dt.datetime | None = None,
) -> CheckWindow:
current = (now or dt.datetime.now(tz=SHANGHAI)).astimezone(SHANGHAI)
today = current.date()
if raw_date:
day = dt.date.fromisoformat(raw_date)
start = dt.datetime(day.year, day.month, day.day, tzinfo=SHANGHAI)
date_from, date_to = shanghai_day_window(start)
freshness = max_raw_age_minutes if day == today else None
return CheckWindow(date_from, date_to, raw_date, freshness)
date_from, date_to = shanghai_day_window(current)
return CheckWindow(date_from, date_to, date_from[:10], max_raw_age_minutes)
def api_url(base_url: str, path: str, params: dict[str, Any]) -> str:
url = base_url.rstrip("/") + path
if not params:
return url
return url + "?" + urllib.parse.urlencode(params)
def query_json(base_url: str, path: str, params: dict[str, Any], timeout: float) -> dict[str, Any]:
request = urllib.request.Request(
api_url(base_url, path, params),
headers={"Accept": "application/json"},
)
with urllib.request.urlopen(request, timeout=timeout) as response:
return json.loads(response.read().decode("utf-8"))
def check_total(
name: str,
payload: dict[str, Any],
minimum: int,
max_age_minutes: float | None = None,
now: dt.datetime | None = None,
require_parsed_json: bool = False,
require_frame_id: bool = False,
require_metric_formula: bool = False,
) -> Check:
count = int(payload.get("total") or 0)
items = payload.get("items") or []
sample = sample_summary(items)
age_message = latest_age_message(items, now)
if count >= minimum:
parsed_message = ""
if max_age_minutes is not None:
latest_age = latest_age_minutes(items, now)
if latest_age is None:
return Check(name, "fail", count, minimum, f"count={count}; missing latest ts; sample={sample}")
if latest_age > max_age_minutes:
return Check(
name,
"fail",
count,
minimum,
f"count={count}; latest_age_minutes={latest_age:.1f}; "
f"expected <= {max_age_minutes:.1f}; sample={sample}",
)
if require_parsed_json:
parsed_message = parsed_json_message(items)
if not parsed_message.startswith("parsed_json=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {parsed_message}; sample={sample}")
frame_message = ""
if require_frame_id:
frame_message = frame_id_message(items)
if not frame_message.startswith("frame_id=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {frame_message}; sample={sample}")
metric_message = ""
if require_metric_formula:
metric_message = metric_formula_message(items)
if not metric_message.startswith("metric_formula=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {metric_message}; sample={sample}")
details = [f"count={count}", age_message]
if parsed_message:
details.append(parsed_message)
if frame_message:
details.append(frame_message)
if metric_message:
details.append(metric_message)
return Check(name, "pass", count, minimum, "; ".join(details) + f"; sample={sample}")
return Check(name, "fail", count, minimum, f"count={count}; expected >= {minimum}; {age_message}; sample={sample}")
def parse_tdengine_utc_timestamp(value: str) -> dt.datetime:
value = str(value or "").strip()
for layout in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S"):
try:
return dt.datetime.strptime(value[:19], layout).replace(tzinfo=dt.timezone.utc)
except ValueError:
continue
raise ValueError(f"unsupported timestamp: {value}")
def latest_age_minutes(items: list[Any], now: dt.datetime | None = None) -> float | None:
if not items or not isinstance(items[0], dict):
return None
timestamp = items[0].get("received_at") or items[0].get("ts")
if not timestamp:
return None
current = now or dt.datetime.now(tz=dt.timezone.utc)
if current.tzinfo is None:
current = current.replace(tzinfo=dt.timezone.utc)
latest = parse_tdengine_utc_timestamp(str(timestamp))
return max(0.0, (current.astimezone(dt.timezone.utc) - latest).total_seconds() / 60)
def latest_age_message(items: list[Any], now: dt.datetime | None = None) -> str:
latest_age = latest_age_minutes(items, now)
if latest_age is None:
return "latest_age_minutes=unknown"
return f"latest_age_minutes={latest_age:.1f}"
def parsed_json_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing parsed_json"
raw = items[0].get("parsed_json")
if isinstance(raw, dict) and raw:
return "parsed_json=ok"
if not isinstance(raw, str) or not raw.strip() or raw.strip().lower() == "null":
return "missing parsed_json"
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return "invalid parsed_json"
if isinstance(parsed, dict) and parsed:
return "parsed_json=ok"
return "invalid parsed_json"
def frame_id_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing frame_id"
frame_id = str(items[0].get("frame_id") or "").strip()
if frame_id:
return "frame_id=ok"
return "missing frame_id"
def metric_formula_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing metric row"
row = items[0]
try:
metric_value = float(row.get("metric_value"))
first_total = float(row.get("first_total_mileage_km"))
latest_total = float(row.get("latest_total_mileage_km"))
except (TypeError, ValueError):
return "missing metric formula fields"
metric_key = str(row.get("metric_key") or "")
if metric_key == "daily_mileage_km":
expected = max(0.0, latest_total - first_total)
elif metric_key == "daily_total_mileage_km":
expected = latest_total
else:
return "unsupported metric_key=" + metric_key
if abs(metric_value - expected) <= 0.001:
return "metric_formula=ok"
return f"metric_formula mismatch value={metric_value:.3f} expected={expected:.3f}"
def check_realtime(
name: str,
payload: dict[str, Any],
max_age_minutes: float | None = None,
now: dt.datetime | None = None,
) -> Check:
if payload.get("online") is False:
return Check(name, "fail", 0, 1, "online=false")
updated_message = ""
if max_age_minutes is not None:
updated_age = realtime_updated_age_minutes(payload, now)
if updated_age is None:
return Check(name, "fail", 0, 1, "missing updated_at_ms")
updated_message = f"updated_age_minutes={updated_age:.1f}"
if updated_age > max_age_minutes:
return Check(
name,
"fail",
0,
1,
updated_message + f"; expected <= {max_age_minutes:.1f}",
)
identifier = payload.get("vin") or payload.get("vehicle_key") or ""
protocols = payload.get("protocols") or []
protocol = payload.get("protocol") or ""
fields = payload.get("fields") or {}
message = json.dumps(
{
key: value
for key, value in {
"identifier": identifier,
"protocol": protocol,
"protocols": protocols,
"field_count": len(fields) if isinstance(fields, dict) else 0,
"online": payload.get("online"),
}.items()
if value not in (None, "", [])
},
ensure_ascii=False,
sort_keys=True,
)
if updated_message:
message = updated_message + "; " + message
return Check(name, "pass", 1, 1, message)
def realtime_updated_age_minutes(payload: dict[str, Any], now: dt.datetime | None = None) -> float | None:
try:
updated_ms = int(payload.get("updated_at_ms"))
except (TypeError, ValueError):
return None
current = now or dt.datetime.now(tz=dt.timezone.utc)
if current.tzinfo is None:
current = current.replace(tzinfo=dt.timezone.utc)
updated = dt.datetime.fromtimestamp(updated_ms / 1000, tz=dt.timezone.utc)
return max(0.0, (current.astimezone(dt.timezone.utc) - updated).total_seconds() / 60)
def sample_summary(items: list[Any]) -> str:
if not items:
return "{}"
first = items[0]
if not isinstance(first, dict):
return json.dumps(first, ensure_ascii=False, sort_keys=True)
keep = {
key: first.get(key)
for key in (
"ts",
"stat_date",
"protocol",
"vehicle_key",
"vin",
"phone",
"message_id_hex",
"metric_key",
"metric_value",
"total_mileage_km",
"latest_total_mileage_km",
)
if first.get(key) not in (None, "")
}
return json.dumps(keep, ensure_ascii=False, sort_keys=True)
def overall_status(checks: list[Check]) -> str:
return "fail" if any(check.status == "fail" for check in checks) else "pass"
def build_check_specs(
*,
date_from: str,
date_to: str,
stat_date: str,
min_raw: int,
min_history: int,
min_stat: int,
max_raw_age_minutes: float | None = None,
) -> list[CheckSpec]:
raw_params = {"limit": 1, "orderBy": "receivedAt", "includeTotal": "false"}
gb32960_history_params = {"protocol": "GB32960", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
jt808_history_params = {"protocol": "JT808", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
yutong_mqtt_history_params = {"protocol": "YUTONG_MQTT", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
return [
CheckSpec(
"gb32960.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "GB32960"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec(
"jt808.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "JT808"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec(
"yutong_mqtt.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "YUTONG_MQTT"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec("gb32960.locations", "/api/history/locations", gb32960_history_params, min_history, require_frame_id=True),
CheckSpec("gb32960.mileage_points", "/api/history/mileage-points", gb32960_history_params, min_history, require_frame_id=True),
CheckSpec("jt808.locations", "/api/history/locations", jt808_history_params, min_history, require_frame_id=True),
CheckSpec("jt808.mileage_points", "/api/history/mileage-points", jt808_history_params, min_history, require_frame_id=True),
CheckSpec("yutong_mqtt.locations", "/api/history/locations", yutong_mqtt_history_params, min_history, require_frame_id=True),
CheckSpec("yutong_mqtt.mileage_points", "/api/history/mileage-points", yutong_mqtt_history_params, min_history, require_frame_id=True),
CheckSpec(
"gb32960.daily_mileage",
"/api/stats/daily-metrics",
{
"protocol": "GB32960",
"metricKey": "daily_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"gb32960.daily_total_mileage",
"/api/stats/daily-metrics",
{
"protocol": "GB32960",
"metricKey": "daily_total_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"jt808.daily_mileage",
"/api/stats/daily-metrics",
{
"protocol": "JT808",
"metricKey": "daily_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"jt808.daily_total_mileage",
"/api/stats/daily-metrics",
{
"protocol": "JT808",
"metricKey": "daily_total_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
]
def vehicle_identifier(item: dict[str, Any]) -> str:
for key in ("vin", "vehicle_key"):
value = str(item.get(key) or "").strip()
if value and value.lower() != "unknown":
return value
return ""
def build_realtime_specs(payloads: dict[str, dict[str, Any]]) -> list[CheckSpec]:
configs = [
("gb32960", "gb32960.raw", "GB32960"),
("jt808", "jt808.raw", "JT808"),
("yutong_mqtt", "yutong_mqtt.raw", "YUTONG_MQTT"),
]
specs: list[CheckSpec] = []
for prefix, source_name, protocol in configs:
items = payloads.get(source_name, {}).get("items") or []
if not items or not isinstance(items[0], dict):
continue
identifier = vehicle_identifier(items[0])
if not identifier:
continue
encoded = urllib.parse.quote(identifier, safe="")
base_path = "/api/realtime/vehicles/" + encoded
specs.extend([
CheckSpec(
prefix + ".realtime",
base_path,
{},
1,
"realtime",
max_age_minutes=DEFAULT_REALTIME_MAX_AGE_MINUTES,
),
CheckSpec(prefix + ".realtime_online", base_path + "/online", {}, 1, "realtime"),
CheckSpec(
prefix + ".realtime_protocol",
base_path + "/protocols/" + protocol,
{},
1,
"realtime",
max_age_minutes=DEFAULT_REALTIME_MAX_AGE_MINUTES,
),
])
return specs
def run_checks(base_url: str, specs: list[CheckSpec], timeout: float) -> list[Check]:
checks: list[Check] = []
for spec in specs:
try:
payload = query_json(base_url, spec.path, spec.params, timeout)
if spec.kind == "realtime":
checks.append(check_realtime(spec.name, payload, spec.max_age_minutes))
else:
checks.append(check_total(
spec.name,
payload,
spec.minimum,
spec.max_age_minutes,
require_parsed_json=spec.require_parsed_json,
require_frame_id=spec.require_frame_id,
require_metric_formula=spec.require_metric_formula,
))
except Exception as exc: # pragma: no cover - exercised by live failures.
checks.append(Check(spec.name, "fail", 0, spec.minimum, f"request failed: {exc}"))
return checks
def query_payloads(base_url: str, specs: list[CheckSpec], timeout: float) -> tuple[list[Check], dict[str, dict[str, Any]]]:
checks: list[Check] = []
payloads: dict[str, dict[str, Any]] = {}
for spec in specs:
try:
payload = query_json(base_url, spec.path, spec.params, timeout)
payloads[spec.name] = payload
checks.append(check_total(
spec.name,
payload,
spec.minimum,
spec.max_age_minutes,
require_parsed_json=spec.require_parsed_json,
require_frame_id=spec.require_frame_id,
require_metric_formula=spec.require_metric_formula,
))
except Exception as exc:
checks.append(Check(spec.name, "fail", 0, spec.minimum, f"request failed: {exc}"))
return checks, payloads
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--date", help="Shanghai local date, YYYY-MM-DD. Defaults to today.")
parser.add_argument("--timeout", type=float, default=5.0)
parser.add_argument("--min-raw", type=int, default=1)
parser.add_argument("--min-history", type=int, default=1)
parser.add_argument("--min-stat", type=int, default=1)
parser.add_argument(
"--max-raw-age-minutes",
type=float,
default=15.0,
help="Require latest RAW sample age to be at most this many minutes when checking today's date.",
)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
window = resolve_check_window(args.date, args.max_raw_age_minutes)
specs = build_check_specs(
date_from=window.date_from,
date_to=window.date_to,
stat_date=window.stat_date,
min_raw=args.min_raw,
min_history=args.min_history,
min_stat=args.min_stat,
max_raw_age_minutes=window.max_raw_age_minutes,
)
checks, payloads = query_payloads(args.base_url, specs, args.timeout)
checks.extend(run_checks(args.base_url, build_realtime_specs(payloads), args.timeout))
status = overall_status(checks)
print(json.dumps({
"status": status,
"baseUrl": args.base_url,
"dateFrom": window.date_from,
"dateTo": window.date_to,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

136
tools/go_prod_acceptance.py Executable file
View File

@@ -0,0 +1,136 @@
#!/usr/bin/env python3
"""Run the Go native production acceptance smoke suite."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
from typing import Any
DEFAULT_APP_HOST = "115.29.187.205"
DEFAULT_KAFKA_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_BASE_URL = "http://115.29.187.205:20210"
@dataclass(frozen=True)
class ChildCommand:
name: str
argv: list[str]
@dataclass(frozen=True)
class ChildResult:
name: str
status: str
exit_code: int
payload: dict[str, Any]
def build_commands(args: argparse.Namespace) -> list[ChildCommand]:
timeout = str(args.timeout)
return [
ChildCommand("systemd", [
sys.executable,
"tools/go_systemd_prod_smoke.py",
"--host",
args.app_host,
"--user",
args.ssh_user,
"--timeout",
timeout,
]),
ChildCommand("kafka", [
sys.executable,
"tools/go_kafka_prod_smoke.py",
"--host",
args.kafka_host,
"--user",
args.ssh_user,
"--max-lag",
str(args.max_lag),
"--timeout",
timeout,
]),
ChildCommand("http", [
sys.executable,
"tools/go_native_prod_smoke.py",
"--base-url",
args.base_url,
"--timeout",
timeout,
] + (["--date", args.date] if args.date else [])),
]
def run_child(command: ChildCommand, timeout: float) -> ChildResult:
completed = subprocess.run(
command.argv,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
payload = parse_json_output(completed.stdout)
if completed.stderr.strip():
payload.setdefault("stderr", completed.stderr.strip())
status = payload.get("status")
if status not in {"pass", "fail"}:
status = "pass" if completed.returncode == 0 else "fail"
return ChildResult(command.name, status, completed.returncode, payload)
def parse_json_output(output: str) -> dict[str, Any]:
output = output.strip()
if not output:
return {}
start = output.find("{")
end = output.rfind("}")
if start < 0 or end < start:
return {"rawOutput": output}
try:
return json.loads(output[start:end + 1])
except json.JSONDecodeError:
return {"rawOutput": output}
def overall_status(results: list[ChildResult]) -> str:
if not results:
return "fail"
return "fail" if any(result.status != "pass" or result.exit_code != 0 for result in results) else "pass"
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--app-host", default=DEFAULT_APP_HOST)
parser.add_argument("--kafka-host", default=DEFAULT_KAFKA_HOST)
parser.add_argument("--ssh-user", default=DEFAULT_USER)
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--date", help="Shanghai local date, YYYY-MM-DD. Defaults to today in go_native_prod_smoke.py.")
parser.add_argument("--timeout", type=float, default=20.0)
parser.add_argument("--max-lag", type=int, default=100)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
results: list[ChildResult] = []
for command in build_commands(args):
try:
results.append(run_child(command, args.timeout + 10))
except Exception as exc:
results.append(ChildResult(command.name, "fail", 1, {"status": "fail", "error": str(exc)}))
status = overall_status(results)
print(json.dumps({
"status": status,
"checks": [asdict(result) for result in results],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

210
tools/go_systemd_prod_smoke.py Executable file
View File

@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""Systemd and port smoke checks for the Go native production ECS."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
DEFAULT_HOST = "115.29.187.205"
DEFAULT_USER = "root"
DEFAULT_SERVICES = [
"lingniu-go-gateway.service",
"lingniu-go-history-writer.service",
"lingniu-go-stat-writer.service",
"lingniu-go-realtime-api.service",
]
DEFAULT_PORTS = {
808: "gateway",
32960: "gateway",
20210: "realtime-api",
}
DEFAULT_RELEASE_ROOT = "/opt/lingniu-go-native"
DEFAULT_SPOOL_DIR = "/opt/lingniu-go-native/spool/gateway"
DEFAULT_BINARIES = ["gateway", "history-writer", "stat-writer", "realtime-api"]
@dataclass(frozen=True)
class Check:
name: str
status: str
message: str
@dataclass(frozen=True)
class ServiceState:
active: str
enabled: str
def parse_service_states(output: str) -> dict[str, ServiceState]:
states: dict[str, ServiceState] = {}
for line in output.splitlines():
parts = line.split()
if len(parts) >= 2:
enabled = parts[2] if len(parts) >= 3 else "enabled"
states[parts[0]] = ServiceState(parts[1], enabled)
return states
def service_checks(output: str, services: list[str]) -> list[Check]:
states = parse_service_states(output)
checks: list[Check] = []
for service in services:
state = states.get(service, ServiceState("missing", "missing"))
status = "pass" if state.active == "active" and state.enabled == "enabled" else "fail"
checks.append(Check(
"service." + service,
status,
"state=" + state.active + "; enabled=" + state.enabled,
))
return checks
def port_checks(output: str, ports: dict[int, str]) -> list[Check]:
lines = output.splitlines()
checks: list[Check] = []
for port, process in sorted(ports.items()):
matching = [line.strip() for line in lines if has_port(line, port)]
if not matching:
checks.append(Check("port." + str(port), "fail", "not listening"))
continue
owned = [line for line in matching if process in line]
if owned:
checks.append(Check("port." + str(port), "pass", owned[0]))
else:
checks.append(Check("port." + str(port), "fail", matching[0]))
return checks
def has_port(line: str, port: int) -> bool:
needle = ":" + str(port)
return needle + " " in line or needle + "\t" in line
def spool_check(output: str) -> Check:
values = parse_key_values(output)
files = int(values.get("files", "-1"))
recent = int(values.get("recent", "-1"))
status = "pass" if files == 0 and recent == 0 else "fail"
return Check("spool.gateway", status, output.strip() or "missing")
def release_check(output: str) -> Check:
values = parse_key_values(output)
current = values.get("current", "")
binaries_present = [values.get(binary, "0") == "1" for binary in DEFAULT_BINARIES]
status = "pass" if current and all(binaries_present) else "fail"
return Check("release.current", status, output.strip() or "missing")
def parse_key_values(output: str) -> dict[str, str]:
values: dict[str, str] = {}
for part in output.split():
if "=" not in part:
continue
key, value = part.split("=", 1)
values[key] = value
return values
def ssh(host: str, user: str, command: str, timeout: float) -> str:
target = user + "@" + host if user else host
completed = subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", target, command],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
return completed.stdout
def remote_command(services: list[str]) -> str:
quoted = " ".join(services)
return (
"for svc in " + quoted + "; do "
"printf '%s ' \"$svc\"; "
"printf '%s ' \"$(systemctl is-active \"$svc\" 2>/dev/null || true)\"; "
"systemctl is-enabled \"$svc\" 2>/dev/null || true; "
"done; "
"printf '\\n--PORTS--\\n'; "
"ss -lntp; "
"printf '\\n--SPOOL--\\n'; "
"spool='" + DEFAULT_SPOOL_DIR + "'; "
"if test -d \"$spool\"; then "
"files=$(find \"$spool\" -type f | wc -l); "
"bytes=$(du -sb \"$spool\" 2>/dev/null | awk '{print $1}'); "
"recent=$(find \"$spool\" -type f -mmin -5 | wc -l); "
"printf 'files=%s bytes=%s recent=%s\\n' \"$files\" \"${bytes:-0}\" \"$recent\"; "
"else printf 'files=-1 bytes=0 recent=-1 missing=%s\\n' \"$spool\"; fi; "
"printf '\\n--RELEASE--\\n'; "
"root='" + DEFAULT_RELEASE_ROOT + "'; "
"current=$(readlink -f \"$root/current\" 2>/dev/null || true); "
"printf 'current=%s' \"$current\"; "
"for bin in " + " ".join(DEFAULT_BINARIES) + "; do "
"if test -x \"$root/current/$bin\"; then present=1; else present=0; fi; "
"printf ' %s=%s' \"$bin\" \"$present\"; "
"done; printf '\\n'"
)
def split_remote_output(output: str) -> tuple[str, str, str, str]:
ports_marker = "\n--PORTS--\n"
spool_marker = "\n--SPOOL--\n"
release_marker = "\n--RELEASE--\n"
if ports_marker not in output:
return output, "", "", ""
service_output, rest = output.split(ports_marker, 1)
if spool_marker not in rest:
return service_output, rest, "", ""
port_output, spool_output = rest.split(spool_marker, 1)
if release_marker not in spool_output:
return service_output, port_output, spool_output, ""
spool_output, release_output = spool_output.split(release_marker, 1)
return service_output, port_output, spool_output, release_output
def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
output = ssh(args.host, args.user, remote_command(DEFAULT_SERVICES), args.timeout)
service_output, port_output, spool_output, release_output = split_remote_output(output)
checks = (
service_checks(service_output, DEFAULT_SERVICES)
+ port_checks(port_output, DEFAULT_PORTS)
+ [spool_check(spool_output)]
+ [release_check(release_output)]
)
status = "fail" if any(check.status == "fail" for check in checks) else "pass"
return status, checks
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--timeout", type=float, default=20.0)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
status, checks = run(args)
except Exception as exc:
status = "fail"
checks = [Check("systemd.ssh", "fail", str(exc))]
print(json.dumps({
"status": status,
"host": args.host,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

View File

@@ -0,0 +1,62 @@
import unittest
from tools import go_kafka_prod_smoke as smoke
class GoKafkaProdSmokeTest(unittest.TestCase):
def test_parse_topic_list_marks_required_topics_present(self):
topics = smoke.parse_topic_list("""
vehicle.raw.go.gb32960.v1
vehicle.raw.go.jt808.v1
vehicle.raw.go.yutong-mqtt.v1
vehicle.event.go.unified.v1
""")
checks = smoke.topic_checks(topics, smoke.DEFAULT_TOPICS)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_topic_checks_fail_when_required_topic_missing(self):
topics = {"vehicle.raw.go.gb32960.v1"}
checks = smoke.topic_checks(topics, ["vehicle.raw.go.gb32960.v1", "vehicle.event.go.unified.v1"])
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["topic.vehicle.event.go.unified.v1"].status, "fail")
def test_parse_consumer_group_describe_reads_lag_rows(self):
rows = smoke.parse_consumer_group_describe("""
GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG
go-realtime-api vehicle.event.go.unified.v1 0 15992 15992 0
go-realtime-api vehicle.event.go.unified.v1 1 2036 2036 3
""")
self.assertEqual(rows[0].group, "go-realtime-api")
self.assertEqual(rows[1].topic, "vehicle.event.go.unified.v1")
self.assertEqual(rows[1].lag, 3)
def test_lag_checks_fail_when_lag_exceeds_threshold(self):
rows = [
smoke.ConsumerLag("go-history-writer", "vehicle.raw.go.jt808.v1", 0, 100, 100, 0),
smoke.ConsumerLag("go-history-writer", "vehicle.raw.go.jt808.v1", 1, 100, 110, 10),
]
check = smoke.group_lag_check("go-history-writer", rows, max_lag=5)
self.assertEqual(check.status, "fail")
self.assertIn("max_lag=10", check.message)
def test_lag_checks_pass_when_group_has_rows_and_lag_is_small(self):
rows = [
smoke.ConsumerLag("go-stat-writer", "vehicle.raw.go.gb32960.v1", 0, 100, 100, 0),
smoke.ConsumerLag("go-stat-writer", "vehicle.raw.go.jt808.v1", 1, 100, 101, 1),
]
check = smoke.group_lag_check("go-stat-writer", rows, max_lag=5)
self.assertEqual(check.status, "pass")
self.assertIn("max_lag=1", check.message)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,63 @@
import unittest
from tools import go_native_deploy as deploy
class GoNativeDeployTest(unittest.TestCase):
def test_build_commands_cover_four_production_binaries(self):
commands = deploy.build_commands("/repo/go/vehicle-gateway", "/tmp/out")
self.assertEqual([command.output_name for command in commands], [
"gateway",
"history-writer",
"stat-writer",
"realtime-api",
])
for command in commands:
self.assertEqual(command.argv[:3], ["go", "build", "-trimpath"])
self.assertIn("GOOS=linux", command.env)
self.assertIn("GOARCH=amd64", command.env)
self.assertIn("CGO_ENABLED=0", command.env)
def test_remote_switch_command_creates_release_and_restarts_all_services(self):
command = deploy.remote_switch_command("409f55b", "/opt/lingniu-go-native", "/tmp/lingniu-go-native-409f55b.tar.gz")
self.assertIn("/opt/lingniu-go-native/releases/409f55b", command)
self.assertIn("ln -sfn", command)
self.assertIn("systemctl restart lingniu-go-gateway", command)
self.assertIn("systemctl restart lingniu-go-history-writer", command)
self.assertIn("systemctl restart lingniu-go-stat-writer", command)
self.assertIn("systemctl restart lingniu-go-realtime-api", command)
def test_acceptance_command_uses_deployed_hosts_and_date(self):
command = deploy.acceptance_command(
app_host="115.29.187.205",
kafka_host="114.55.58.251",
date="2026-07-02",
timeout=20,
)
self.assertEqual(command[:2], ["python3", "tools/go_prod_acceptance.py"])
self.assertIn("--app-host", command)
self.assertIn("115.29.187.205", command)
self.assertIn("--kafka-host", command)
self.assertIn("114.55.58.251", command)
self.assertIn("--date", command)
self.assertIn("2026-07-02", command)
def test_parse_spool_status_reads_file_and_recent_counts(self):
status = deploy.parse_spool_status("files=42 recent=3")
self.assertEqual(status.files, 42)
self.assertEqual(status.recent, 3)
def test_spool_status_command_targets_gateway_spool_directory(self):
command = deploy.spool_status_command("/opt/lingniu-go-native")
self.assertIn("/opt/lingniu-go-native/spool/gateway", command)
self.assertIn("files=", command)
self.assertIn("recent=", command)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,358 @@
import datetime as dt
import unittest
from tools import go_native_prod_smoke as smoke
class GoNativeProdSmokeTest(unittest.TestCase):
def test_shanghai_day_window_uses_plus_eight_bounds(self):
now = dt.datetime(2026, 7, 2, 1, 35, tzinfo=smoke.SHANGHAI)
date_from, date_to = smoke.shanghai_day_window(now)
self.assertEqual(date_from, "2026-07-02T00:00:00+08:00")
self.assertEqual(date_to, "2026-07-03T00:00:00+08:00")
def test_api_url_encodes_plus_eight_date_range(self):
url = smoke.api_url(
"http://example.test/base/",
"/api/history/raw-frames",
{
"protocol": "JT808",
"dateFrom": "2026-07-02T00:00:00+08:00",
"dateTo": "2026-07-03T00:00:00+08:00",
"limit": 1,
},
)
self.assertEqual(
url,
"http://example.test/base/api/history/raw-frames?"
"protocol=JT808&dateFrom=2026-07-02T00%3A00%3A00%2B08%3A00"
"&dateTo=2026-07-03T00%3A00%3A00%2B08%3A00&limit=1",
)
def test_explicit_today_date_still_enforces_raw_freshness(self):
now = dt.datetime(2026, 7, 2, 9, 30, tzinfo=smoke.SHANGHAI)
window = smoke.resolve_check_window("2026-07-02", 15.0, now)
self.assertEqual(window.date_from, "2026-07-02T00:00:00+08:00")
self.assertEqual(window.date_to, "2026-07-03T00:00:00+08:00")
self.assertEqual(window.stat_date, "2026-07-02")
self.assertEqual(window.max_raw_age_minutes, 15.0)
def test_explicit_historical_date_disables_raw_freshness(self):
now = dt.datetime(2026, 7, 2, 9, 30, tzinfo=smoke.SHANGHAI)
window = smoke.resolve_check_window("2026-07-01", 15.0, now)
self.assertEqual(window.date_from, "2026-07-01T00:00:00+08:00")
self.assertEqual(window.date_to, "2026-07-02T00:00:00+08:00")
self.assertEqual(window.stat_date, "2026-07-01")
self.assertIsNone(window.max_raw_age_minutes)
def test_total_check_passes_when_total_reaches_minimum(self):
check = smoke.check_total(
"jt808.raw",
{"total": 2, "items": [{"vehicle_key": "JT808:013307811170"}]},
minimum=1,
)
self.assertEqual(check.status, "pass")
self.assertEqual(check.count, 2)
self.assertIn("JT808:013307811170", check.message)
def test_total_check_fails_when_total_is_below_minimum(self):
check = smoke.check_total("gb32960.raw", {"total": 0, "items": []}, minimum=1)
self.assertEqual(check.status, "fail")
self.assertEqual(check.count, 0)
self.assertIn("expected >= 1", check.message)
def test_overall_status_fails_if_any_check_fails(self):
checks = [
smoke.Check("a", "pass", 1, 1, "ok"),
smoke.Check("b", "fail", 0, 1, "bad"),
]
self.assertEqual(smoke.overall_status(checks), "fail")
def test_build_checks_cover_three_raw_protocols_and_two_daily_metric_protocols(self):
specs = smoke.build_check_specs(
date_from="2026-07-02T00:00:00+08:00",
date_to="2026-07-03T00:00:00+08:00",
stat_date="2026-07-02",
min_raw=1,
min_history=1,
min_stat=1,
)
names = [spec.name for spec in specs]
self.assertIn("gb32960.raw", names)
self.assertIn("jt808.raw", names)
self.assertIn("yutong_mqtt.raw", names)
self.assertIn("gb32960.locations", names)
self.assertIn("gb32960.mileage_points", names)
self.assertIn("jt808.locations", names)
self.assertIn("jt808.mileage_points", names)
self.assertIn("yutong_mqtt.locations", names)
self.assertIn("yutong_mqtt.mileage_points", names)
self.assertIn("gb32960.daily_mileage", names)
self.assertIn("jt808.daily_total_mileage", names)
def test_raw_checks_order_by_received_at_for_ingest_freshness(self):
specs = smoke.build_check_specs(
date_from="2026-07-02T00:00:00+08:00",
date_to="2026-07-03T00:00:00+08:00",
stat_date="2026-07-02",
min_raw=1,
min_history=1,
min_stat=1,
max_raw_age_minutes=15,
)
raw_specs = [spec for spec in specs if spec.name.endswith(".raw")]
self.assertTrue(raw_specs)
for spec in raw_specs:
self.assertEqual(spec.params.get("orderBy"), "receivedAt")
self.assertEqual(spec.params.get("includeTotal"), "false")
self.assertNotIn("dateFrom", spec.params)
self.assertNotIn("dateTo", spec.params)
def test_vehicle_identifier_prefers_vin_over_vehicle_key(self):
identifier = smoke.vehicle_identifier({
"vin": "LB9A32A20R0LS1343",
"vehicle_key": "GB32960:ignored",
})
self.assertEqual(identifier, "LB9A32A20R0LS1343")
def test_vehicle_identifier_falls_back_to_vehicle_key(self):
identifier = smoke.vehicle_identifier({
"vin": "",
"vehicle_key": "JT808:013307811170",
})
self.assertEqual(identifier, "JT808:013307811170")
def test_realtime_specs_are_built_from_raw_samples_with_vin(self):
payloads = {
"gb32960.raw": {"items": [{"vin": "LB9A32A20R0LS1343", "vehicle_key": "LB9A32A20R0LS1343"}]},
"jt808.raw": {"items": [{"vin": "", "vehicle_key": "JT808:013307811170"}]},
"yutong_mqtt.raw": {"items": [{"vin": "LMRKH9AC6R1004108"}]},
}
specs = smoke.build_realtime_specs(payloads)
self.assertEqual(
[(spec.name, spec.path) for spec in specs],
[
("gb32960.realtime", "/api/realtime/vehicles/LB9A32A20R0LS1343"),
("gb32960.realtime_online", "/api/realtime/vehicles/LB9A32A20R0LS1343/online"),
("gb32960.realtime_protocol", "/api/realtime/vehicles/LB9A32A20R0LS1343/protocols/GB32960"),
("jt808.realtime", "/api/realtime/vehicles/JT808%3A013307811170"),
("jt808.realtime_online", "/api/realtime/vehicles/JT808%3A013307811170/online"),
("jt808.realtime_protocol", "/api/realtime/vehicles/JT808%3A013307811170/protocols/JT808"),
("yutong_mqtt.realtime", "/api/realtime/vehicles/LMRKH9AC6R1004108"),
("yutong_mqtt.realtime_online", "/api/realtime/vehicles/LMRKH9AC6R1004108/online"),
("yutong_mqtt.realtime_protocol", "/api/realtime/vehicles/LMRKH9AC6R1004108/protocols/YUTONG_MQTT"),
],
)
def test_realtime_check_requires_online_true_when_field_exists(self):
check = smoke.check_realtime(
"gb32960.realtime_online",
{"vin": "LB9A32A20R0LS1343", "online": True, "protocols": ["GB32960"]},
)
self.assertEqual(check.status, "pass")
self.assertEqual(check.count, 1)
def test_realtime_check_fails_when_online_false(self):
check = smoke.check_realtime(
"gb32960.realtime_online",
{"vin": "LB9A32A20R0LS1343", "online": False},
)
self.assertEqual(check.status, "fail")
self.assertIn("online=false", check.message)
def test_realtime_check_fails_when_snapshot_is_stale(self):
now = dt.datetime(2026, 7, 2, 0, 30, 0, tzinfo=dt.timezone.utc)
check = smoke.check_realtime(
"jt808.realtime",
{"vehicle_key": "JT808:013307811170", "updated_at_ms": 1782950400000},
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "fail")
self.assertIn("updated_age_minutes=30.0", check.message)
def test_realtime_check_passes_when_snapshot_is_fresh(self):
now = dt.datetime(2026, 7, 2, 0, 30, 0, tzinfo=dt.timezone.utc)
check = smoke.check_realtime(
"jt808.realtime",
{"vehicle_key": "JT808:013307811170", "updated_at_ms": 1782951600000},
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("updated_age_minutes=10.0", check.message)
def test_parse_tdengine_utc_timestamp_as_aware_utc(self):
parsed = smoke.parse_tdengine_utc_timestamp("2026-07-01 17:36:02")
self.assertEqual(parsed, dt.datetime(2026, 7, 1, 17, 36, 2, tzinfo=dt.timezone.utc))
def test_total_check_fails_when_latest_sample_is_stale(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{"total": 1, "items": [{"ts": "2026-07-01 17:00:00"}]},
minimum=1,
max_age_minutes=30,
now=now,
)
self.assertEqual(check.status, "fail")
self.assertIn("latest_age_minutes=60.0", check.message)
def test_total_check_uses_received_at_for_raw_freshness_when_present(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:00:00",
"received_at": "2026-07-01 17:58:00",
}],
},
minimum=1,
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("latest_age_minutes=2.0", check.message)
def test_total_check_passes_when_latest_sample_is_fresh(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00"}]},
minimum=1,
max_age_minutes=30,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("latest_age_minutes=15.0", check.message)
def test_raw_check_requires_structured_parsed_json(self):
check = smoke.check_total(
"jt808.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:45:00",
"parsed_json": "",
}],
},
minimum=1,
require_parsed_json=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("missing parsed_json", check.message)
def test_raw_check_accepts_structured_parsed_json(self):
check = smoke.check_total(
"gb32960.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:45:00",
"parsed_json": "{\"header\":{\"vin\":\"LTEST\"}}",
}],
},
minimum=1,
require_parsed_json=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("parsed_json=ok", check.message)
def test_history_check_requires_frame_id_backlink(self):
check = smoke.check_total(
"jt808.locations",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00", "frame_id": ""}]},
minimum=1,
require_frame_id=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("missing frame_id", check.message)
def test_history_check_accepts_frame_id_backlink(self):
check = smoke.check_total(
"jt808.mileage_points",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00", "frame_id": "go_abc"}]},
minimum=1,
require_frame_id=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("frame_id=ok", check.message)
def test_daily_mileage_formula_requires_latest_minus_first(self):
check = smoke.check_total(
"jt808.daily_mileage",
{
"total": 1,
"items": [{
"metric_key": "daily_mileage_km",
"metric_value": 40.0,
"first_total_mileage_km": 4434.9,
"latest_total_mileage_km": 4481.0,
}],
},
minimum=1,
require_metric_formula=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("metric_formula mismatch", check.message)
def test_daily_total_formula_requires_latest_total(self):
check = smoke.check_total(
"jt808.daily_total_mileage",
{
"total": 1,
"items": [{
"metric_key": "daily_total_mileage_km",
"metric_value": 4481.0,
"first_total_mileage_km": 4434.9,
"latest_total_mileage_km": 4481.0,
}],
},
minimum=1,
require_metric_formula=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("metric_formula=ok", check.message)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,53 @@
import argparse
import sys
import unittest
from tools import go_prod_acceptance as acceptance
class GoProdAcceptanceTest(unittest.TestCase):
def test_build_commands_runs_systemd_kafka_and_http_smokes(self):
args = argparse.Namespace(
app_host="115.29.187.205",
kafka_host="114.55.58.251",
ssh_user="root",
base_url="http://115.29.187.205:20210",
date="2026-07-02",
timeout=8.0,
max_lag=100,
)
commands = acceptance.build_commands(args)
self.assertEqual(len(commands), 3)
self.assertEqual(commands[0].name, "systemd")
self.assertEqual(commands[0].argv[:2], [sys.executable, "tools/go_systemd_prod_smoke.py"])
self.assertIn("--host", commands[0].argv)
self.assertIn("115.29.187.205", commands[0].argv)
self.assertEqual(commands[1].name, "kafka")
self.assertIn("tools/go_kafka_prod_smoke.py", commands[1].argv)
self.assertIn("--max-lag", commands[1].argv)
self.assertEqual(commands[2].name, "http")
self.assertIn("tools/go_native_prod_smoke.py", commands[2].argv)
self.assertIn("--base-url", commands[2].argv)
def test_overall_status_fails_when_any_child_fails(self):
results = [
acceptance.ChildResult("systemd", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("kafka", "fail", 1, {"status": "fail"}),
]
self.assertEqual(acceptance.overall_status(results), "fail")
def test_overall_status_passes_when_all_children_pass(self):
results = [
acceptance.ChildResult("systemd", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("kafka", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("http", "pass", 0, {"status": "pass"}),
]
self.assertEqual(acceptance.overall_status(results), "pass")
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,101 @@
import unittest
from tools import go_systemd_prod_smoke as smoke
class GoSystemdProdSmokeTest(unittest.TestCase):
def test_service_checks_require_all_go_units_active(self):
output = """
lingniu-go-gateway.service active enabled
lingniu-go-history-writer.service active enabled
lingniu-go-stat-writer.service active enabled
lingniu-go-realtime-api.service active enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_service_checks_fail_when_unit_is_missing_or_inactive(self):
output = """
lingniu-go-gateway.service active enabled
lingniu-go-history-writer.service failed enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["service.lingniu-go-history-writer.service"].status, "fail")
self.assertEqual(by_name["service.lingniu-go-stat-writer.service"].status, "fail")
def test_service_checks_fail_when_unit_is_not_enabled(self):
output = """
lingniu-go-gateway.service active disabled
lingniu-go-history-writer.service active enabled
lingniu-go-stat-writer.service active enabled
lingniu-go-realtime-api.service active enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["service.lingniu-go-gateway.service"].status, "fail")
self.assertIn("enabled=disabled", by_name["service.lingniu-go-gateway.service"].message)
def test_port_checks_require_expected_go_processes(self):
output = """
LISTEN 0 4096 *:32960 *:* users:(("gateway",pid=10,fd=3))
LISTEN 0 4096 *:808 *:* users:(("gateway",pid=10,fd=4))
LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
"""
checks = smoke.port_checks(output, smoke.DEFAULT_PORTS)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_port_checks_fail_when_java_owns_ingest_port(self):
output = """
LISTEN 0 4096 *:808 *:* users:(("java",pid=20,fd=4))
LISTEN 0 4096 *:32960 *:* users:(("gateway",pid=10,fd=3))
LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
"""
checks = smoke.port_checks(output, smoke.DEFAULT_PORTS)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["port.808"].status, "fail")
self.assertIn("java", by_name["port.808"].message)
def test_spool_check_passes_when_no_pending_files(self):
check = smoke.spool_check("files=0 bytes=208896 recent=0")
self.assertEqual(check.status, "pass")
self.assertIn("files=0", check.message)
def test_spool_check_fails_when_pending_or_recent_files_exist(self):
for output in ["files=2 bytes=300 recent=0", "files=0 bytes=208896 recent=1"]:
check = smoke.spool_check(output)
self.assertEqual(check.status, "fail")
def test_release_check_passes_when_current_release_has_all_binaries(self):
check = smoke.release_check(
"current=/opt/lingniu-go-native/releases/8def635 "
"gateway=1 history-writer=1 stat-writer=1 realtime-api=1"
)
self.assertEqual(check.status, "pass")
self.assertIn("8def635", check.message)
def test_release_check_fails_when_binary_is_missing(self):
check = smoke.release_check(
"current=/opt/lingniu-go-native/releases/8def635 "
"gateway=1 history-writer=0 stat-writer=1 realtime-api=1"
)
self.assertEqual(check.status, "fail")
self.assertIn("history-writer=0", check.message)
if __name__ == "__main__":
unittest.main()