78 Commits

Author SHA1 Message Date
lingniu
cbb6f3b741 perf: skip raw freshness count scan 2026-07-02 09:30:15 +08:00
lingniu
42de422935 test: check raw freshness without event date filter 2026-07-02 09:27:22 +08:00
lingniu
318486b1e9 fix: verify raw freshness by received time 2026-07-02 09:16:45 +08:00
lingniu
0a024f9a76 chore: wait for gateway spool before acceptance 2026-07-02 09:11:50 +08:00
lingniu
ea9d9f5d35 chore: add go native ecs deploy script 2026-07-02 08:50:19 +08:00
lingniu
409f55b5ae test: verify go native release layout 2026-07-02 08:47:05 +08:00
lingniu
8def635bf5 test: verify realtime snapshots are fresh 2026-07-02 08:43:27 +08:00
lingniu
c211447747 test: verify daily metric formulas 2026-07-02 08:38:32 +08:00
lingniu
1741aacaaa test: verify history rows link raw frames 2026-07-02 08:35:34 +08:00
lingniu
27f4ad3468 test: verify raw parsed json in production smoke 2026-07-02 02:00:50 +08:00
lingniu
601150b364 test: verify gateway spool is drained 2026-07-02 01:57:53 +08:00
lingniu
5b7f386c5a test: verify go services are enabled 2026-07-02 01:55:39 +08:00
lingniu
f4d9c9ef5a test: add go production acceptance smoke 2026-07-02 01:53:43 +08:00
lingniu
96514898eb test: add go systemd production smoke 2026-07-02 01:51:41 +08:00
lingniu
0ef88e1a4c test: add kafka production smoke 2026-07-02 01:49:02 +08:00
lingniu
0fd4d319f9 docs: record go kafka production topics 2026-07-02 01:44:27 +08:00
lingniu
4e8434c511 test: verify mqtt history tables in production smoke 2026-07-02 01:41:08 +08:00
lingniu
576644eb65 test: verify gb32960 history tables in production smoke 2026-07-02 01:39:45 +08:00
lingniu
9bd48c5781 test: require fresh raw data in production smoke 2026-07-02 01:38:16 +08:00
lingniu
2214abe9c5 test: include realtime in production smoke 2026-07-02 01:36:14 +08:00
lingniu
1477b997c9 test: add go native production smoke 2026-07-02 01:34:28 +08:00
lingniu
4ea8d18972 docs: document go native ecs deployment 2026-07-02 01:31:58 +08:00
lingniu
947e645090 fix: query history with utc tdengine filters 2026-07-02 01:28:45 +08:00
lingniu
78b805e3a2 fix: normalize timezone history filters 2026-07-02 01:24:51 +08:00
lingniu
a7b24f4c0b feat: expose realtime snapshot protocols 2026-07-02 01:23:03 +08:00
lingniu
eb124d929b fix: accept datetime-local history filters 2026-07-02 01:19:19 +08:00
lingniu
4711287655 fix: return empty arrays for query APIs 2026-07-02 01:16:48 +08:00
lingniu
2c224396c9 fix: return real pagination totals 2026-07-02 01:14:06 +08:00
lingniu
4f2cded760 fix: treat yutong mqtt mileage as meters 2026-07-02 01:08:59 +08:00
lingniu
7ad1a25e8e fix: normalize yutong mqtt mileage units 2026-07-02 01:04:14 +08:00
lingniu
3ee1d6f52b fix: stabilize yutong mqtt ingestion 2026-07-02 01:01:33 +08:00
lingniu
6d1d0aa85e feat: write protocol acknowledgements from go gateway 2026-07-02 00:54:31 +08:00
lingniu
055373c405 fix: support extended gb32960 platform login 2026-07-02 00:47:01 +08:00
lingniu
e37dd39c89 fix: enrich jt808 location additional parsing 2026-07-02 00:36:51 +08:00
lingniu
dd73fdbb5b feat: add vehicle mileage history query 2026-07-02 00:29:49 +08:00
lingniu
fb9ab8525a feat: add vehicle location history query 2026-07-02 00:27:20 +08:00
lingniu
cc89e0d537 fix: filter raw frames by vehicle key 2026-07-02 00:24:53 +08:00
lingniu
f3ecf430cc fix: cache realtime snapshots by vehicle key 2026-07-02 00:21:50 +08:00
lingniu
bcf7cdae2e fix: key daily metrics by vehicle identity 2026-07-02 00:18:35 +08:00
lingniu
9045b871d6 fix: align jt808 parser with protocol variants 2026-07-02 00:14:02 +08:00
lingniu
b0150a5503 docs: record yutong mqtt eof diagnosis 2026-07-02 00:04:23 +08:00
lingniu
8f3b337fd6 docs: add native go deployment runbook 2026-07-02 00:02:21 +08:00
lingniu
594ab2d9c7 feat: support yutong mqtt tls config 2026-07-01 23:50:30 +08:00
lingniu
229ffcf61f docs: record go production port verification 2026-07-01 23:46:33 +08:00
lingniu
593713283c fix: query tdengine raw frames with literals 2026-07-01 23:42:09 +08:00
lingniu
6bee0499ae feat: expose raw frame query api 2026-07-01 23:39:10 +08:00
lingniu
59c1c9df0f docs: record daily metric api verification 2026-07-01 23:35:50 +08:00
lingniu
56f4811c68 fix: format daily metric query timestamps 2026-07-01 23:33:03 +08:00
lingniu
1a53011408 feat: expose daily metric query api 2026-07-01 23:30:54 +08:00
lingniu
c2d70dad86 docs: record production chain verification 2026-07-01 23:28:27 +08:00
lingniu
91db6c959e docs: record jt808 identity tracking verification 2026-07-01 23:23:59 +08:00
lingniu
7b44f97ddb fix: decode jt808 registration plate text 2026-07-01 23:21:53 +08:00
lingniu
a72977cd79 feat: track jt808 registration identity 2026-07-01 23:17:29 +08:00
lingniu
dd95101499 docs: record mileage metric guard verification 2026-07-01 23:11:35 +08:00
lingniu
d347525a75 fix: keep valid realtime mileage 2026-07-01 23:08:31 +08:00
lingniu
19008e840c fix: ignore invalid zero mileage metrics 2026-07-01 23:06:06 +08:00
lingniu
c803a19bf5 docs: record go production port switch 2026-07-01 23:03:33 +08:00
lingniu
243de5c2b2 feat: spool gateway publishes to disk 2026-07-01 23:00:06 +08:00
lingniu
4a32e2181a docs: record gateway publish retry deployment 2026-07-01 22:56:55 +08:00
lingniu
7e7066dd86 feat: retry gateway kafka publishes 2026-07-01 22:54:47 +08:00
lingniu
e7c8c57296 docs: record jt808 identity verification 2026-07-01 22:51:18 +08:00
lingniu
6b55694c43 fix: resolve jt808 phone without leading zero 2026-07-01 22:34:47 +08:00
lingniu
eb48d16181 docs: record gb32960 vendor verification 2026-07-01 22:29:46 +08:00
lingniu
a15de91912 feat: parse gb32960 guangdong fuel cell blocks 2026-07-01 22:27:34 +08:00
lingniu
796d79dd30 docs: record go gateway real frame verification 2026-07-01 22:23:44 +08:00
lingniu
5f4f4fd124 feat: parse gb32960 realtime data units 2026-07-01 22:21:21 +08:00
lingniu
e0068a750d fix: support jt808 versioned header 2026-07-01 22:12:47 +08:00
lingniu
665122b23b docs: record go gateway ecs verification 2026-07-01 22:08:45 +08:00
lingniu
60ca42e5d5 fix: quote tdengine history inserts 2026-07-01 22:06:12 +08:00
lingniu
80474eeccb feat: add go vehicle identity resolver 2026-07-01 21:57:18 +08:00
lingniu
17024651c8 chore: add go gateway deployment stack 2026-07-01 21:55:27 +08:00
lingniu
d8c77d7882 feat: add yutong mqtt ingestion 2026-07-01 21:53:39 +08:00
lingniu
b55b075a97 feat: add go realtime redis api 2026-07-01 21:51:13 +08:00
lingniu
bac8864e79 feat: add go daily metric writer 2026-07-01 21:48:52 +08:00
lingniu
5d844701ff feat: add go tdengine history writer 2026-07-01 21:47:38 +08:00
lingniu
967981c040 feat: add go tcp gateway runtime 2026-07-01 21:45:28 +08:00
lingniu
1fdbc29e27 feat: scaffold go vehicle gateway core 2026-07-01 21:43:16 +08:00
lingniu
d148a4e3af docs: design go ingest redesign phase one 2026-07-01 21:36:07 +08:00
66 changed files with 13750 additions and 0 deletions

View File

@@ -0,0 +1,117 @@
version: "3.8"
# Go sidecar stack for the vehicle ingest redesign.
# It is intentionally separate from deploy/portainer/docker-compose.yml so the
# current Java production stack can keep running until Go evidence is captured.
x-common-env: &common-env
TZ: Asia/Shanghai
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.yutong-mqtt.v1}
KAFKA_TOPIC_UNIFIED: ${KAFKA_TOPIC_UNIFIED:-vehicle.event.unified.v1}
x-restart: &restart-policy
restart: unless-stopped
networks:
- vehicle-ingest
logging:
driver: json-file
options:
max-size: "100m"
max-file: "5"
services:
go-vehicle-gateway:
<<: *restart-policy
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
container_name: go-vehicle-gateway
command: ["/app/gateway"]
mem_limit: ${GO_GATEWAY_MEM_LIMIT:-512m}
environment:
<<: *common-env
GB32960_TCP_ADDR: ":32960"
JT808_TCP_ADDR: ":808"
TCP_READ_BUFFER_BYTES: ${TCP_READ_BUFFER_BYTES:-65536}
TCP_IDLE_TIMEOUT_SECONDS: ${TCP_IDLE_TIMEOUT_SECONDS:-180}
TCP_MAX_CONNECTIONS: ${TCP_MAX_CONNECTIONS:-20000}
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
YUTONG_MQTT_ENDPOINT: ${YUTONG_MQTT_ENDPOINT:-yutong}
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
YUTONG_MQTT_TOPICS: ${YUTONG_MQTT_TOPICS:-/ytforward/shln/+}
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-2}
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt}
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-}
VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding}
ports:
- "${GO_GB32960_TCP_PORT:-32960}:32960"
- "${GO_JT808_TCP_PORT:-808}:808"
volumes:
- "${KAFKA_SPOOL_HOST_DIR:-/opt/lingniu-go/spool/gateway}:${KAFKA_SPOOL_DIR:-/data/spool/gateway}"
- "${YUTONG_MQTT_CERT_HOST_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:${YUTONG_MQTT_CERT_CONTAINER_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:ro"
go-history-writer:
<<: *restart-policy
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
container_name: go-history-writer
command: ["/app/history-writer"]
mem_limit: ${GO_HISTORY_WRITER_MEM_LIMIT:-768m}
environment:
<<: *common-env
KAFKA_GROUP: ${KAFKA_GROUP_GO_HISTORY:-go-history-writer}
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_HISTORY:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1}
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
TDENGINE_DSN: ${TDENGINE_DSN:?set TDENGINE_DSN, example root:password@ws(172.17.111.57:6041)/lingniu_vehicle_ts}
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
TDENGINE_ENSURE_SCHEMA: ${TDENGINE_ENSURE_SCHEMA:-true}
go-stat-writer:
<<: *restart-policy
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
container_name: go-stat-writer
command: ["/app/stat-writer"]
mem_limit: ${GO_STAT_WRITER_MEM_LIMIT:-512m}
environment:
<<: *common-env
KAFKA_GROUP: ${KAFKA_GROUP_GO_STAT:-go-stat-writer}
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_STAT:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1}
MYSQL_DSN: ${MYSQL_DSN:?set MYSQL_DSN}
MYSQL_ENSURE_SCHEMA: ${MYSQL_ENSURE_SCHEMA:-true}
LOCAL_TZ: ${LOCAL_TZ:-Asia/Shanghai}
go-realtime-api:
<<: *restart-policy
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
container_name: go-realtime-api
command: ["/app/realtime-api"]
mem_limit: ${GO_REALTIME_API_MEM_LIMIT:-512m}
environment:
<<: *common-env
HTTP_ADDR: ":20210"
KAFKA_GROUP: ${KAFKA_GROUP_GO_REALTIME:-go-realtime-api}
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_REALTIME:-vehicle.event.unified.v1}
REDIS_ADDR: ${REDIS_ADDR:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379}
REDIS_USERNAME: ${REDIS_USERNAME:-}
REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
REDIS_DB: ${REDIS_DB:-50}
ONLINE_TTL_SECONDS: ${ONLINE_TTL_SECONDS:-600}
MYSQL_DSN: ${MYSQL_DSN:-}
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
TDENGINE_DSN: ${TDENGINE_DSN:-}
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
ports:
- "${GO_REALTIME_HTTP_PORT:-20210}:20210"
networks:
vehicle-ingest:
name: vehicle-ingest

66
deploy/systemd/README.md Normal file
View File

@@ -0,0 +1,66 @@
# Go Vehicle Gateway Native Deployment
本目录用于 ECS 原生部署。当前 goal 后续不再使用 Docker/Portainer 作为 Go 接入链路的部署方式。
## Runtime Layout
```text
/opt/lingniu-go-native/
current -> /opt/lingniu-go-native/releases/<git-short-sha>
releases/<git-short-sha>/
gateway
history-writer
stat-writer
realtime-api
env/
gateway.env
history-writer.env
stat-writer.env
realtime-api.env
spool/gateway/
```
## Services
| systemd unit | Binary | Purpose |
|---|---|---|
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP `32960`、JT808 TCP `808`、宇通 MQTT 接入,写 Kafka RAW/unified |
| `lingniu-go-history-writer.service` | `history-writer` | 消费 RAW topic写 TDengine `raw_frames` 和核心时序表 |
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 RAW topic写 MySQL `vehicle_daily_metric` |
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费 unified topic写 Redis并提供 realtime/raw/stat 查询 API |
## Build
在开发机或 CI 上构建 Linux amd64 二进制:
```bash
cd go/vehicle-gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/gateway ./cmd/gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/history-writer ./cmd/history-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/stat-writer ./cmd/stat-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/realtime-api ./cmd/realtime-api
```
## Cutover Notes
1. 先上传二进制到新 release 目录并更新 `current` symlink。
2. 从现有生产 env 生成四个 systemd 专用 env 文件;不要把密钥写入仓库。
3. 停止旧 Docker Go 容器或 Java 容器,释放 `808``32960``20210`
4. 执行:
```bash
systemctl daemon-reload
systemctl enable lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
```
## Verification
```bash
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=JT808&limit=1'
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=GB32960&limit=1'
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=YUTONG_MQTT&limit=1'
```

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Vehicle Gateway
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/gateway.env
ExecStart=/opt/lingniu-go-native/current/gateway
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go History Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/history-writer.env
ExecStart=/opt/lingniu-go-native/current/history-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Realtime API
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/realtime-api.env
ExecStart=/opt/lingniu-go-native/current/realtime-api
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Stat Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/stat-writer.env
ExecStart=/opt/lingniu-go-native/current/stat-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,227 @@
# 当前 ECS Go 原生部署说明
更新时间2026-07-02 01:30 CST
本文档记录当前 `lingniu-vehicle-ingest` 的生产运行面。当前 goal 的接入链路已经切到 Go 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。密钥只维护在 ECS 环境文件或受控凭据中,不写入 Git。
## 部署范围
| systemd 服务 | 二进制 | 说明 | 对外端口 |
| --- | --- | --- | --- |
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP、JT808 TCP、宇通 MQTT 接入,解析后写 Kafka RAW 和统一事件 | TCP `32960`、TCP `808` |
| `lingniu-go-history-writer.service` | `history-writer` | 消费 Kafka RAW写 TDengine RAW、位置点、里程点核心表 | 无 HTTP |
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 Kafka RAW按总里程差值法写 MySQL 每日指标 | 无 HTTP |
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费统一事件写 Redis并提供实时、历史、统计查询 API | HTTP `20210` |
信达 Push 已废弃,不参与当前 Go 生产链路。旧 Java/Docker 服务不应占用 `808``32960``20210`
## 应用 ECS
| 项 | 值 |
| --- | --- |
| 公网 IP | `115.29.187.205` |
| 登录用户 | `root` |
| 部署目录 | `/opt/lingniu-go-native` |
| 当前 release | `/opt/lingniu-go-native/current` |
| 环境文件目录 | `/opt/lingniu-go-native/env` |
| systemd unit 目录 | `/etc/systemd/system` |
| 对外服务 | `32960``808``20210` |
运行目录结构:
```text
/opt/lingniu-go-native/
current -> /opt/lingniu-go-native/releases/<git-short-sha>
releases/<git-short-sha>/
gateway
history-writer
stat-writer
realtime-api
env/
gateway.env
history-writer.env
stat-writer.env
realtime-api.env
spool/gateway/
```
## 中间件
| 组件 | 内网地址 | 公网地址 | 用途 |
| --- | --- | --- | --- |
| Kafka | `172.17.111.56:9092` | `114.55.58.251:9092` | RAW 和统一事件消息总线 |
| TDengine | `172.17.111.57:6041` | `115.29.185.82:6041` | RAW、位置、里程点时序热存储 |
| MySQL RDS | `rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306` | `rm-bp179zbv481rnw3e2no.mysql.rds.aliyuncs.com:3306` | 身份映射、JT808 注册、每日指标 |
| Redis RDS | `r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379` | 无 | 准实时车辆状态缓存 |
生产应用之间访问中间件优先使用内网地址。RDS 白名单需要允许应用 ECS 私网访问。
## Kafka Topic
| Topic | 生产者 | 消费者 | 内容 |
| --- | --- | --- | --- |
| `vehicle.raw.go.gb32960.v1` | `gateway` | `history-writer``stat-writer` | GB32960 完整 RAW 记录 |
| `vehicle.raw.go.jt808.v1` | `gateway` | `history-writer``stat-writer` | JT808 完整 RAW 记录 |
| `vehicle.raw.go.yutong-mqtt.v1` | `gateway` | `history-writer` | 宇通 MQTT 完整 RAW 记录 |
| `vehicle.event.go.unified.v1` | `gateway` | `realtime-api` | 三类协议统一事件,用于 Redis 实时状态 |
`gateway` 配置了 Kafka retry生产 env 中启用 `KAFKA_SPOOL_DIR` 时,本地 spool 可在 Kafka 短暂不可用后回放。
Kafka 消费组:
| Consumer Group | Topic | 当前验证 |
| --- | --- | --- |
| `go-history-writer` | 三个 `vehicle.raw.go.*` topic | 2026-07-02 验证 lag 为 `0` |
| `go-stat-writer` | `vehicle.raw.go.gb32960.v1``vehicle.raw.go.jt808.v1` | 2026-07-02 验证 lag 为 `0` |
| `go-realtime-api` | `vehicle.event.go.unified.v1` | 2026-07-02 验证 lag 为 `0` |
可重复 smoke
```bash
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
```
该脚本通过 SSH 登录 Kafka ECS检查 Go 生产 topic 是否存在,并校验 `go-history-writer``go-stat-writer``go-realtime-api` 的消费组 lag。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
## TDengine 数据层
默认数据库:`lingniu_vehicle_ts`
| 表 | 类型 | 说明 |
| --- | --- | --- |
| `raw_frames` | stable | 完整 RAW 帧,包含 `raw_hex``parsed_json``fields_json``source_endpoint` 和协议/车辆 tags |
| `vehicle_locations` | stable | 最小化位置历史,保留 `frame_id` 回链 RAW |
| `vehicle_mileage_points` | stable | 最小化总里程点,供历史查询和统计复核 |
设计原则:
- 完整结构化 payload 只落在 `raw_frames.parsed_json` / `fields_json`
- `vehicle_locations``vehicle_mileage_points` 只保留核心查询字段,不重复保存完整 JSON。
- `telemetry_fields` 不由当前服务维护,字段配置解析由独立子服务处理。
- API 传入东八区时间时,服务会转换为 TDengine 当前存储使用的 UTC 字面量再查询。
## MySQL 数据层
默认业务库:`lingniu_vehicle_data`
| 表 | 维护方 | 说明 |
| --- | --- | --- |
| `vehicle_identity_binding` | 人工/外部主数据 | VIN 映射主表,用 `phone``device_id``plate` 降级定位 VIN |
| `jt808_registration` | `gateway` 自动写入 | 仅 JT808 注册和鉴权记录,以 `phone` 作为主键,记录设备、车牌、厂家、鉴权、来源端点 |
| `vehicle_daily_metric` | `stat-writer` 自动写入 | 每日指标表,保存 `daily_mileage_km``daily_total_mileage_km` |
每日里程算法:
```text
daily_mileage_km = 当日最大 total_mileage_km - 当日最小 total_mileage_km
daily_total_mileage_km = 当日最大 total_mileage_km
```
统计按 `Asia/Shanghai` 自然日归档,当前支持 GB32960、JT808、宇通 MQTT 中能解析出 `total_mileage_km` 的数据。JT808 的总里程来自位置附加信息 `0x01`,单位按协议转换为 km宇通 MQTT 的 `TOTAL_MILEAGE` 原始单位为米,入库前转换为 km。
## Redis 实时层
`realtime-api` 消费 `vehicle.event.go.unified.v1`,按 VIN/vehicle key 维护准实时快照。主要能力:
| API | 说明 |
| --- | --- |
| `/api/realtime/vehicles/{vin}` | 查询车辆合并实时快照 |
| `/api/realtime/vehicles/{vin}/online` | 查询车辆是否在线 |
| `/api/realtime/vehicles/{vin}/protocols/{protocol}` | 查询指定协议的实时快照 |
Redis 只作为实时缓存;历史和统计以 TDengine/MySQL 为准。
## API 入口
公网基础地址:
```text
http://115.29.187.205:20210
```
常用查询:
```bash
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=YUTONG_MQTT&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/locations?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/mileage-points?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/stats/daily-metrics?dateFrom=2026-07-02&dateTo=2026-07-02&limit=20'
```
分页参数统一使用 `limit``offset`
部署后推荐直接运行 Go 原生生产 smoke
```bash
python3 tools/go_prod_acceptance.py --date 2026-07-02
```
该聚合脚本会串行执行 systemd/端口检查、Kafka topic/consumer lag 检查、HTTP 数据链路检查,任一子检查失败时整体退出码为非 0。需要 SSH 可登录应用 ECS 和 Kafka ECS。
也可以单独运行子检查:
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
```
`go_systemd_prod_smoke.py` 通过 SSH 检查四个 Go systemd 服务是否 `active``enabled`,并确认 `808``32960``gateway` 监听、`20210``realtime-api` 监听,同时要求 `/opt/lingniu-go-native/current` 指向有效 release、四个生产二进制均可执行、`/opt/lingniu-go-native/spool/gateway` 没有待回放文件,避免旧 Java/Docker 进程占用生产端口、部署目录错乱或 Kafka spool 静默堆积。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
`go_native_prod_smoke.py` 通过 `20210` HTTP API 验证 GB32960、JT808、宇通 MQTT 的 RAW 查询及结构化 `parsed_json`,三类协议的位置和里程点查询及 `frame_id` RAW 回链GB32960/JT808 的 `daily_mileage_km``daily_total_mileage_km` 统计公式,以及三类协议的 Redis realtime snapshot、online、protocol 查询。实时 snapshot/protocol 查询会校验 `updated_at_ms` 新鲜度;默认检查今天东八区数据时,还会要求三类 RAW 最新样本不超过 15 分钟,避免旧数据误判为接收正常;任一检查不达标时退出码为非 0。
## 部署命令
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收:
```bash
python3 tools/go_native_deploy.py --date 2026-07-02
```
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout``--spool-poll-interval` 调整。
脚本执行的发布动作等价于:
```bash
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/history-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/stat-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/realtime-api
scp release.tar.gz root@115.29.187.205:/tmp/
ln -sfn /opt/lingniu-go-native/releases/<git-short-sha> /opt/lingniu-go-native/current
systemctl daemon-reload
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
```
只更新查询 API 时,可以仅替换 `/opt/lingniu-go-native/current/realtime-api` 并重启:
```bash
systemctl restart lingniu-go-realtime-api.service
```
## 运行检查
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-history-writer --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-stat-writer --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-realtime-api --since '5 minutes ago' --no-pager
```
2026-07-02 01:30 CST 已验证:
- 四个 Go systemd 服务均为 `active``enabled`
- `/opt/lingniu-go-native/current` 指向有效 release四个生产二进制均存在且可执行。
- `gateway` 正在监听 `32960``808`
- `realtime-api` 正在监听 `20210`
- gateway Kafka spool 当前无待回放文件。
- GB32960、JT808、YUTONG_MQTT 的东八区 RAW 查询均可命中生产数据,且最新 RAW 样本包含结构化 `parsed_json`
- GB32960、JT808、YUTONG_MQTT 的最新 RAW 样本均在 15 分钟内。
- GB32960、JT808、YUTONG_MQTT 的 `vehicle_locations``vehicle_mileage_points` 查询均可命中生产数据,且样本包含 `frame_id` 回链 RAW。
- `vehicle_daily_metric` 可查到 `daily_mileage_km``daily_total_mileage_km`,且公式满足 `daily_mileage_km = latest_total_mileage_km - first_total_mileage_km``daily_total_mileage_km = latest_total_mileage_km`
- Redis realtime 可查到 GB32960、JT808、YUTONG_MQTT 的在线状态和协议快照,且 snapshot/protocol 快照最近刷新。

View File

@@ -0,0 +1,888 @@
# Go Vehicle Gateway 旁路验证记录
验证时间2026-07-01 22:00-22:10 CST
## 部署版本
- Git commit`60ca42e`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-60ca42e-20260701220622`
- 部署方式ECS `115.29.187.205` 上 Docker Compose 旁路部署
- Compose 目录:`/opt/lingniu-go`
## 旁路端口和 topic
为了不影响现有 Java 生产服务,本次没有占用生产端口和生产 topic。
| 项 | 值 |
|---|---|
| GB32960 旁路 TCP | `115.29.187.205:23296 -> container :32960` |
| JT808 旁路 TCP | `115.29.187.205:18080 -> container :808` |
| Realtime API | `115.29.187.205:20210` |
| GB32960 RAW topic | `vehicle.raw.go.gb32960.v1` |
| JT808 RAW topic | `vehicle.raw.go.jt808.v1` |
| Yutong MQTT RAW topic | `vehicle.raw.go.yutong-mqtt.v1` |
| Unified topic | `vehicle.event.go.unified.v1` |
## 容器状态
ECS 上四个 Go 容器均已启动:
```text
go-vehicle-gateway Up
go-history-writer Up
go-stat-writer Up
go-realtime-api Up
```
## Kafka 验证
`18080` 注入 JT808 样例帧后,`vehicle.raw.go.jt808.v1`
`vehicle.event.go.unified.v1` 均消费到统一 JSON envelope。
样例解析结果包含:
- `protocol=JT808`
- `message_id=0x0200`
- `phone=013307795425`
- `longitude=121.069881`
- `latitude=30.590151`
- `speed_kmh=23`
- `total_mileage_km=10241.2`
- 表 27 附加项 `0x01` 原始值 `0001900C`
## TDengine 验证
目标库:`lingniu_vehicle_ts`
```text
raw_frames count = 3
vehicle_locations count = 2
vehicle_mileage_points count = 2
```
最新 RAW 标识:
```text
GB32960 | LNBSCB3D4R1234567 | LNBSCB3D4R1234567 |
JT808 | JT808:013307795425 | | 013307795425
JT808 | JT808:013307795425 | | 013307795425
```
说明JT808 样例 phone 当前没有在 `vehicle_identity_binding` 中解析到 VIN因此
TDengine tag 使用 `vehicle_key=JT808:013307795425`
## Redis 验证
`23296` 注入带 VIN 的 GB32960 合成帧后Realtime API 可查询:
```text
GET /api/realtime/vehicles/LNBSCB3D4R1234567
```
返回字段包含:
- `vin=LNBSCB3D4R1234567`
- `protocols=["GB32960"]`
- `online=true`
- `speed_kmh=30`
- `total_mileage_km=10000`
- `soc_percent=85`
- `longitude=121`
- `latitude=30.56`
## MySQL 统计验证
`23296` 注入带 VIN 的 GB32960 合成帧后,`vehicle_daily_metric` 写入:
```text
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_mileage_km | 0.000 | 10000.000 | 10000.000 | 1
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_total_mileage_km | 10000.000 | 10000.000 | 10000.000 | 1
```
说明:本次 GB32960 合成帧的设备时间为测试值,因此统计日期为 `2020-07-01`
## 已修复问题
- TDengine WebSocket 普通 `INSERT` 使用参数占位符时,字符串未按预期加引号,导致语法错误。已在 `60ca42e` 中改为显式 SQL literal 转义。
- MySQL Go DSN 中 `charset=utf8mb4%2Cutf8` 会被 driver 传给 MySQL导致语法错误ECS 旁路 env 已改为 `charset=utf8mb4`
- TDengine ECS 当前可用 root 密码与早先记录不一致,本次旁路 env 已按实际可认证值修正。
## 未完成
- 未切换真实生产端口 `32960/808`
- 未将真实外部 32960/808 流量转发到 Go 旁路端口。
- 未开启宇通 MQTT 正式订阅。
- JT808 样例未解析到 VIN需维护 `vehicle_identity_binding` 后再验证 808 每日统计。
## 2026-07-01 22:22 真实归档帧复验
部署版本已更新:
- Git commit`5f4f4fd`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5f4f4fd-20260701222128`
本轮补充验证内容:
- JT808 支持 2019 版 versioned header真实帧 `0x0200` 可解析出 `phone=14894135060``longitude=117.178187``latitude=30.570155``speed_kmh=33.4``total_mileage_km=7868.9`
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 可解析 `0x01` 整车、`0x02` 驱动电机、`0x03` 燃料电池、`0x04` 发动机、`0x05` 位置、`0x06` 极值、`0x07` 报警、`0x08` 电压、`0x09` 温度;后续 `0x30` 广东燃料电池扩展暂以 unknown raw 保留。
- GB32960 真实帧字段包含 `fuel_cell_hydrogen_consumption_kg_per_100km=1.8``longitude=120.800326``latitude=31.634907``total_mileage_km=53490.9`
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回上述经纬度和氢耗字段。
- TDengine `vehicle_locations``vehicle_mileage_points` 已写入真实 GB32960 位置和里程点:
```text
2020-07-01 08:09:22 | GB32960 | LB9A32A21R0LS1707 | lon=120.800326 | lat=31.634907 | mileage=53490.9
```
后续仍需补齐:
- JT808 真实 phone `14894135060` 仍未解析到 VIN需维护身份绑定后再验证 VIN 维度实时和统计。
## 2026-07-01 22:28 广东燃料电池扩展复验
部署版本已更新:
- Git commit`a15de91`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-a15de91-20260701222743`
本轮补充验证内容:
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 的 vendor 尾部已从 `unknown_unit` 补为结构化块:
`0x30 gd_fc_stack``0x31 gd_fc_auxiliary``0x32 gd_fc_dcdc``0x33 gd_fc_air_conditioner`
`0x34 gd_fc_vehicle_info``0x80 gd_fc_demo_extension``0x83 gd_fc_vendor_tlv`
- Kafka `vehicle.raw.go.gb32960.v1` 最新消息包含上述 block并输出关键扁平字段
`gd_fc_stack_hydrogen_inlet_pressure_kpa=97``gd_fc_stack_cell_count=108`
`gd_fc_vehicle_hydrogen_mass_kg=4.3``gd_fc_demo_stack_temp_c=65`
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回 vendor 扁平字段:
```text
gd_fc_dcdc_controller_temp_c=47
gd_fc_demo_stack_temp_c=65
gd_fc_stack_cell_count=108
gd_fc_stack_frame_cell_count=108
gd_fc_stack_hydrogen_inlet_pressure_kpa=97
gd_fc_stack_water_outlet_temp_c=65
gd_fc_vehicle_hydrogen_mass_kg=4.3
```
- TDengine `raw_frames` 中 GB32960 行数增加到 `4`,最新 RAW 标识:
```text
2026-07-01 14:28:24.869 | 172.20.0.1:47082 | GB32960 | LB9A32A21R0LS1707 | OK
```
后续仍需补齐:
- JT808 真实 phone `14894135060` 到 VIN 的绑定数据维护与统计复验。
- Go 旁路尚未切换到生产 `32960/808` 端口;仍运行在 `23296/18080`
## 2026-07-01 22:43 JT808 前导零 phone 身份解析复验
部署版本已更新:
- Git commit`6b55694`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-6b55694-20260701224300`
本轮修复内容:
- JT808 2011 老版包头的终端手机号按 BCD[6] 解析会保留前导 `0`,例如包头为 `013079963379`
- `vehicle_identity_binding.phone` 中实际维护的是去前导零后的手机号 `13079963379`
- 身份解析候选键已调整为先查原始 phone再查去前导零 phone之后再按 `device_id``plate` 降级。
真实帧复验样本:
- 归档帧:`/opt/lingniu/vehicle-ingest/gb32960/archive/2026/07/01/JT808/unknown/1782903942845000.bin`
- 消息:`0x0200` 位置信息汇报
- 包头 phone`013079963379`
- 绑定表 phone`13079963379`
- 解析 VIN`LKLG7C4E3NA774736`
Kafka `vehicle.raw.go.jt808.v1` 最新消息确认:
```text
vin=LKLG7C4E3NA774736
phone=013079963379
parsed.header.phone_trim_zero=13079963379
parsed.identity.resolved=true
parsed.identity.source=phone
parsed.identity.value=13079963379
```
Realtime API 确认:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736
longitude=119.557997
latitude=29.049092
speed_kmh=0
total_mileage_km=0
device_time=2026-07-01T19:05:40+08:00
```
TDengine 确认:
```text
raw_frames:
2026-07-01 14:49:10.913 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | message_id=512
vehicle_locations:
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | lon=119.557997 | lat=29.049092 | speed=0 | mileage=0
vehicle_mileage_points:
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | mileage=0
```
MySQL `vehicle_daily_metric` 确认:
```text
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
```
说明:该真实位置帧携带的表 27 附加项 `0x01` 总里程值为 `0`,因此本次只验证身份解析、链路落地和差值统计写入;要验证非零日里程,需要回放或等待同一 VIN 的非零总里程连续位置点。
## 2026-07-01 22:55 Kafka 发布重试部署复验
部署版本已更新:
- Git commit`7e7066d`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7e7066d-20260701225503`
本轮修复内容:
- gateway 的 Kafka 发布增加同步重试封装。
- 默认配置:
- `KAFKA_PUBLISH_ATTEMPTS=3`
- `KAFKA_PUBLISH_BACKOFF_MS=100`
- `KAFKA_PUBLISH_TIMEOUT_MS=3000`
- RAW 仍先于 unified event 写入RAW 写失败不会继续写 unified。
- 单元测试覆盖首次失败后成功、重试耗尽返回错误、上下文取消时停止重试。
部署后容器状态:
```text
go-vehicle-gateway | go-7e7066d-20260701225503 | Up
go-history-writer | go-7e7066d-20260701225503 | Up
go-stat-writer | go-7e7066d-20260701225503 | Up
go-realtime-api | go-7e7066d-20260701225503 | Up
```
真实 808 帧回放复验:
```text
source_endpoint=172.20.0.1:51966
protocol=JT808
message_id=0x0200
phone=013079963379
vin=LKLG7C4E3NA774736
parsed.identity.value=13079963379
```
Kafka `vehicle.raw.go.jt808.v1` 确认新消息写入partition 6 offset 从 `1` 增加到 `2`。最新消息包含:
```text
received_at_ms=1782917751903
source_endpoint=172.20.0.1:51966
vin=LKLG7C4E3NA774736
fields.longitude=119.557997
fields.latitude=29.049092
fields.total_mileage_km=0
```
TDengine `raw_frames` 确认新 RAW 落地:
```text
2026-07-01 14:55:51.903 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | source=172.20.0.1:51966
```
Realtime API 确认 Redis 已刷新:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736
received_at_ms=1782917751903
source_endpoint=172.20.0.1:51966
longitude=119.557997
latitude=29.049092
total_mileage_km=0
```
说明:当前重试层只能覆盖 Kafka 短暂抖动;如果 Kafka 长时间不可用,仍需后续增加本地磁盘 spool/WAL 和恢复补发。
## 2026-07-01 23:02 切换 Go 到生产 32960/808 端口
本轮操作:
- 停止 ECS 上原有 Java Docker 容器:
- `gb32960-ingest-app`
- `jt808-ingest-app`
- `yutong-mqtt-app`
- `vehicle-history-app`
- `vehicle-analytics-app`
- `vehicle-state-app`
- `telemetry-field-parser-app`
- 保留 Go sidecar 和 Portainer agent。
- 修改 `/opt/lingniu-go/.env`
- `GO_GB32960_TCP_PORT=32960`
- `GO_JT808_TCP_PORT=808`
- Go gateway 使用镜像:
- `crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-243de5c-20260701230028`
- 开启 Kafka 本地 spool
- host`/opt/lingniu-go/spool/gateway`
- container`/data/spool/gateway`
端口验证:
```text
0.0.0.0:32960 -> go-vehicle-gateway:32960
0.0.0.0:808 -> go-vehicle-gateway:808
18080/23296 -> 已释放
```
启动日志确认:
```text
kafka durable spool enabled dir=/data/spool/gateway replay_interval_ms=1000
tcp listener started protocol=GB32960 addr=[::]:32960
tcp listener started protocol=JT808 addr=[::]:808
```
真实连接确认:
```text
GB32960 external remote: 8.134.95.166:50084
JT808 external remote: 115.231.168.135:13801
```
生产 `808` 回放复验:
```text
source_endpoint=172.20.0.1:54680
vin=LKLG7C4E3NA774736
phone=013079963379
longitude=119.557997
latitude=29.049092
total_mileage_km=0
```
TDengine 最新 RAW 复验显示生产 808 已有真实外部数据持续落地:
```text
2026-07-01 15:03:02.960 | JT808 | LKLG7C4E1NA774802 | 013079963291 | 222.66.200.68:28394
2026-07-01 15:03:02.114 | JT808 | | 14894135583 | 122.152.221.156:33871
2026-07-01 15:03:01.352 | JT808 | | 013307795519 | 115.231.168.135:13801
2026-07-01 15:03:01.330 | JT808 | LKLG7C4E3NA774736 | 013079963379 | 222.66.200.68:28393
```
Kafka spool 目录确认:
```text
/opt/lingniu-go/spool/gateway file count = 0
```
说明:生产 808 中仍有部分 phone 未解析出 VIN后续需要继续维护 `vehicle_identity_binding` 的 phone/device_id/plate 到 VIN 映射,或者增强 808 注册/鉴权帧中的 identity 回写。
## 2026-07-01 23:09 808 里程统计防 0 污染复验
部署版本已更新:
- Git commit`d347525`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-d347525-20260701230845`
本轮修复内容:
- `stat-writer` 不再把 `total_mileage_km <= 0` 的采样写入 MySQL 每日指标。
- MySQL upsert 对历史 `first_total_mileage_km=0` / `latest_total_mileage_km=0` 做纠偏;后续首次有效总里程会替换掉历史 0。
- `realtime-api` 合并实时快照时,非正 `total_mileage_km` 不再覆盖已有正值;速度、位置、状态等其他字段仍按事件时间更新。
测试验证:
```text
go test ./...
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
```
生产回放验证:
- 第一条 808 帧:`device_time=2026-07-01T23:09:35+08:00``total_mileage_km=12345.6`
- 第二条 808 帧:`device_time=2026-07-01T23:09:36+08:00``total_mileage_km=0`
- 两条帧使用同一 VIN`LKLG7C4E3NA774736`
Redis merged 快照确认:第二条 0 里程帧没有覆盖已有有效总里程。
```text
event_time_ms=1782918576000
source_endpoint=172.20.0.1:40946
speed_kmh=0
longitude=119.557997
latitude=29.049092
total_mileage_km=12345.6
field_times_ms.total_mileage_km=1782918575000
```
MySQL `vehicle_daily_metric` 确认0 里程采样未进入统计sample_count 只因有效总里程采样增加一次。
```text
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=13 | first=12345.600 | latest=12345.600
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 12345.600 | sample_count=13 | first=12345.600 | latest=12345.600
```
说明:生产 TDengine 中已经存在大量 808 非零 `total_mileage_km` 采样,但许多 phone 尚未映射到 VIN因此 MySQL 按 VIN 的统计只会覆盖已解析 VIN 的车辆。下一步应继续补齐 `vehicle_identity_binding`,让更多 808 车辆进入统计。
## 2026-07-01 23:23 808 注册身份自动维护复验
部署版本已更新:
- Git commit`7b44f97`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7b44f97-20260701232200`
本轮修复内容:
- 808 `0x0100` 注册帧解析 `province``city``manufacturer``device_type``device_id``plate_color``plate`
- 808 `0x0102` 鉴权帧解析 `auth_token`
- Go gateway 在 identity resolve 后写入 `jt808_registration`,普通上行也会刷新 `latest_seen_at`
- 注册帧通过车牌解析到 VIN 后,会尝试把 phone/device/plate 反写到 `vehicle_identity_binding`
- 注册帧车牌支持 GBK 解码,避免 `Incorrect string value` 写入错误。
验证命令:
```text
go test ./...
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
```
ECS 运行状态:
```text
go-vehicle-gateway | go-7b44f97-20260701232200 | 0.0.0.0:808->808, 0.0.0.0:32960->32960
go-history-writer | go-7b44f97-20260701232200 | Up
go-stat-writer | go-7b44f97-20260701232200 | Up
go-realtime-api | go-7b44f97-20260701232200 | 0.0.0.0:20210->20210
```
生产日志确认:
- 新版本启动后真实 808、32960 连接持续进入。
- GBK 车牌注册帧不再出现 `Incorrect string value`
MySQL `jt808_registration` 确认:
```text
registration_rows=214
distinct_phone=214
vin_rows=74
plate_rows=74
device_rows=72
013079963379 | device_id=9963379 | plate=沪A06788F | vin=LKLG7C4E3NA774736 | latest_registered_at=2026-07-01 23:23:01
013079963321 | device_id=9963321 | plate=沪A59613F | vin=LKLG7C4EXNA774796 | latest_registered_at=2026-07-01 23:23:04
```
说明:`jt808_registration.phone` 保存 808 包头原始 phone`vehicle_identity_binding.phone` 中已有部分去前导零 phone。Go resolver 会同时查询原始 phone 和去前导零 phone。
## 2026-07-01 23:28 生产链路与指标清理复验
本轮复验目标:
- 确认 Go gateway 的 Kafka durable spool 是重启窗口残留,而不是持续发布失败。
- 确认 stat-writer / history-writer / realtime-api 的 Kafka consumer group 无 lag。
- 确认真实 808 车辆 registration、TDengine RAW/里程点、Redis 实时查询均在持续更新。
- 清理之前生产合成回放产生的 JT808 当日指标测试数据。
Kafka topic / consumer group
```text
vehicle.raw.go.jt808.v1 有持续 offset
vehicle.raw.go.gb32960.v1 有持续 offset
vehicle.event.go.unified.v1 有持续 offset
go-stat-writer LAG=0
go-history-writer LAG=0
go-realtime-api LAG=0
```
TDengine
```text
raw_frames:
JT808 2922
GB32960 27
vehicle_mileage_points:
JT808 2050
GB32960 2
```
Redis 实时查询样例:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736
vin=LKLG7C4E3NA774736
source_endpoint=222.66.200.68:29646
plate=沪A06788F
longitude=119.556866
latitude=29.047911
total_mileage_km=12345.6
```
说明:该 Redis merged 快照保留了此前有效 `total_mileage_km=12345.6`,后续真实 808 位置帧上报的 `total_mileage_km=0` 不会覆盖该字段。TDengine 中该 VIN 后续真实里程点的 `total_mileage_km` 均为 0因此 stat-writer 按“只统计正总里程”的规则不再写 JT808 每日指标。
MySQL 指标清理:
```text
删除前:
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_mileage_km | first=0 | latest=0 | sample_count=7
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_total_mileage_km | first=0 | latest=0 | sample_count=7
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
删除后等待真实流量窗口:
JT808 2026-07-01 metric rows = 0
```
清理原因:
- `LKLG7C4E3NA774736` 的 12345.6 来自此前生产连通性合成回放,不是真实平台上报。
- `LKLG7C4E2NA774775` 的 0 指标来自旧逻辑污染,当前 stat-writer 已跳过非正总里程。
Kafka spool
```text
23:24 spool_count=904
23:25 spool_count=862
23:28 spool_count=722
new_spool_1min=0
```
说明spool 正在回放下降,且最近 1 分钟没有新增文件;生产 gateway 日志未见 `Incorrect string value`、Kafka publish error 或 replay error。
## 2026-07-01 23:34 每日指标查询 API 复验
部署版本已更新:
- Git commit`56f4811`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-56f4811-20260701233311`
新增接口:
```text
GET /api/stats/daily-metrics
```
查询参数:
```text
vin 可选
protocol 可选GB32960 / JT808 / YUTONG_MQTT
metricKey 可选daily_mileage_km / daily_total_mileage_km
dateFrom 可选YYYY-MM-DD
dateTo 可选YYYY-MM-DD
limit 可选,默认 50最大 1000
offset 可选,默认 0
```
生产可访问样例:
```text
http://115.29.187.205:20210/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01&limit=10
```
返回确认:
```json
{
"items": [
{
"vin": "LB9A32A21R0LS1707",
"stat_date": "2020-07-01",
"protocol": "GB32960",
"metric_key": "daily_mileage_km",
"metric_value": 0,
"metric_unit": "km",
"first_total_mileage_km": 53490.9,
"latest_total_mileage_km": 53490.9,
"sample_count": 3,
"calculation_method": "TOTAL_MILEAGE_DIFF",
"created_at": "2026-07-01 22:14:13",
"updated_at": "2026-07-01 22:28:25"
},
{
"vin": "LB9A32A21R0LS1707",
"stat_date": "2020-07-01",
"protocol": "GB32960",
"metric_key": "daily_total_mileage_km",
"metric_value": 53490.9,
"metric_unit": "km",
"first_total_mileage_km": 53490.9,
"latest_total_mileage_km": 53490.9,
"sample_count": 3,
"calculation_method": "TOTAL_MILEAGE_DIFF",
"created_at": "2026-07-01 22:14:13",
"updated_at": "2026-07-01 22:28:25"
}
],
"limit": 10,
"offset": 0,
"total": 2
}
```
JT808 真实生产查询:
```text
http://115.29.187.205:20210/api/stats/daily-metrics?protocol=JT808&dateFrom=2026-07-01&dateTo=2026-07-01&limit=10
```
返回 `total=0`。原因是此前合成回放产生的测试指标已清理,当前真实 808 已绑定 VIN 的车辆仍上报 `total_mileage_km=0`stat-writer 按规则不写非正总里程指标。
实时接口回归:
```text
http://115.29.187.205:20210/api/realtime/vehicles/LKLG7C4E3NA774736/online
```
返回 `online=true`,说明新增 stats API 没有影响 Redis 实时查询。
发布后 spool 观察:
```text
发布后立即spool_count=1101new_spool_1min=102
等待 60 秒spool_count=1033new_spool_1min=0
```
说明:本次发布重启窗口产生的 spool 正在回放下降,且最近 1 分钟没有新增gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。
## 2026-07-01 23:50 生产端口接管与 RAW 查询 API 复验
部署版本:
- Git commit`5937132`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5937132-20260701234220`
生产端口状态:
```text
go-vehicle-gateway 0.0.0.0:808->808/tcp
go-vehicle-gateway 0.0.0.0:32960->32960/tcp
```
旧 Java 容器状态:
```text
gb32960-ingest-app restart=no status=exited
jt808-ingest-app restart=no status=exited
yutong-mqtt-app restart=no status=exited
vehicle-history-app restart=no status=exited
vehicle-analytics-app restart=no status=exited
vehicle-state-app restart=no status=exited
telemetry-field-parser-app restart=no status=exited
```
生产连接确认:
- JT808 端口 `808` 已收到外部连接:`222.66.200.68``115.231.168.135`
- GB32960 端口 `32960` 已收到外部连接:`8.134.95.166``117.160.0.65`
- ECS 当前没有 `8089` 监听。
RAW 查询 API
```text
GET /api/history/raw-frames
```
生产可访问样例:
```text
http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
```
接口复验结果:
```text
JT808_RAW:
total=1
protocol=JT808
vin=LKLG7C4E3NA774736
phone=013079963379
message_id=512
message_id_hex=0x0200
parse_status=OK
raw_hex_len=94
parsed_json_len=651
fields_json_len=186
GB32960_RAW:
total=1
protocol=GB32960
vin=LB9A32A21R0LS1707
message_id=2
message_id_hex=0x0002
parse_status=OK
raw_hex_len=1548
parsed_json_len=3698
fields_json_len=563
```
同轮回归验证:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
online=true
protocols=["JT808"]
GET /api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2026-07-01&limit=1
total=1
protocol=GB32960
metric_key=daily_mileage_km
calculation_method=TOTAL_MILEAGE_DIFF
```
说明:
- RAW 查询已返回 `raw_hex``parsed_json``fields_json`,可以用于查看完整解析结果。
- 日统计和实时查询在生产端口接管后仍可用。
## 2026-07-02 00:00 原生 systemd 部署与宇通 MQTT 复验
根据本次 goal 的新要求Go 接入链路后续不再使用 Docker 部署。本轮已中断 Docker 镜像构建路径,改为 ECS 原生 Linux 二进制 + systemd。
部署版本:
- Git commit`594ab2d`
- 部署目录:`/opt/lingniu-go-native`
- 当前 release`/opt/lingniu-go-native/releases/594ab2d`
- 当前指针:`/opt/lingniu-go-native/current -> /opt/lingniu-go-native/releases/594ab2d`
systemd 服务:
```text
lingniu-go-gateway.service active
lingniu-go-history-writer.service active
lingniu-go-stat-writer.service active
lingniu-go-realtime-api.service active
```
端口确认:
```text
[::]:32960 gateway
[::]:808 gateway
[::]:20210 realtime-api
```
Go Docker 容器已停止:
```text
go-vehicle-gateway Exited
go-history-writer Exited
go-stat-writer Exited
go-realtime-api Exited
```
宇通 MQTT
```text
yutong mqtt client started
mqtt subscribed broker=ssl://cpxlm.axxc.cn:38883 topic=/ytforward/shln/+ qos=1
```
说明MQTT broker 存在短连接 `EOF` 后自动重连现象,但订阅后已收到真实数据并落 RAW。
RAW 查询复验:
```text
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=3
total=3
protocol=YUTONG_MQTT
vin=LMRKH9AC1R1004131
vehicle_key=LMRKH9AC1R1004131
parse_status=OK
parsed_json_len=1104
fields_json_len=116
```
同轮回归:
```text
GET /api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
total=1
parse_status=OK
GET /api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
total=1
parse_status=OK
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
online=true
protocols=["JT808"]
```
本轮落地文件:
- `deploy/systemd/README.md`
- `deploy/systemd/lingniu-go-gateway.service`
- `deploy/systemd/lingniu-go-history-writer.service`
- `deploy/systemd/lingniu-go-stat-writer.service`
- `deploy/systemd/lingniu-go-realtime-api.service`
## 2026-07-02 00:04 宇通 MQTT EOF 诊断
现象:
Go 原生部署后,`lingniu-go-gateway` 日志中出现多次:
```text
mqtt connection lost error=EOF
mqtt connection lost error=write: broken pipe
mqtt subscribed topic=/ytforward/shln/+ qos=1
```
10 分钟窗口统计:
```text
mqtt subscribed = 33
mqtt connection lost = 32
```
对照旧 Java `yutong-mqtt-app` 历史日志,停止前也存在同类行为:
```text
mqtt endpoint [yutong] received topic=/ytforward/shln/1 bytes=574
mqtt endpoint [yutong] connection lost
Caused by: java.io.EOFException
mqtt endpoint [yutong] re-subscribed topic=/ytforward/shln/+ qos=1
mqtt endpoint [yutong] received topic=/ytforward/shln/3 bytes=586
```
判断:
- EOF/短连接不是 Go TLS 支持或 systemd 原生部署新引入的问题。
- 旧 Java 和新 Go 都表现为订阅、收到数据、broker 断开、自动重连。
- 当前链路可持续收到宇通 MQTT 数据并写入 TDengine RAW。
复验:
```text
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=1
total=1
protocol=YUTONG_MQTT
vin=LMRKH9ACXR1004094
parse_status=OK
parsed_json_len=1080
fields_json_len=115
```
运维建议:
- 单条 EOF 不作为不可用判断。
-`YUTONG_MQTT` RAW 入库增长、最近成功 `mqtt subscribed` 时间、服务 `systemctl is-active` 作为可用性判断。
- 如果 EOF 频率继续升高且 RAW 不再增长,再联系宇通侧确认 broker 长连接策略、同一 clientId 并发限制或网络出口策略。

View File

@@ -0,0 +1,158 @@
# Go Vehicle Gateway 验证手册
本文档用于验证 Go 重构运行面:
- `go-vehicle-gateway`GB32960 TCP、JT808 TCP、宇通 MQTT 接入。
- `go-history-writer`Kafka RAW 写 TDengine。
- `go-stat-writer`Kafka RAW 写 MySQL 每日指标。
- `go-realtime-api`Kafka unified event 写 Redis并提供实时查询 API。
## 本地构建验证
```bash
cd go/vehicle-gateway
go test ./...
go build ./cmd/gateway
go build ./cmd/history-writer
go build ./cmd/stat-writer
go build ./cmd/realtime-api
```
## 生产原生部署验证
```bash
python3 tools/go_prod_acceptance.py --date 2026-07-02
```
该命令会聚合执行:
- `go_systemd_prod_smoke.py`:应用 ECS systemd 服务 active/enabled、端口归属和 gateway spool
- `go_kafka_prod_smoke.py`Kafka topic 和消费组 lag
- `go_native_prod_smoke.py`HTTP API、RAW `parsed_json`、历史核心表 `frame_id` 回链、每日指标公式、Redis 实时新鲜度、TDengine/MySQL/Redis 数据链路
如需单独检查 systemd
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
```
脚本通过 SSH 检查:
- `lingniu-go-gateway.service`
- `lingniu-go-history-writer.service`
- `lingniu-go-stat-writer.service`
- `lingniu-go-realtime-api.service`
- 上述服务均为 `active``enabled`
- `808``32960``gateway` 监听
- `20210``realtime-api` 监听
- `/opt/lingniu-go-native/spool/gateway` 无待回放文件
## 生产环境变量
当前 goal 的生产面使用 ECS 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。以下环境变量维护在 `/opt/lingniu-go-native/env/*.env`
```bash
KAFKA_BROKERS=172.17.111.56:9092
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
IDENTITY_MYSQL_DSN=${MYSQL_DSN}
VEHICLE_IDENTITY_TABLE=vehicle_identity_binding
REDIS_ADDR=r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379
REDIS_PASSWORD=<password>
REDIS_DB=50
```
宇通 MQTT 开启时再配置:
```bash
YUTONG_MQTT_ENABLED=true
YUTONG_MQTT_URI=<mqtt-uri>
YUTONG_MQTT_TOPICS=/ytforward/shln/+
YUTONG_MQTT_CLIENT_ID=lingniu-go-yutong-mqtt
YUTONG_MQTT_USERNAME=<username>
YUTONG_MQTT_PASSWORD=<password>
```
## ECS 进程检查
```bash
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
```
## Kafka 验证
优先使用 smoke 脚本做可重复检查:
```bash
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
```
脚本会检查 `vehicle.raw.go.gb32960.v1``vehicle.raw.go.jt808.v1``vehicle.raw.go.yutong-mqtt.v1``vehicle.event.go.unified.v1` 是否存在,并汇总 `go-history-writer``go-stat-writer``go-realtime-api` 的消费 lag。运行环境需要 SSH 免密或已建立可用的 SSH 认证方式。
手工核对命令:
```bash
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.go.jt808.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.go.gb32960.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.go.yutong-mqtt.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.event.go.unified.v1 --max-messages 1 --timeout-ms 10000
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-history-writer
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-stat-writer
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-realtime-api
```
## TDengine 验证
```sql
USE lingniu_vehicle_ts;
SHOW STABLES;
SELECT COUNT(*) FROM raw_frames;
SELECT COUNT(*) FROM vehicle_locations;
SELECT COUNT(*) FROM vehicle_mileage_points;
```
## MySQL 验证
```sql
SELECT protocol, metric_key, COUNT(*)
FROM vehicle_daily_metric
GROUP BY protocol, metric_key;
SELECT *
FROM vehicle_daily_metric
WHERE metric_key IN ('daily_mileage_km', 'daily_total_mileage_km')
ORDER BY updated_at DESC
LIMIT 20;
```
## Redis 实时查询
```bash
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>'
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/online'
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/protocols/JT808'
```
## 验收证据
正式切换前至少记录:
- 一个真实 GB32960 VIN 的 RAW、location、mileage point、Redis snapshot。
- 一个真实 JT808 VIN/phone 的 RAW、location、mileage point、daily metric、Redis snapshot。
- 一个真实宇通 MQTT VIN 或 device_id 的 RAW、Redis snapshot。
- `go-history-writer``go-stat-writer` 重启后 Kafka offset 能继续推进。

View File

@@ -0,0 +1,720 @@
# Go Vehicle Ingest Redesign Phase 1 Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Build the first production-capable Go runtime for GB32960, JT808, and Yutong MQTT ingestion with unified Kafka, TDengine, MySQL statistics, and Redis realtime state boundaries.
**Architecture:** Create a new single Go module at `go/vehicle-gateway` and migrate only the useful pieces from the existing `go/ingest-edge` and `go/vehicle-state` prototypes. The gateway produces unified envelopes to Kafka; independent consumers write TDengine history, MySQL daily metrics, and Redis realtime state.
**Tech Stack:** Go 1.26+, Kafka (`segmentio/kafka-go`), Redis (`redis/go-redis`), MySQL (`go-sql-driver/mysql`), TDengine official Go connector, MQTT (`eclipse/paho.mqtt.golang`), standard library TCP.
---
## File Structure
- Create: `go/vehicle-gateway/go.mod`
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
- Create: `go/vehicle-gateway/internal/protocol/jt808/*`
- Create: `go/vehicle-gateway/internal/protocol/gb32960/*`
- Create: `go/vehicle-gateway/internal/protocol/yutongmqtt/*`
- Create: `go/vehicle-gateway/internal/gateway/*`
- Create: `go/vehicle-gateway/internal/identity/*`
- Create: `go/vehicle-gateway/internal/eventbus/*`
- Create: `go/vehicle-gateway/internal/history/*`
- Create: `go/vehicle-gateway/internal/stats/*`
- Create: `go/vehicle-gateway/internal/realtime/*`
- Create: `go/vehicle-gateway/internal/observability/*`
- Modify: `README.md`
- Modify: `docs/target-architecture.md`
- Create: `deploy/portainer/docker-compose-go.yml`
The existing `go/ingest-edge` and `go/vehicle-state` directories are migration sources only. After phase 1 is verified, remove or mark them superseded.
---
### Task 1: Create Single Go Module
**Files:**
- Create: `go/vehicle-gateway/go.mod`
- Create: `go/vehicle-gateway/internal/observability/logger.go`
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
- [ ] **Step 1: Create module manifest**
Add `go/vehicle-gateway/go.mod`:
```go
module lingniu-vehicle-ingest/go/vehicle-gateway
go 1.26
require (
github.com/eclipse/paho.mqtt.golang v1.5.1
github.com/go-sql-driver/mysql v1.9.3
github.com/redis/go-redis/v9 v9.17.2
github.com/segmentio/kafka-go v0.4.49
github.com/taosdata/driver-go/v3 v3.8.1
)
```
- [ ] **Step 2: Add logger helper**
Add `go/vehicle-gateway/internal/observability/logger.go`:
```go
package observability
import (
"log/slog"
"os"
)
func NewLogger(service string) *slog.Logger {
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
return slog.New(handler).With("service", service)
}
```
- [ ] **Step 3: Add temporary gateway entrypoint**
Add `go/vehicle-gateway/cmd/gateway/main.go`:
```go
package main
import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
func main() {
logger := observability.NewLogger("vehicle-gateway")
logger.Info("vehicle gateway scaffold started")
}
```
- [ ] **Step 4: Verify scaffold builds**
Run:
```bash
cd go/vehicle-gateway
go mod tidy
go test ./...
go build ./cmd/gateway
```
Expected: all commands exit `0`.
---
### Task 2: Define Unified Envelope
**Files:**
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
- [ ] **Step 1: Write envelope tests**
Add `go/vehicle-gateway/internal/envelope/envelope_test.go`:
```go
package envelope
import "testing"
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
t.Fatalf("VehicleKey() = %q", got)
}
}
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
if got := e.VehicleKey(); got != "JT808:013307795425" {
t.Fatalf("VehicleKey() = %q", got)
}
}
func TestFrameEnvelopeEventIDStable(t *testing.T) {
e := FrameEnvelope{
Protocol: ProtocolJT808,
MessageID: "0x0200",
Phone: "013307795425",
Sequence: 1,
EventTimeMS: 1782745114000,
ReceivedAtMS: 1782745114999,
RawHex: "7e02000000ff7e",
}
a := e.StableEventID()
b := e.StableEventID()
if a == "" || a != b {
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
}
}
```
- [ ] **Step 2: Run tests and confirm failure**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/envelope
```
Expected: fail because `FrameEnvelope` is not defined.
- [ ] **Step 3: Implement envelope**
Add `go/vehicle-gateway/internal/envelope/envelope.go`:
```go
package envelope
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
)
type Protocol string
const (
ProtocolGB32960 Protocol = "GB32960"
ProtocolJT808 Protocol = "JT808"
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
)
type ParseStatus string
const (
ParseOK ParseStatus = "OK"
ParsePartial ParseStatus = "PARTIAL"
ParseBadFrame ParseStatus = "BAD_FRAME"
)
type FrameEnvelope struct {
EventID string `json:"event_id"`
TraceID string `json:"trace_id"`
Protocol Protocol `json:"protocol"`
MessageID string `json:"message_id"`
Sequence uint16 `json:"sequence"`
VIN string `json:"vin,omitempty"`
VehicleKeyHint string `json:"vehicle_key,omitempty"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
Plate string `json:"plate,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
Parsed map[string]any `json:"parsed,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
ParseStatus ParseStatus `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
}
func (e FrameEnvelope) VehicleKey() string {
if key := strings.TrimSpace(e.VIN); key != "" {
return key
}
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
return key
}
if key := strings.TrimSpace(e.Phone); key != "" {
return string(e.Protocol) + ":" + key
}
if key := strings.TrimSpace(e.DeviceID); key != "" {
return string(e.Protocol) + ":" + key
}
return string(e.Protocol) + ":unknown"
}
func (e FrameEnvelope) StableEventID() string {
if strings.TrimSpace(e.EventID) != "" {
return e.EventID
}
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
sum := sha256.Sum256([]byte(input))
return hex.EncodeToString(sum[:16])
}
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
if e.EventID == "" {
e.EventID = e.StableEventID()
}
if e.ParseStatus == "" {
e.ParseStatus = ParseOK
}
return json.Marshal(e)
}
```
- [ ] **Step 4: Verify tests pass**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/envelope
```
Expected: pass.
---
### Task 3: Implement JT808 Frame and 0200 Parser
**Files:**
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser.go`
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser_test.go`
- [ ] **Step 1: Add sample frame tests**
Use the production sample provided in the thread:
```text
7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E
```
Expected assertions:
- message id is `0x0200`
- phone keeps normalized BCD string `013307795425`
- sequence is `1`
- `fields.total_mileage_km` is parsed from additional item `0x01`
- latitude, longitude, speed, direction, alarm, status, and device time are present
- [ ] **Step 2: Implement parser**
Implementation rules:
- strip `0x7e` start/end delimiters
- unescape `0x7d 0x02 -> 0x7e`
- unescape `0x7d 0x01 -> 0x7d`
- verify XOR checksum
- parse 2011/2013 common header
- parse BCD phone from 6 bytes and keep both raw and normalized values in `parsed.header`
- parse 0200 fixed body
- parse additional item list as `id,length,value_hex`
- parse additional `0x01` as `total_mileage_km = uint32 / 10`
- [ ] **Step 3: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/protocol/jt808
```
Expected: pass.
---
### Task 4: Implement GB32960 Frame and Data Unit Parser
**Files:**
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser.go`
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser_test.go`
- [ ] **Step 1: Add parser tests**
Tests must cover:
- `##` frame boundary
- command id
- response flag
- 17-byte VIN
- encryption flag
- payload length
- BCC verification
- realtime data command `0x02`
- reissue data command `0x03`
- data unit `0x01` vehicle status fields
- data unit `0x05` position fields
- [ ] **Step 2: Implement parser**
Implementation rules:
- parse header without allocating large temporary buffers
- keep RAW as hex in envelope
- write all recognized data units to `Parsed`
- write core fields to `Fields`
- unsupported data unit stays in `Parsed["unknown_units"]`
- [ ] **Step 3: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/protocol/gb32960
```
Expected: pass.
---
### Task 5: Implement Kafka Sink
**Files:**
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink.go`
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink_test.go`
- [ ] **Step 1: Add sink interface**
Create a sink interface:
```go
package eventbus
import (
"context"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Sink interface {
PublishRaw(context.Context, envelope.FrameEnvelope) error
PublishUnified(context.Context, envelope.FrameEnvelope) error
Close() error
}
```
- [ ] **Step 2: Implement Kafka topic routing**
Rules:
- `GB32960 -> vehicle.raw.gb32960.v1`
- `JT808 -> vehicle.raw.jt808.v1`
- `YUTONG_MQTT -> vehicle.raw.yutong-mqtt.v1`
- unified topic is `vehicle.event.unified.v1`
- message key is `env.VehicleKey()`
- [ ] **Step 3: Verify routing with unit tests**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/eventbus
```
Expected: pass.
---
### Task 6: Implement TDengine History Writer
**Files:**
- Create: `go/vehicle-gateway/internal/history/schema.go`
- Create: `go/vehicle-gateway/internal/history/writer.go`
- Create: `go/vehicle-gateway/internal/history/writer_test.go`
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
- [ ] **Step 1: Add schema bootstrap SQL**
Implement schema strings for:
- database `lingniu_vehicle_ts`
- stable `raw_frames`
- stable `vehicle_locations`
- stable `vehicle_mileage_points`
- [ ] **Step 2: Implement writer**
Rules:
- `AppendRawFrame` always writes one raw row.
- `AppendLocation` writes only when longitude and latitude exist.
- `AppendMileagePoint` writes only when `total_mileage_km` exists.
- child table name is deterministic hash of `protocol + vehicle_key`.
- escape tag values.
- [ ] **Step 3: Use TDengine official driver**
Import WebSocket driver in command:
```go
import _ "github.com/taosdata/driver-go/v3/taosWS"
```
Default driver name:
```text
TDENGINE_DRIVER=taosWS
```
Short-term compatibility:
```text
TDENGINE_DRIVER=taosSql
```
Only use `taosSql` when ECS TDengine WebSocket is not available.
- [ ] **Step 4: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/history
go build ./cmd/history-writer
```
Expected: pass.
---
### Task 7: Implement MySQL Daily Metric Writer
**Files:**
- Create: `go/vehicle-gateway/internal/stats/schema.go`
- Create: `go/vehicle-gateway/internal/stats/daily_metric.go`
- Create: `go/vehicle-gateway/internal/stats/daily_metric_test.go`
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
- [ ] **Step 1: Add schema bootstrap**
Implement `vehicle_daily_metric` schema from the design spec.
- [ ] **Step 2: Add metric derivation tests**
Test cases:
- no `total_mileage_km` produces no metric
- one sample produces:
- `daily_mileage_km = 0`
- `daily_total_mileage_km = sample`
- later larger sample updates:
- `latest_total_mileage_km`
- `daily_mileage_km`
- `daily_total_mileage_km`
- out-of-order smaller sample updates:
- `first_total_mileage_km`
- `daily_mileage_km`
- [ ] **Step 3: Implement MySQL upsert**
Use one table and one idempotent upsert:
```sql
INSERT INTO vehicle_daily_metric
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
ON DUPLICATE KEY UPDATE
first_total_mileage_km = LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)),
latest_total_mileage_km = GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)),
metric_value = CASE
WHEN metric_key = 'daily_mileage_km'
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
- LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
WHEN metric_key = 'daily_total_mileage_km'
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
ELSE VALUES(metric_value)
END,
sample_count = sample_count + 1,
updated_at = CURRENT_TIMESTAMP
```
- [ ] **Step 4: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/stats
go build ./cmd/stat-writer
```
Expected: pass.
---
### Task 8: Implement Redis Realtime API
**Files:**
- Create: `go/vehicle-gateway/internal/realtime/repository.go`
- Create: `go/vehicle-gateway/internal/realtime/repository_test.go`
- Create: `go/vehicle-gateway/internal/realtime/http.go`
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
- [ ] **Step 1: Add repository contract**
Methods:
- `Update(ctx, envelope.FrameEnvelope) error`
- `GetMerged(ctx, vin string) (Snapshot, error)`
- `GetProtocol(ctx, vin string, protocol envelope.Protocol) (Snapshot, error)`
- `IsOnline(ctx, vin string) (OnlineStatus, error)`
- [ ] **Step 2: Implement merge logic**
Rules:
- update `vehicle:latest:{vin}:{protocol}`
- update `vehicle:latest:{vin}` with newest fields
- update `vehicle:online:{vin}`
- update sorted set `vehicle:last_seen`
- [ ] **Step 3: Implement HTTP API**
Routes:
- `GET /api/realtime/vehicles/{vin}`
- `GET /api/realtime/vehicles/{vin}/online`
- `GET /api/realtime/vehicles/{vin}/protocols/{protocol}`
- [ ] **Step 4: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/realtime
go build ./cmd/realtime-api
```
Expected: pass.
---
### Task 9: Wire Gateway Runtime
**Files:**
- Create: `go/vehicle-gateway/internal/gateway/tcp_server.go`
- Create: `go/vehicle-gateway/internal/gateway/mqtt_client.go`
- Modify: `go/vehicle-gateway/cmd/gateway/main.go`
- [ ] **Step 1: Implement TCP server**
Rules:
- one goroutine per accepted connection
- bounded max connections
- read timeout and idle timeout
- protocol-specific frame extractor
- structured peer endpoint
- graceful shutdown on SIGTERM
- [ ] **Step 2: Implement MQTT client**
Rules:
- connect with official production config from environment
- subscribe configured topic list
- convert each message into envelope
- publish raw and unified events
- reconnect with backoff
- [ ] **Step 3: Verify local JSON mode**
Run without Kafka:
```bash
cd go/vehicle-gateway
GB32960_TCP_ADDR=:132960 JT808_TCP_ADDR=:18080 go run ./cmd/gateway
```
Expected: service starts and logs configured listeners.
---
### Task 10: Docker and ECS Deployment
**Files:**
- Create: `go/vehicle-gateway/Dockerfile`
- Create: `deploy/portainer/docker-compose-go.yml`
- Modify: `docs/operations/current-ecs-deployment.md`
- [ ] **Step 1: Add multi-stage Dockerfile**
Build all commands:
- `gateway`
- `history-writer`
- `stat-writer`
- `realtime-api`
- [ ] **Step 2: Add Portainer compose**
Services:
- `go-vehicle-gateway`
- `go-history-writer`
- `go-stat-writer`
- `go-realtime-api`
Each service must include:
- restart policy
- memory limit
- Kafka env
- MySQL/TDengine/Redis env as needed
- logging options
- [ ] **Step 3: Verify Linux build**
Run:
```bash
cd go/vehicle-gateway
GOOS=linux GOARCH=amd64 go build ./cmd/gateway
GOOS=linux GOARCH=amd64 go build ./cmd/history-writer
GOOS=linux GOARCH=amd64 go build ./cmd/stat-writer
GOOS=linux GOARCH=amd64 go build ./cmd/realtime-api
```
Expected: all commands exit `0`.
---
### Task 11: Production Verification
**Files:**
- Create: `docs/operations/go-vehicle-gateway-verification.md`
- [ ] **Step 1: Record test commands**
Document commands to verify:
- gateway process health
- Kafka topic consumption
- TDengine row counts
- MySQL daily metric rows
- Redis realtime lookup
- [ ] **Step 2: Validate real traffic**
Evidence required:
- one real 32960 VIN with RAW, location, mileage point, daily metric, Redis snapshot
- one real JT808 phone/VIN with RAW, location, mileage point, daily metric, Redis snapshot
- one real Yutong MQTT VIN with RAW and Redis snapshot
- [ ] **Step 3: Keep old Java services until evidence is captured**
Only disable Java equivalents after the evidence file contains successful command outputs and timestamps.
---
## Self-Review Checklist
- The plan creates one new Go module instead of extending scattered prototypes.
- The plan covers GB32960, JT808, and Yutong MQTT ingress.
- The plan covers Kafka, TDengine, MySQL, and Redis.
- The plan includes 32960 and 808 daily mileage and daily total mileage.
- The plan includes local and ECS verification.
- The plan does not restore Xinda Push.
- The plan keeps Java services untouched until Go evidence exists.

View File

@@ -0,0 +1,498 @@
# Go 车辆接入重构设计规格
## 目标
使用 Go 重新设计车辆数据接入运行面,让系统回到第一性原则:
- 接入层只负责可靠收包、协议解析、应答、身份解析和投递。
- Kafka 是唯一在线消息通道。
- TDengine 保存高吞吐时序明细和完整 RAW 解析 JSON。
- MySQL 保存低频配置、身份映射和每日统计窄表。
- Redis 保存可重建的准实时状态。
- 32960、JT808、宇通 MQTT 使用同一套数据层接口,避免每个协议各自落库。
第一阶段只做生产链路的骨架和核心数据闭环接收、解析、Kafka、TDengine、MySQL 统计、Redis 实时查询、ECS 验证。历史查询 API 和业务侧复杂分析放在后续阶段。
## 参考原则
本设计只吸收开源项目的边界思想,不复制外部实现代码。
- GB32960 参考 `DarkInno/gb32960-go-sdk`连接管理、Handler、Forwarder、鉴权接口分离。
- JT808 参考 `cuteLittleDevil/go-jt808`:高并发 Go 原生 TCP、协议核心小而可扩展、事件/适配器机制。
- JT808 参考 `fakeyanss/jt808-server-go``FramePayload -> PacketData -> JT808Msg -> Reply` 的分层处理、Gateway 模式、保活和版本兼容。
- TDengine 参考官方数据模型:按数据采集点建立 supertable静态维度做 tags明细数据按子表写入Go 连接优先使用 WebSocket 驱动。
## 非目标
- 第一阶段不继续扩展 Java 服务。
- 第一阶段不做统一大查询 API。
- 第一阶段不恢复信达 Push。
- 第一阶段不把所有协议字段展开成一张超宽 TDengine 表。
- Redis 不作为历史事实源。
- MySQL 不保存高频原始明细。
## 目标目录
新的 Go 运行面放在 `go/vehicle-gateway`,不继续沿用实验性的多 module 分散结构。现有 `go/ingest-edge``go/vehicle-state` 可以作为迁移素材,最终归并或删除。
```text
go/vehicle-gateway/
cmd/
gateway/ # 协议接入进程32960 TCP、JT808 TCP、宇通 MQTT
history-writer/ # Kafka -> TDengine RAW/location/mileage points
stat-writer/ # Kafka -> MySQL 每日统计窄表
realtime-api/ # Redis 准实时查询 API
internal/
config/ # 环境变量、配置文件、校验
gateway/ # TCP/MQTT 生命周期、连接限制、优雅停机
protocol/
gb32960/ # 32960 编解码
jt808/ # 808 编解码
yutongmqtt/ # 宇通 MQTT payload 解析
identity/ # VIN、phone、device_id、plate 绑定
envelope/ # 统一事件模型
eventbus/ # Kafka producer/consumer
history/ # TDengine adapter
stats/ # 每日里程聚合
realtime/ # Redis adapter 与快照合并
observability/ # 日志、metrics、健康检查
```
## 协议接入设计
### GB32960
职责:
- 监听 TCP `32960`
- 支持车辆登入、实时信息上报、补发信息上报、车辆登出、平台登入/登出、心跳、校时。
- 正确返回协议应答。
- 完整解析 2016 数据单元:
- 整车数据
- 驱动电机数据
- 燃料电池数据
- 发动机数据
- 车辆位置数据
- 极值数据
- 报警数据
- 可充电储能装置电压数据
- 可充电储能装置温度数据
- RAW 中保存完整 parsed JSON。
- 关键字段标准化为统一 field
- `speed_kmh`
- `total_mileage_km`
- `soc_percent`
- `longitude`
- `latitude`
- `vehicle_status`
- `charge_status`
### JT808
职责:
- 监听 TCP `808`
- 支持 2011/2013 基础兼容,预留 2019 版本标记。
- 分层处理:
- Frame`0x7e` 包边界、转义、校验。
- Packet消息头、手机号 BCD、流水号、分包。
- Message注册、鉴权、心跳、注销、位置、批量位置、未知上行。
- Reply注册应答、平台通用应答。
- 终端手机号按协议头 BCD 解析,内部同时保留原始 BCD、规范化 phone、去前导零 phone。
- 注册帧和鉴权帧写入 808 registration 表。
- 如果设备直接上报位置帧,也要走 identity 解析,尝试用 phone、device_id、plate 找到 VIN。
- 0200 附加信息必须解析表 27
- `0x01` GPS 总里程DWORD单位 0.1 km。
- 其他已知附加项结构化放入 `parsed_json.additional`
- 标准化字段:
- `speed_kmh`
- `total_mileage_km`
- `longitude`
- `latitude`
- `altitude_m`
- `direction_deg`
- `alarm_flag`
- `status_flag`
### 宇通 MQTT
职责:
- 使用正式 MQTT 配置订阅生产 topic。
- 接收 payload 后解析为统一 envelope。
- 按 payload 中的车辆标识解析 VIN、车牌、设备号。
- 保存完整 payload 和 parsed JSON。
- 标准化字段向 32960/808 对齐:
- `speed_kmh`
- `total_mileage_km`
- `longitude`
- `latitude`
- `soc_percent`
- `vehicle_status`
## 统一 Envelope
所有协议接入后输出同一结构。
```json
{
"event_id": "string",
"trace_id": "string",
"protocol": "GB32960|JT808|YUTONG_MQTT",
"message_id": "string",
"vin": "string",
"vehicle_key": "string",
"phone": "string",
"device_id": "string",
"plate": "string",
"source_endpoint": "ip:port or mqtt://broker/topic",
"event_time_ms": 0,
"received_at_ms": 0,
"raw_hex": "string",
"raw_text": "string",
"parsed": {},
"fields": {
"speed_kmh": 0,
"total_mileage_km": 0,
"longitude": 0,
"latitude": 0
},
"parse_status": "OK|PARTIAL|BAD_FRAME",
"parse_error": ""
}
```
规则:
- `event_id` 使用协议、设备标识、消息流水、事件时间、RAW hash 生成,保证幂等。
- Kafka partition key 优先使用 VIN没有 VIN 时使用 `protocol:phone/device_id`
- `parsed` 保存协议完整结构化字段。
- `fields` 只保存跨协议核心字段,用于统计、位置和实时快照。
## Kafka Topic
生产只支持 Kafka。
| Topic | 内容 | Key |
|---|---|---|
| `vehicle.raw.gb32960.v1` | 32960 完整 RAW envelope | VIN 或 vehicle_key |
| `vehicle.raw.jt808.v1` | 808 完整 RAW envelope | VIN 或 phone |
| `vehicle.raw.yutong-mqtt.v1` | 宇通 MQTT 完整 RAW envelope | VIN 或 vehicle_key |
| `vehicle.event.unified.v1` | 统一轻量事件,用于业务消费 | VIN 或 vehicle_key |
接入层必须先写 RAW topic再写 unified event。后续消费者只依赖 Kafka不直接依赖接入进程内存。
发布可靠性:
- Kafka 生产端必须开启同步写入RAW 写成功后才允许写 unified event。
- Kafka 写入必须支持可配置重试、单次写超时和短退避,默认值为 `KAFKA_PUBLISH_ATTEMPTS=3``KAFKA_PUBLISH_TIMEOUT_MS=3000``KAFKA_PUBLISH_BACKOFF_MS=100`
- gateway 支持本地磁盘 spool/WAL配置 `KAFKA_SPOOL_DIR` 后启用Kafka 长时间不可用时,发布失败的 envelope 先原子写入本地 JSON 文件。
- spool 补发按文件名顺序执行,补发成功后删除文件;默认补发间隔由 `KAFKA_SPOOL_REPLAY_INTERVAL_MS=1000` 控制。
- 如果 RAW 写 Kafka 失败并已落盘,同一个 event 的 unified event 不允许抢先写 Kafka也必须进入 spool确保恢复后按 RAW -> unified 顺序补发。
- Kafka topic 不允许自动创建topic 和分区数由部署脚本或运维初始化,避免生产拼写错误造成隐性分流。
## TDengine 数据库设计
如果现有 TDengine 库不符合目标,可以新建库并迁移。建议库名:
```sql
CREATE DATABASE IF NOT EXISTS lingniu_vehicle_ts KEEP 7300 DURATION 10 BUFFER 256;
```
### raw_frames
保存完整 RAW 和完整 parsed JSON。
```sql
CREATE STABLE IF NOT EXISTS raw_frames (
ts TIMESTAMP,
frame_id NCHAR(64),
event_id NCHAR(64),
message_id INT,
event_time TIMESTAMP,
received_at TIMESTAMP,
raw_size_bytes INT,
raw_hex BINARY(16374),
raw_text BINARY(16374),
parsed_json BINARY(16374),
fields_json BINARY(4096),
parse_status NCHAR(16),
parse_error BINARY(1024),
source_endpoint NCHAR(128)
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
);
```
说明:
- 子表按 `protocol + vehicle_key hash` 创建。
- RAW 查询优先按 tags 和时间范围过滤。
- `parsed_json` 是完整协议字段;查询 API 可选择是否返回,避免默认大字段拖慢。
### vehicle_locations
只保存位置查询核心字段。
```sql
CREATE STABLE IF NOT EXISTS vehicle_locations (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
longitude DOUBLE,
latitude DOUBLE,
altitude_m DOUBLE,
speed_kmh DOUBLE,
direction_deg INT,
alarm_flag BIGINT,
status_flag BIGINT,
total_mileage_km DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
);
```
### vehicle_mileage_points
保存总里程采样点,用于回放重算和查询。
```sql
CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
total_mileage_km DOUBLE,
speed_kmh DOUBLE,
longitude DOUBLE,
latitude DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
);
```
## MySQL 数据库设计
MySQL 只保存配置、身份映射、808 注册鉴权和每日统计窄表。
### vehicle_identity_binding
用于把外部标识定位到 VIN。
```sql
CREATE TABLE vehicle_identity_binding (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vin VARCHAR(32) NOT NULL,
plate VARCHAR(32) NULL,
phone VARCHAR(32) NULL,
device_id VARCHAR(64) NULL,
remark VARCHAR(255) NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_vin (vin),
KEY idx_plate (plate),
KEY idx_phone (phone),
KEY idx_device_id (device_id)
);
```
### jt808_registration
只记录 808 独有的注册、鉴权和在线身份信息。
```sql
CREATE TABLE jt808_registration (
phone VARCHAR(32) PRIMARY KEY,
vin VARCHAR(32) NULL,
plate VARCHAR(32) NULL,
device_id VARCHAR(64) NULL,
manufacturer_id VARCHAR(32) NULL,
terminal_model VARCHAR(64) NULL,
terminal_id VARCHAR(64) NULL,
auth_code VARCHAR(128) NULL,
source_endpoint VARCHAR(128) NULL,
first_registered_at DATETIME NULL,
latest_registered_at DATETIME NULL,
latest_authed_at DATETIME NULL,
latest_seen_at DATETIME NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
KEY idx_vin (vin),
KEY idx_plate (plate),
KEY idx_device_id (device_id)
);
```
### vehicle_daily_metric
32960 和 808 的每日里程、每日总里程都进入同一张窄表。
```sql
CREATE TABLE vehicle_daily_metric (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vin VARCHAR(32) NOT NULL,
stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL,
metric_key VARCHAR(64) NOT NULL,
metric_value DECIMAL(18,3) NOT NULL,
metric_unit VARCHAR(16) NOT NULL,
first_total_mileage_km DECIMAL(18,3) NULL,
latest_total_mileage_km DECIMAL(18,3) NULL,
sample_count BIGINT NOT NULL DEFAULT 0,
calculation_method VARCHAR(64) NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
KEY idx_stat_date (stat_date),
KEY idx_protocol_metric (protocol, metric_key)
);
```
指标:
- `daily_mileage_km = max(total_mileage_km) - min(total_mileage_km)`
- `daily_total_mileage_km = max(total_mileage_km)`
统计消费者可幂等重放。每个样本 upsert 时只维护当天 `min/max total_mileage_km`,不依赖进程内状态。
## Redis 实时数据设计
Redis 保存可从 Kafka 重建的数据。
| Key | 内容 |
|---|---|
| `vehicle:online:{vin}` | 在线状态、最后活跃时间、协议列表 |
| `vehicle:latest:{vin}` | 合并后的 VIN 最新快照 |
| `vehicle:latest:{vin}:{protocol}` | 单协议最新快照 |
| `vehicle:last_seen` | VIN 到最后接收时间的 sorted set |
合并规则:
- 位置以最新 event_time 为准。
- 总里程取最新上报值,不做递增假设。
- 32960 多帧字段按 field timestamp 合并。
- 808 和 MQTT 不覆盖其他协议独有字段。
- Redis TTL 用于在线判断,不作为数据删除依据。
## 数据流
```mermaid
flowchart LR
source["车辆 / 平台"] --> gateway["Go gateway"]
gateway --> parser["protocol parser"]
parser --> identity["identity resolver"]
identity --> kafkaRaw["Kafka RAW topics"]
identity --> kafkaEvent["Kafka unified event"]
kafkaRaw --> history["history-writer"]
kafkaRaw --> stats["stat-writer"]
kafkaEvent --> realtime["realtime consumer"]
history --> td["TDengine raw_frames / locations / mileage_points"]
stats --> mysql["MySQL vehicle_daily_metric"]
realtime --> redis["Redis latest state"]
```
## 错误处理
- 协议坏帧仍写 RAW topic`parse_status=BAD_FRAME`
- 可部分解析的帧写 `parse_status=PARTIAL`,保留 parse error。
- Kafka 写失败时接入层先按配置重试;重试耗尽后不写 unified event必须打错误日志和 metrics。
- 启用 `KAFKA_SPOOL_DIR`Kafka 重试耗尽会落本地 spool如果 spool 写失败,才视为接入层最终失败。
- TDengine 写失败不提交 Kafka offset。
- MySQL 统计写失败不提交 Kafka offset。
- Redis 写失败不影响历史和统计,但要通过 metrics 暴露。
- 808 相同 phone 新连接时,旧连接应被替换并记录 latest_seen。
## 部署设计
生产 ECS 运行 Go 二进制或容器:
- `vehicle-gateway`
- `vehicle-history-writer`
- `vehicle-stat-writer`
- `vehicle-realtime-api`
配置来源:
- 环境变量优先。
- Nacos 可作为后续配置中心,但 Go 第一阶段必须能只靠环境变量启动,便于 Portainer 部署和故障恢复。
TDengine 连接:
- 优先 WebSocket `taosWS`
- 若当前 ECS 只开放原生端口,可短期兼容 `taosSql`,但代码配置必须显式标记为兼容模式。
## 验证计划
### 本地验证
- 用样例 808 报文验证:
- phone BCD 解析。
- 0200 经纬度、速度、方向、时间。
- 表 27 `0x01` GPS 总里程。
- 用真实 32960 报文验证:
- 登录应答正确。
- 实时帧和补发帧都能写 RAW。
- 位置和总里程字段进入统一 fields。
- 用宇通 MQTT payload 验证:
- payload 完整保存。
- 核心字段归一化。
- 不连 Kafka 时可输出 JSON log。
- 连接 Kafka 时 RAW topic 可消费到 envelope。
### ECS 验证
- 部署前先旁路接收或短窗口切流。
- 验证端口:
- `32960`
- `808`
- MQTT 订阅连接。
- 验证 Kafka topic 有持续写入。
- 验证 TDengine
- `raw_frames` 有三种协议数据。
- `vehicle_locations` 能按 VIN、协议、时间分页查询。
- `vehicle_mileage_points` 有 32960 和 808 总里程采样。
- 验证 MySQL
- `vehicle_daily_metric` 有 32960 和 808 的 `daily_mileage_km``daily_total_mileage_km`
- 验证 Redis
- 查询 VIN 在线。
- 查询 VIN 合并实时快照。
- 查询 VIN 分协议实时快照。
## 验收标准
- Go gateway 能在 ECS 上接收真实 32960、808、宇通 MQTT 数据。
- 三种协议 RAW 都进入 Kafka。
- Kafka 短暂写失败时gateway 按配置重试;单元测试覆盖首次失败后成功和重试耗尽返回错误。
- Kafka 长时间不可用时gateway 可把 RAW/unified 写入本地 spool单元测试覆盖 RAW 已落盘时 unified 也落盘、恢复后按顺序补发并删除文件。
- 三种协议 RAW 和 parsed JSON 都进入 TDengine。
- 32960 和 808 的位置进入 `vehicle_locations`
- 32960 和 808 的总里程采样进入 `vehicle_mileage_points`
- 32960 和 808 的每日里程、每日总里程进入 MySQL 窄表。
- Redis 可以查询 VIN 在线和实时数据。
- 任一消费者宕机后可通过 Kafka offset 继续恢复。
- 查询和统计验证使用 ECS 上的真实数据完成。
## 实施顺序
1. 创建 `go/vehicle-gateway` 单 module迁移现有 Go 原型中可用的解析、Kafka、TDengine、MySQL、Redis 代码。
2. 先写协议解析单元测试,覆盖 808 样例报文和 32960 核心字段。
3. 完成统一 envelope 和 Kafka sink。
4. 完成 TDengine schema bootstrap 与 history writer。
5. 完成 MySQL schema bootstrap 与 stat writer。
6. 完成 Redis realtime writer/API。
7. 本地启动 gateway用 ECS 转发流量验证。
8. 构建 Linux 镜像,部署 ECS。
9. 按验收标准逐项验证并记录证据。

View File

@@ -0,0 +1,29 @@
FROM golang:1.26-bookworm AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/gateway ./cmd/gateway \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/history-writer ./cmd/history-writer \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/stat-writer ./cmd/stat-writer \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/realtime-api ./cmd/realtime-api
FROM debian:bookworm-slim
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates tzdata \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=build /out/gateway /app/gateway
COPY --from=build /out/history-writer /app/history-writer
COPY --from=build /out/stat-writer /app/stat-writer
COPY --from=build /out/realtime-api /app/realtime-api
ENV TZ=Asia/Shanghai
EXPOSE 808 32960 20210
CMD ["/app/gateway"]

View File

@@ -0,0 +1,223 @@
package main
import (
"context"
"database/sql"
"log/slog"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
_ "github.com/go-sql-driver/mysql"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/gateway"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
)
func main() {
logger := observability.NewLogger("vehicle-gateway")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
sink, err := buildSink(ctx, logger)
if err != nil {
logger.Error("build sink failed", "error", err)
os.Exit(1)
}
defer sink.Close()
resolver, closeResolver, err := buildIdentityResolver(ctx, logger)
if err != nil {
logger.Error("build identity resolver failed", "error", err)
os.Exit(1)
}
defer closeResolver()
protocols := []gateway.TCPProtocol{
{
Protocol: envelope.ProtocolGB32960,
Addr: env("GB32960_TCP_ADDR", ":32960"),
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Respond: gb32960.AutoResponse,
},
{
Protocol: envelope.ProtocolJT808,
Addr: env("JT808_TCP_ADDR", ":808"),
Extract: jt808.ExtractFrames,
Parse: jt808.ParseFrame,
Respond: jt808.NewAutoResponder(env("JT808_REGISTER_AUTH_CODE", "g7gps")).Respond,
},
}
var wg sync.WaitGroup
errs := make(chan error, len(protocols))
for _, protocol := range protocols {
server, err := gateway.NewTCPServer(gateway.TCPServerConfig{
Protocol: protocol,
Sink: sink,
Resolver: resolver,
Logger: logger,
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
})
if err != nil {
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
os.Exit(1)
}
wg.Add(1)
go func() {
defer wg.Done()
if err := server.ListenAndServe(ctx); err != nil {
errs <- err
}
}()
}
if envBool("YUTONG_MQTT_ENABLED", false) {
client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{
EndpointName: env("YUTONG_MQTT_ENDPOINT", env("YUTONG_MQTT_ENDPOINT_NAME", "yutong")),
Broker: env("YUTONG_MQTT_URI", ""),
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
Username: env("YUTONG_MQTT_USERNAME", ""),
Password: env("YUTONG_MQTT_PASSWORD", ""),
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
CleanSession: envBool("YUTONG_MQTT_CLEAN_SESSION", false),
KeepAlive: time.Duration(envInt("YUTONG_MQTT_KEEP_ALIVE_SECONDS", 20)) * time.Second,
ConnectTimeout: time.Duration(envInt("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", 10)) * time.Second,
TLSCACertPath: env("YUTONG_MQTT_TLS_CA_PEM", ""),
TLSClientCertPath: env("YUTONG_MQTT_TLS_CLIENT_PEM", ""),
TLSClientKeyPath: env("YUTONG_MQTT_TLS_CLIENT_KEY", ""),
TLSHostnameVerification: envBool("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", true),
Sink: sink,
Resolver: resolver,
Logger: logger,
})
if err != nil {
logger.Error("build yutong mqtt client failed", "error", err)
os.Exit(1)
}
if err := client.Start(ctx); err != nil {
logger.Error("start yutong mqtt client failed", "error", err)
os.Exit(1)
}
logger.Info("yutong mqtt client started")
}
logger.Info("vehicle gateway started")
select {
case <-ctx.Done():
case err := <-errs:
logger.Error("vehicle gateway listener failed", "error", err)
stop()
}
wg.Wait()
}
func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.Resolver, func(), error) {
dsn := env("IDENTITY_MYSQL_DSN", strings.TrimSpace(os.Getenv("MYSQL_DSN")))
if strings.TrimSpace(dsn) == "" {
logger.Warn("identity mysql dsn is empty; using noop identity resolver")
return identity.NoopResolver{}, func() {}, nil
}
db, err := sql.Open("mysql", dsn)
if err != nil {
return nil, nil, err
}
if err := db.PingContext(ctx); err != nil {
_ = db.Close()
return nil, nil, err
}
table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding")
logger.Info("identity mysql resolver enabled", "table", table)
return identity.NewMySQLResolver(db, table), func() { _ = db.Close() }, nil
}
func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error) {
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
if len(brokers) == 0 {
logger.Warn("KAFKA_BROKERS is empty; using log sink")
return eventbus.NewLogSink(logger), nil
}
sink, err := eventbus.NewKafkaSink(eventbus.KafkaConfig{
Brokers: brokers,
RawTopics: map[envelope.Protocol]string{
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1"),
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1"),
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1"),
},
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1"),
})
if err != nil {
return nil, err
}
var out eventbus.Sink = eventbus.NewRetryingSink(sink, eventbus.RetryConfig{
Attempts: envInt("KAFKA_PUBLISH_ATTEMPTS", 3),
Backoff: time.Duration(envInt("KAFKA_PUBLISH_BACKOFF_MS", 100)) * time.Millisecond,
AttemptTimeout: time.Duration(envInt("KAFKA_PUBLISH_TIMEOUT_MS", 3000)) * time.Millisecond,
})
spoolDir := strings.TrimSpace(os.Getenv("KAFKA_SPOOL_DIR"))
if spoolDir != "" {
durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{Directory: spoolDir})
interval := time.Duration(envInt("KAFKA_SPOOL_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond
go durable.ReplayLoop(ctx, interval, func(err error) {
logger.Warn("kafka spool replay failed", "error", err)
})
logger.Info("kafka durable spool enabled", "dir", spoolDir, "replay_interval_ms", interval.Milliseconds())
out = durable
}
return out, nil
}
func env(key string, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func envInt(key string, fallback int) int {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
var out int
for _, r := range value {
if r < '0' || r > '9' {
return fallback
}
out = out*10 + int(r-'0')
}
if out <= 0 {
return fallback
}
return out
}
func envBool(key string, fallback bool) bool {
value := strings.ToLower(strings.TrimSpace(os.Getenv(key)))
if value == "" {
return fallback
}
return value == "1" || value == "true" || value == "yes" || value == "on"
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item != "" {
out = append(out, item)
}
}
return out
}

View File

@@ -0,0 +1,123 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"os"
"os/signal"
"strings"
"syscall"
"github.com/segmentio/kafka-go"
_ "github.com/taosdata/driver-go/v3/taosWS"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
)
func main() {
logger := observability.NewLogger("vehicle-history-writer")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cfg := loadConfig()
db, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
if err != nil {
logger.Error("tdengine open failed", "error", err)
os.Exit(1)
}
defer db.Close()
if err := db.PingContext(ctx); err != nil {
logger.Error("tdengine ping failed", "error", err)
os.Exit(1)
}
writer := history.NewWriter(db)
if cfg.EnsureSchema {
if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
logger.Error("tdengine schema bootstrap failed", "error", err)
os.Exit(1)
}
}
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: cfg.KafkaBrokers,
GroupID: cfg.KafkaGroup,
GroupTopics: cfg.KafkaTopics,
MinBytes: 1,
MaxBytes: 10e6,
})
defer reader.Close()
logger.Info("history writer started",
"driver", cfg.TDengineDriver,
"group", cfg.KafkaGroup,
"topics", strings.Join(cfg.KafkaTopics, ","))
for {
message, err := reader.FetchMessage(ctx)
if err != nil {
if ctx.Err() != nil {
return
}
logger.Error("kafka fetch failed", "error", err)
continue
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = reader.CommitMessages(ctx, message)
continue
}
if err := writer.AppendAll(ctx, env); err != nil {
logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
continue
}
if err := reader.CommitMessages(ctx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
}
}
type config struct {
KafkaBrokers []string
KafkaTopics []string
KafkaGroup string
TDengineDriver string
TDengineDSN string
TDengineDatabase string
EnsureSchema bool
}
func loadConfig() config {
return config{
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1")),
KafkaGroup: env("KAFKA_GROUP", "go-history-writer"),
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
TDengineDSN: env("TDENGINE_DSN", ""),
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
EnsureSchema: env("TDENGINE_ENSURE_SCHEMA", "true") != "false",
}
}
func env(key string, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item != "" {
out = append(out, item)
}
}
return out
}

View File

@@ -0,0 +1,198 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
_ "github.com/go-sql-driver/mysql"
"github.com/redis/go-redis/v9"
"github.com/segmentio/kafka-go"
_ "github.com/taosdata/driver-go/v3/taosWS"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
func main() {
logger := observability.NewLogger("vehicle-realtime-api")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
client := redis.NewClient(&redis.Options{
Addr: env("REDIS_ADDR", "127.0.0.1:6379"),
Username: env("REDIS_USERNAME", ""),
Password: env("REDIS_PASSWORD", ""),
DB: envInt("REDIS_DB", 0),
})
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
logger.Error("redis ping failed", "error", err)
os.Exit(1)
}
repository := realtime.NewRepository(client, realtime.Config{
OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", 600)) * time.Second,
})
if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
go consumeKafka(ctx, logger, repository, brokers)
} else {
logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
}
mux := http.NewServeMux()
mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository))
closeStats := func() {}
if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" {
db, err := sql.Open("mysql", dsn)
if err != nil {
logger.Error("stats mysql open failed", "error", err)
os.Exit(1)
}
if err := db.PingContext(ctx); err != nil {
_ = db.Close()
logger.Error("stats mysql ping failed", "error", err)
os.Exit(1)
}
closeStats = func() { _ = db.Close() }
mux.Handle("/api/stats/daily-metrics", stats.NewMetricHandler(stats.NewMetricRepository(db)))
logger.Info("stats mysql query enabled")
} else {
mux.HandleFunc("/api/stats/daily-metrics", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(map[string]any{"error": "MYSQL_DSN is not configured"})
})
logger.Warn("MYSQL_DSN is empty; stats query api disabled")
}
defer closeStats()
closeHistory := func() {}
if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" {
driver := env("TDENGINE_DRIVER", "taosWS")
db, err := sql.Open(driver, dsn)
if err != nil {
logger.Error("history tdengine open failed", "driver", driver, "error", err)
os.Exit(1)
}
if err := db.PingContext(ctx); err != nil {
_ = db.Close()
logger.Error("history tdengine ping failed", "driver", driver, "error", err)
os.Exit(1)
}
closeHistory = func() { _ = db.Close() }
database := env("TDENGINE_DATABASE", history.DefaultDatabase)
mux.Handle("/api/history/raw-frames", history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)))
mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database)))
mux.Handle("/api/history/mileage-points", history.NewMileagePointHandler(history.NewMileagePointRepository(db, database)))
logger.Info("history query enabled", "driver", driver, "database", database)
} else {
historyUnavailable := func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"})
}
mux.HandleFunc("/api/history/raw-frames", historyUnavailable)
mux.HandleFunc("/api/history/locations", historyUnavailable)
mux.HandleFunc("/api/history/mileage-points", historyUnavailable)
logger.Warn("TDENGINE_DSN is empty; history query api disabled")
}
defer closeHistory()
server := &http.Server{
Addr: env("HTTP_ADDR", ":20210"),
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
}
go func() {
<-ctx.Done()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = server.Shutdown(shutdownCtx)
}()
logger.Info("realtime api started", "addr", server.Addr)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error("http server failed", "error", err)
os.Exit(1)
}
}
func consumeKafka(ctx context.Context, logger interface {
Info(string, ...any)
Error(string, ...any)
Warn(string, ...any)
}, repository *realtime.Repository, brokers []string) {
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: brokers,
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
GroupTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.event.unified.v1")),
MinBytes: 1,
MaxBytes: 10e6,
})
defer reader.Close()
logger.Info("realtime kafka consumer started", "topics", env("KAFKA_TOPICS", "vehicle.event.unified.v1"))
for {
message, err := reader.FetchMessage(ctx)
if err != nil {
if ctx.Err() != nil {
return
}
logger.Error("kafka fetch failed", "error", err)
continue
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = reader.CommitMessages(ctx, message)
continue
}
if err := repository.Update(ctx, env); err != nil {
logger.Error("redis realtime update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
continue
}
if err := reader.CommitMessages(ctx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
}
}
func env(key string, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func envInt(key string, fallback int) int {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
parsed, err := strconv.Atoi(value)
if err != nil {
return fallback
}
return parsed
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item != "" {
out = append(out, item)
}
}
return out
}

View File

@@ -0,0 +1,122 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"os"
"os/signal"
"strings"
"syscall"
"time"
_ "github.com/go-sql-driver/mysql"
"github.com/segmentio/kafka-go"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
func main() {
logger := observability.NewLogger("vehicle-stat-writer")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cfg := loadConfig()
db, err := sql.Open("mysql", cfg.MySQLDSN)
if err != nil {
logger.Error("mysql open failed", "error", err)
os.Exit(1)
}
defer db.Close()
if err := db.PingContext(ctx); err != nil {
logger.Error("mysql ping failed", "error", err)
os.Exit(1)
}
writer := stats.NewWriter(db, cfg.Location)
if cfg.EnsureSchema {
if err := writer.EnsureSchema(ctx); err != nil {
logger.Error("mysql schema bootstrap failed", "error", err)
os.Exit(1)
}
}
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: cfg.KafkaBrokers,
GroupID: cfg.KafkaGroup,
GroupTopics: cfg.KafkaTopics,
MinBytes: 1,
MaxBytes: 10e6,
})
defer reader.Close()
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","))
for {
message, err := reader.FetchMessage(ctx)
if err != nil {
if ctx.Err() != nil {
return
}
logger.Error("kafka fetch failed", "error", err)
continue
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = reader.CommitMessages(ctx, message)
continue
}
if err := writer.Append(ctx, env); err != nil {
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
continue
}
if err := reader.CommitMessages(ctx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
}
}
type config struct {
KafkaBrokers []string
KafkaTopics []string
KafkaGroup string
MySQLDSN string
EnsureSchema bool
Location *time.Location
}
func loadConfig() config {
loc, err := time.LoadLocation(env("LOCAL_TZ", "Asia/Shanghai"))
if err != nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
return config{
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1")),
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
MySQLDSN: env("MYSQL_DSN", ""),
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",
Location: loc,
}
}
func env(key string, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item != "" {
out = append(out, item)
}
}
return out
}

30
go/vehicle-gateway/go.mod Normal file
View File

@@ -0,0 +1,30 @@
module lingniu-vehicle-ingest/go/vehicle-gateway
go 1.26
require (
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/alicebob/miniredis/v2 v2.35.0
github.com/eclipse/paho.mqtt.golang v1.5.1
github.com/go-sql-driver/mysql v1.9.3
github.com/redis/go-redis/v9 v9.17.2
github.com/segmentio/kafka-go v0.4.49
github.com/taosdata/driver-go/v3 v3.8.1
golang.org/x/text v0.29.0
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
golang.org/x/net v0.44.0 // indirect
golang.org/x/sync v0.17.0 // indirect
)

73
go/vehicle-gateway/go.sum Normal file
View File

@@ -0,0 +1,73 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE=
github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/taosdata/driver-go/v3 v3.8.1 h1:kkd4ABsGiU+oXDbsw/sic985LKAvnpF9Gb/TEunTnLE=
github.com/taosdata/driver-go/v3 v3.8.1/go.mod h1:S6OGOinfR0xxxaMGsvBi9cLkYxEIW1p6qqr8QJATTlg=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,95 @@
package envelope
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
)
type Protocol string
const (
ProtocolGB32960 Protocol = "GB32960"
ProtocolJT808 Protocol = "JT808"
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
)
type ParseStatus string
const (
ParseOK ParseStatus = "OK"
ParsePartial ParseStatus = "PARTIAL"
ParseBadFrame ParseStatus = "BAD_FRAME"
)
const (
FieldSpeedKMH = "speed_kmh"
FieldTotalMileageKM = "total_mileage_km"
FieldLongitude = "longitude"
FieldLatitude = "latitude"
FieldSOCPercent = "soc_percent"
)
type FrameEnvelope struct {
EventID string `json:"event_id"`
TraceID string `json:"trace_id"`
Protocol Protocol `json:"protocol"`
MessageID string `json:"message_id"`
Sequence uint16 `json:"sequence"`
VIN string `json:"vin,omitempty"`
VehicleKeyHint string `json:"vehicle_key,omitempty"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
Plate string `json:"plate,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
Parsed map[string]any `json:"parsed,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
ParseStatus ParseStatus `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
}
func (e FrameEnvelope) VehicleKey() string {
if key := strings.TrimSpace(e.VIN); key != "" {
return key
}
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
return key
}
if key := strings.TrimSpace(e.Phone); key != "" {
return string(e.Protocol) + ":" + key
}
if key := strings.TrimSpace(e.DeviceID); key != "" {
return string(e.Protocol) + ":" + key
}
return string(e.Protocol) + ":unknown"
}
func (e FrameEnvelope) StableEventID() string {
if strings.TrimSpace(e.EventID) != "" {
return e.EventID
}
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
sum := sha256.Sum256([]byte(input))
return hex.EncodeToString(sum[:16])
}
func (e FrameEnvelope) KafkaKey() []byte {
return []byte(e.VehicleKey())
}
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
if e.EventID == "" {
e.EventID = e.StableEventID()
}
if e.ParseStatus == "" {
e.ParseStatus = ParseOK
}
return json.Marshal(e)
}

View File

@@ -0,0 +1,55 @@
package envelope
import (
"encoding/json"
"testing"
)
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
t.Fatalf("VehicleKey() = %q", got)
}
}
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
if got := e.VehicleKey(); got != "JT808:013307795425" {
t.Fatalf("VehicleKey() = %q", got)
}
}
func TestFrameEnvelopeEventIDStable(t *testing.T) {
e := FrameEnvelope{
Protocol: ProtocolJT808,
MessageID: "0x0200",
Phone: "013307795425",
Sequence: 1,
EventTimeMS: 1782745114000,
ReceivedAtMS: 1782745114999,
RawHex: "7e02000000ff7e",
}
a := e.StableEventID()
b := e.StableEventID()
if a == "" || a != b {
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
}
}
func TestFrameEnvelopeMarshalDefaults(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolGB32960, VIN: "LNBVIN00000000002", MessageID: "0x02"}
payload, err := e.MarshalJSONBytes()
if err != nil {
t.Fatalf("MarshalJSONBytes() error = %v", err)
}
var decoded FrameEnvelope
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if decoded.EventID == "" {
t.Fatal("event id should be filled")
}
if decoded.ParseStatus != ParseOK {
t.Fatalf("parse status = %q", decoded.ParseStatus)
}
}

View File

@@ -0,0 +1,187 @@
package eventbus
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type DurableConfig struct {
Directory string
}
type DurableSink struct {
delegate Sink
dir string
mu sync.Mutex
seq uint64
rawPending map[string]struct{}
}
type durableRecord struct {
Kind string `json:"kind"`
Envelope envelope.FrameEnvelope `json:"envelope"`
}
func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
if delegate == nil {
panic("durable delegate sink must not be nil")
}
return &DurableSink{
delegate: delegate,
dir: strings.TrimSpace(cfg.Directory),
rawPending: map[string]struct{}{},
}
}
func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
if err := s.delegate.PublishRaw(ctx, env); err == nil {
return nil
}
if err := s.spool("raw", env); err != nil {
return err
}
s.markRawPending(env)
return nil
}
func (s *DurableSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
if s.isRawPending(env) {
return s.spool("unified", env)
}
if err := s.delegate.PublishUnified(ctx, env); err == nil {
return nil
}
return s.spool("unified", env)
}
func (s *DurableSink) ReplayOnce(ctx context.Context) error {
files, err := filepath.Glob(filepath.Join(s.dir, "*.json"))
if err != nil {
return err
}
sort.Strings(files)
for _, file := range files {
record, err := readDurableRecord(file)
if err != nil {
return err
}
if err := s.publishRecord(ctx, record); err != nil {
return err
}
if err := os.Remove(file); err != nil {
return err
}
if record.Kind == "raw" {
s.clearRawPending(record.Envelope)
}
}
return nil
}
func (s *DurableSink) ReplayLoop(ctx context.Context, interval time.Duration, onError func(error)) {
if interval <= 0 {
interval = time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := s.ReplayOnce(ctx); err != nil && onError != nil {
onError(err)
}
}
}
}
func (s *DurableSink) Close() error {
return s.delegate.Close()
}
func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) error {
switch record.Kind {
case "raw":
return s.delegate.PublishRaw(ctx, record.Envelope)
case "unified":
return s.delegate.PublishUnified(ctx, record.Envelope)
default:
return fmt.Errorf("unknown durable record kind %q", record.Kind)
}
}
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
if s.dir == "" {
return fmt.Errorf("durable spool directory is empty")
}
if env.EventID == "" {
env.EventID = env.StableEventID()
}
if env.ParseStatus == "" {
env.ParseStatus = envelope.ParseOK
}
if err := os.MkdirAll(s.dir, 0o750); err != nil {
return err
}
payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env})
if err != nil {
return err
}
name := s.nextFileName(env, kind)
path := filepath.Join(s.dir, name)
tmp := path + ".tmp"
if err := os.WriteFile(tmp, payload, 0o640); err != nil {
return err
}
return os.Rename(tmp, path)
}
func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string {
s.mu.Lock()
defer s.mu.Unlock()
s.seq++
eventID := env.StableEventID()
if len(eventID) > 12 {
eventID = eventID[:12]
}
return fmt.Sprintf("%020d-%06d-%s-%s.json", time.Now().UnixNano(), s.seq, kind, eventID)
}
func (s *DurableSink) markRawPending(env envelope.FrameEnvelope) {
s.mu.Lock()
defer s.mu.Unlock()
s.rawPending[env.StableEventID()] = struct{}{}
}
func (s *DurableSink) clearRawPending(env envelope.FrameEnvelope) {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.rawPending, env.StableEventID())
}
func (s *DurableSink) isRawPending(env envelope.FrameEnvelope) bool {
s.mu.Lock()
defer s.mu.Unlock()
_, ok := s.rawPending[env.StableEventID()]
return ok
}
func readDurableRecord(path string) (durableRecord, error) {
var record durableRecord
payload, err := os.ReadFile(path)
if err != nil {
return record, err
}
return record, json.Unmarshal(payload, &record)
}

View File

@@ -0,0 +1,153 @@
package eventbus
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) {
dir := t.TempDir()
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
env := durableTestEnvelope()
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("PublishUnified() error = %v", err)
}
if delegate.rawCalls != 1 {
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
}
if delegate.unifiedCalls != 0 {
t.Fatalf("unified should not be delegated while raw is spooled, calls = %d", delegate.unifiedCalls)
}
files := spoolFiles(t, dir)
if len(files) != 2 {
t.Fatalf("spool files = %d, want 2: %#v", len(files), files)
}
assertSpoolKind(t, files[0], "raw")
assertSpoolKind(t, files[1], "unified")
}
func TestDurableSinkReplayPublishesInFileOrderAndDeletesFiles(t *testing.T) {
dir := t.TempDir()
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
env := durableTestEnvelope()
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("PublishUnified() error = %v", err)
}
delegate.rawErrors = nil
if err := sink.ReplayOnce(context.Background()); err != nil {
t.Fatalf("ReplayOnce() error = %v", err)
}
got := delegate.calls
want := []string{"raw", "raw", "unified"}
if len(got) != len(want) {
t.Fatalf("calls = %#v, want %#v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("calls[%d] = %q, want %q; all=%#v", i, got[i], want[i], got)
}
}
if files := spoolFiles(t, dir); len(files) != 0 {
t.Fatalf("spool files after replay = %#v, want none", files)
}
}
func durableTestEnvelope() envelope.FrameEnvelope {
return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
Phone: "013079963379",
VIN: "LKLG7C4E3NA774736",
Sequence: 183,
EventTimeMS: 1782903940000,
ReceivedAtMS: 1782917751903,
RawHex: "0200002201307996337900B7",
}
}
func spoolFiles(t *testing.T, dir string) []string {
t.Helper()
matches, err := filepath.Glob(filepath.Join(dir, "*.json"))
if err != nil {
t.Fatalf("glob spool files: %v", err)
}
return matches
}
func assertSpoolKind(t *testing.T, path string, kind string) {
t.Helper()
payload, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read spool file: %v", err)
}
if !containsString(string(payload), `"kind":"`+kind+`"`) {
t.Fatalf("spool file %s payload = %s, want kind %q", path, payload, kind)
}
}
func containsString(value string, needle string) bool {
return len(needle) == 0 || (len(value) >= len(needle) && indexString(value, needle) >= 0)
}
func indexString(value string, needle string) int {
for i := 0; i+len(needle) <= len(value); i++ {
if value[i:i+len(needle)] == needle {
return i
}
}
return -1
}
var errSpoolTest = errors.New("delegate unavailable")
type scriptedSink struct {
rawErrors []error
unifiedErrors []error
rawCalls int
unifiedCalls int
calls []string
}
func (s *scriptedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
s.rawCalls++
s.calls = append(s.calls, "raw")
if len(s.rawErrors) > 0 {
err := s.rawErrors[0]
s.rawErrors = s.rawErrors[1:]
return err
}
return nil
}
func (s *scriptedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
s.unifiedCalls++
s.calls = append(s.calls, "unified")
if len(s.unifiedErrors) > 0 {
err := s.unifiedErrors[0]
s.unifiedErrors = s.unifiedErrors[1:]
return err
}
return nil
}
func (s *scriptedSink) Close() error {
return nil
}

View File

@@ -0,0 +1,94 @@
package eventbus
import (
"context"
"errors"
"fmt"
"github.com/segmentio/kafka-go"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Sink interface {
PublishRaw(context.Context, envelope.FrameEnvelope) error
PublishUnified(context.Context, envelope.FrameEnvelope) error
Close() error
}
type KafkaSink struct {
writer kafkaWriter
rawTopics map[envelope.Protocol]string
unifiedTopic string
}
type kafkaWriter interface {
WriteMessages(context.Context, ...kafka.Message) error
Close() error
}
type KafkaConfig struct {
Brokers []string
RawTopics map[envelope.Protocol]string
UnifiedTopic string
}
func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) {
if len(cfg.Brokers) == 0 {
return nil, errors.New("kafka brokers are required")
}
return newKafkaSinkWithWriter(&kafka.Writer{
Addr: kafka.TCP(cfg.Brokers...),
Balancer: &kafka.Hash{},
AllowAutoTopicCreation: false,
}, cfg), nil
}
func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
rawTopics := map[envelope.Protocol]string{
envelope.ProtocolGB32960: "vehicle.raw.gb32960.v1",
envelope.ProtocolJT808: "vehicle.raw.jt808.v1",
envelope.ProtocolYutongMQTT: "vehicle.raw.yutong-mqtt.v1",
}
for protocol, topic := range cfg.RawTopics {
if topic != "" {
rawTopics[protocol] = topic
}
}
unifiedTopic := cfg.UnifiedTopic
if unifiedTopic == "" {
unifiedTopic = "vehicle.event.unified.v1"
}
return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic}
}
func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
topic, ok := s.rawTopics[env.Protocol]
if !ok || topic == "" {
return fmt.Errorf("raw topic not configured for protocol %s", env.Protocol)
}
return s.publish(ctx, topic, env)
}
func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
return s.publish(ctx, s.unifiedTopic, env)
}
func (s *KafkaSink) Close() error {
if s == nil || s.writer == nil {
return nil
}
return s.writer.Close()
}
func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
payload, err := env.MarshalJSONBytes()
if err != nil {
return err
}
return s.writer.WriteMessages(ctx, kafka.Message{
Topic: topic,
Key: env.KafkaKey(),
Value: payload,
})
}

View File

@@ -0,0 +1,77 @@
package eventbus
import (
"context"
"encoding/json"
"testing"
"github.com/segmentio/kafka-go"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestKafkaSinkRoutesRawTopics(t *testing.T) {
writer := &recordingWriter{}
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "013307795425", MessageID: "0x0200"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if len(writer.messages) != 1 {
t.Fatalf("messages = %d", len(writer.messages))
}
msg := writer.messages[0]
if msg.Topic != "vehicle.raw.jt808.v1" {
t.Fatalf("topic = %q", msg.Topic)
}
if string(msg.Key) != "JT808:013307795425" {
t.Fatalf("key = %q", string(msg.Key))
}
var decoded envelope.FrameEnvelope
if err := json.Unmarshal(msg.Value, &decoded); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if decoded.EventID == "" || decoded.ParseStatus != envelope.ParseOK {
t.Fatalf("unexpected payload defaults: %#v", decoded)
}
}
func TestKafkaSinkRoutesUnifiedTopic(t *testing.T) {
writer := &recordingWriter{}
sink := newKafkaSinkWithWriter(writer, KafkaConfig{UnifiedTopic: "custom.unified"})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000001", MessageID: "0x02"}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("PublishUnified() error = %v", err)
}
if writer.messages[0].Topic != "custom.unified" {
t.Fatalf("topic = %q", writer.messages[0].Topic)
}
if string(writer.messages[0].Key) != "LNBVIN00000000001" {
t.Fatalf("key = %q", string(writer.messages[0].Key))
}
}
func TestKafkaSinkRejectsUnknownProtocol(t *testing.T) {
writer := &recordingWriter{}
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")})
if err == nil {
t.Fatal("expected unknown protocol error")
}
}
type recordingWriter struct {
messages []kafka.Message
}
func (w *recordingWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
w.messages = append(w.messages, messages...)
return nil
}
func (w *recordingWriter) Close() error {
return nil
}

View File

@@ -0,0 +1,33 @@
package eventbus
import (
"context"
"log/slog"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type LogSink struct {
logger *slog.Logger
}
func NewLogSink(logger *slog.Logger) *LogSink {
if logger == nil {
logger = slog.Default()
}
return &LogSink{logger: logger}
}
func (s *LogSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
s.logger.Info("raw envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID, "status", env.ParseStatus)
return nil
}
func (s *LogSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
s.logger.Info("unified envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID)
return nil
}
func (s *LogSink) Close() error {
return nil
}

View File

@@ -0,0 +1,80 @@
package eventbus
import (
"context"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type RetryConfig struct {
Attempts int
Backoff time.Duration
AttemptTimeout time.Duration
}
type RetryingSink struct {
delegate Sink
cfg RetryConfig
}
func NewRetryingSink(delegate Sink, cfg RetryConfig) *RetryingSink {
if delegate == nil {
panic("retry delegate sink must not be nil")
}
if cfg.Attempts <= 0 {
cfg.Attempts = 1
}
return &RetryingSink{delegate: delegate, cfg: cfg}
}
func (s *RetryingSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
return s.withRetry(ctx, func(attemptCtx context.Context) error {
return s.delegate.PublishRaw(attemptCtx, env)
})
}
func (s *RetryingSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
return s.withRetry(ctx, func(attemptCtx context.Context) error {
return s.delegate.PublishUnified(attemptCtx, env)
})
}
func (s *RetryingSink) Close() error {
return s.delegate.Close()
}
func (s *RetryingSink) withRetry(ctx context.Context, publish func(context.Context) error) error {
var lastErr error
for attempt := 1; attempt <= s.cfg.Attempts; attempt++ {
attemptCtx := ctx
cancel := func() {}
if s.cfg.AttemptTimeout > 0 {
attemptCtx, cancel = context.WithTimeout(ctx, s.cfg.AttemptTimeout)
}
err := publish(attemptCtx)
cancel()
if err == nil {
return nil
}
lastErr = err
if ctx.Err() != nil {
return ctx.Err()
}
if attempt == s.cfg.Attempts {
break
}
if s.cfg.Backoff > 0 {
timer := time.NewTimer(s.cfg.Backoff)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return ctx.Err()
case <-timer.C:
}
}
}
return lastErr
}

View File

@@ -0,0 +1,80 @@
package eventbus
import (
"context"
"errors"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestRetryingSinkRetriesRawUntilSuccess(t *testing.T) {
delegate := &flakySink{rawFailures: 1}
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 3})
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
if err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if delegate.rawCalls != 2 {
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
}
}
func TestRetryingSinkReturnsLastErrorAfterAttempts(t *testing.T) {
delegate := &flakySink{rawFailures: 3}
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 2})
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
if !errors.Is(err, errFlakyPublish) {
t.Fatalf("PublishRaw() error = %v, want %v", err, errFlakyPublish)
}
if delegate.rawCalls != 2 {
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
}
}
func TestRetryingSinkStopsWhenContextCanceledDuringBackoff(t *testing.T) {
delegate := &flakySink{rawFailures: 10}
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 5, Backoff: time.Minute})
ctx, cancel := context.WithCancel(context.Background())
cancel()
err := sink.PublishRaw(ctx, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
if !errors.Is(err, context.Canceled) {
t.Fatalf("PublishRaw() error = %v, want context canceled", err)
}
if delegate.rawCalls != 1 {
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
}
}
var errFlakyPublish = errors.New("publish failed")
type flakySink struct {
rawFailures int
unifiedFailures int
rawCalls int
unifiedCalls int
}
func (s *flakySink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
s.rawCalls++
if s.rawCalls <= s.rawFailures {
return errFlakyPublish
}
return nil
}
func (s *flakySink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
s.unifiedCalls++
if s.unifiedCalls <= s.unifiedFailures {
return errFlakyPublish
}
return nil
}
func (s *flakySink) Close() error {
return nil
}

View File

@@ -0,0 +1,218 @@
package gateway
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"errors"
"fmt"
"log/slog"
"os"
"strings"
"time"
mqtt "github.com/eclipse/paho.mqtt.golang"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
)
type MQTTClientConfig struct {
EndpointName string
Broker string
ClientID string
Username string
Password string
Topics []string
QoS byte
CleanSession bool
KeepAlive time.Duration
ConnectTimeout time.Duration
TLSCACertPath string
TLSClientCertPath string
TLSClientKeyPath string
TLSHostnameVerification bool
Sink eventbus.Sink
Resolver identity.Resolver
Logger *slog.Logger
}
type MQTTClient struct {
cfg MQTTClientConfig
client mqtt.Client
}
func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
if strings.TrimSpace(cfg.Broker) == "" {
return nil, errors.New("mqtt broker is required")
}
if strings.TrimSpace(cfg.ClientID) == "" {
return nil, errors.New("mqtt client id is required")
}
if len(cfg.Topics) == 0 {
return nil, errors.New("mqtt topics are required")
}
if cfg.Sink == nil {
return nil, errors.New("sink is required")
}
if cfg.Logger == nil {
cfg.Logger = slog.Default()
}
if cfg.Resolver == nil {
cfg.Resolver = identity.NoopResolver{}
}
if cfg.EndpointName == "" {
cfg.EndpointName = "yutong"
}
return &MQTTClient{cfg: cfg}, nil
}
func (c *MQTTClient) Start(ctx context.Context) error {
opts, err := c.buildOptions(ctx)
if err != nil {
return err
}
c.client = mqtt.NewClient(opts)
token := c.client.Connect()
if token.Wait() && token.Error() != nil {
return token.Error()
}
go func() {
<-ctx.Done()
if c.client != nil && c.client.IsConnected() {
c.client.Disconnect(250)
}
}()
return nil
}
func (c *MQTTClient) buildOptions(ctx context.Context) (*mqtt.ClientOptions, error) {
keepAlive := c.cfg.KeepAlive
if keepAlive <= 0 {
keepAlive = 20 * time.Second
}
connectTimeout := c.cfg.ConnectTimeout
if connectTimeout <= 0 {
connectTimeout = 10 * time.Second
}
opts := mqtt.NewClientOptions().
AddBroker(c.cfg.Broker).
SetClientID(c.cfg.ClientID).
SetUsername(c.cfg.Username).
SetPassword(c.cfg.Password).
SetCleanSession(c.cfg.CleanSession).
SetAutoReconnect(true).
SetConnectRetry(true).
SetConnectRetryInterval(5 * time.Second).
SetResumeSubs(true).
SetOrderMatters(false).
SetKeepAlive(keepAlive).
SetConnectTimeout(connectTimeout)
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
c.handleMessage(ctx, message.Topic(), message.Payload())
})
opts.OnConnect = func(client mqtt.Client) {
for _, topic := range c.cfg.Topics {
token := client.Subscribe(topic, c.cfg.QoS, nil)
if token.Wait() && token.Error() != nil {
c.cfg.Logger.Error("mqtt subscribe failed", "broker", c.cfg.Broker, "topic", topic, "error", token.Error())
continue
}
c.cfg.Logger.Info("mqtt subscribed", "broker", c.cfg.Broker, "topic", topic, "qos", c.cfg.QoS)
}
}
opts.OnConnectionLost = func(_ mqtt.Client, err error) {
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
}
tlsConfig, err := c.buildTLSConfig()
if err != nil {
return nil, err
}
if tlsConfig != nil {
opts.SetTLSConfig(tlsConfig)
}
return opts, nil
}
func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
caPath := strings.TrimSpace(c.cfg.TLSCACertPath)
certPath := strings.TrimSpace(c.cfg.TLSClientCertPath)
keyPath := strings.TrimSpace(c.cfg.TLSClientKeyPath)
if caPath == "" && certPath == "" && keyPath == "" {
return nil, nil
}
config := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: !c.cfg.TLSHostnameVerification,
}
if caPath != "" {
caPEM, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("read mqtt ca certificate: %w", err)
}
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(caPEM) {
return nil, fmt.Errorf("parse mqtt ca certificate %s", caPath)
}
config.RootCAs = roots
}
if certPath != "" || keyPath != "" {
if certPath == "" || keyPath == "" {
return nil, errors.New("mqtt client certificate and key must be configured together")
}
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, fmt.Errorf("load mqtt client certificate: %w", err)
}
config.Certificates = []tls.Certificate{cert}
}
return config, nil
}
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
receivedAtMS := time.Now().UnixMilli()
env, err := yutongmqtt.ParseMessage(c.cfg.EndpointName, topic, payload, receivedAtMS)
if err != nil {
env = envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
MessageID: "MQTT",
VehicleKeyHint: "unknown",
SourceEndpoint: "mqtt://" + c.cfg.EndpointName + topic,
EventTimeMS: receivedAtMS,
ReceivedAtMS: receivedAtMS,
RawText: string(payload),
RawHex: strings.ToUpper(hex.EncodeToString(payload)),
ParseStatus: envelope.ParseBadFrame,
ParseError: err.Error(),
}
env.EventID = env.StableEventID()
} else {
resolved, resolveErr := c.cfg.Resolver.Resolve(ctx, env)
if resolveErr != nil {
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
env.ParseStatus = envelope.ParsePartial
} else {
env = resolved
}
}
if err := c.cfg.Sink.PublishRaw(ctx, env); err != nil {
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
return
}
if env.ParseStatus == envelope.ParseBadFrame {
return
}
if err := c.cfg.Sink.PublishUnified(ctx, env); err != nil {
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
}
}

View File

@@ -0,0 +1,183 @@
package gateway
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"log/slog"
"math/big"
"os"
"path/filepath"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
sink := &recordingSink{}
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
"device":"LTEST000000000001",
"time":"20260413100000",
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
}`))
if len(sink.raw) != 1 || len(sink.unified) != 1 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
t.Fatalf("unexpected raw envelope: %#v", sink.raw[0])
}
}
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
sink := &recordingSink{}
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/bad", []byte("{bad-json"))
if len(sink.raw) != 1 || len(sink.unified) != 0 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
}
}
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
dir := t.TempDir()
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "ssl://mqtt.example.test:8883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
QoS: 1,
Sink: &recordingSink{},
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
TLSCACertPath: caPath,
TLSClientCertPath: certPath,
TLSClientKeyPath: keyPath,
TLSHostnameVerification: false,
CleanSession: true,
KeepAlive: 20 * time.Second,
ConnectTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
opts, err := client.buildOptions(context.Background())
if err != nil {
t.Fatalf("buildOptions() error = %v", err)
}
if opts.TLSConfig == nil {
t.Fatal("TLSConfig is nil")
}
if opts.TLSConfig.RootCAs == nil {
t.Fatal("RootCAs is nil")
}
if len(opts.TLSConfig.Certificates) != 1 {
t.Fatalf("client certificates = %d, want 1", len(opts.TLSConfig.Certificates))
}
if !opts.TLSConfig.InsecureSkipVerify {
t.Fatal("InsecureSkipVerify should be true when hostname verification is disabled")
}
if !opts.CleanSession {
t.Fatal("CleanSession should be true")
}
if opts.Order {
t.Fatal("OrderMatters should be false so MQTT network handling is not blocked by Kafka/DB work")
}
if !opts.ResumeSubs {
t.Fatal("ResumeSubs should be true to avoid subscribe failures during reconnect churn")
}
if got := opts.KeepAlive; got != 20 {
t.Fatalf("KeepAlive = %d, want 20", got)
}
if got := opts.ConnectTimeout; got != 10*time.Second {
t.Fatalf("ConnectTimeout = %v, want 10s", got)
}
}
func writeTestTLSMaterial(t *testing.T, dir string) (string, string, string) {
t.Helper()
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate ca key: %v", err)
}
caTemplate := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "test-ca"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
BasicConstraintsValid: true,
IsCA: true,
}
caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
if err != nil {
t.Fatalf("create ca cert: %v", err)
}
clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate client key: %v", err)
}
clientTemplate := &x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{CommonName: "test-client"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey)
if err != nil {
t.Fatalf("create client cert: %v", err)
}
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "client.pem")
keyPath := filepath.Join(dir, "client-key.pem")
writePEM(t, caPath, "CERTIFICATE", caDER)
writePEM(t, certPath, "CERTIFICATE", clientDER)
keyDER := x509.MarshalPKCS1PrivateKey(clientKey)
writePEM(t, keyPath, "RSA PRIVATE KEY", keyDER)
return caPath, certPath, keyPath
}
func writePEM(t *testing.T, path, typ string, der []byte) {
t.Helper()
file, err := os.Create(path)
if err != nil {
t.Fatalf("create %s: %v", path, err)
}
defer file.Close()
if err := pem.Encode(file, &pem.Block{Type: typ, Bytes: der}); err != nil {
t.Fatalf("write pem %s: %v", path, err)
}
}

View File

@@ -0,0 +1,238 @@
package gateway
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"net"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
)
type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
type TCPProtocol struct {
Protocol envelope.Protocol
Addr string
Extract FrameExtractor
Parse FrameParser
Respond FrameResponder
}
type TCPServer struct {
protocol TCPProtocol
sink eventbus.Sink
resolver identity.Resolver
logger *slog.Logger
readBufferSize int
idleTimeout time.Duration
maxConnections int
}
type TCPServerConfig struct {
Protocol TCPProtocol
Sink eventbus.Sink
Resolver identity.Resolver
Logger *slog.Logger
ReadBufferSize int
IdleTimeout time.Duration
MaxConnections int
}
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
if cfg.Protocol.Protocol == "" {
return nil, errors.New("protocol is required")
}
if strings.TrimSpace(cfg.Protocol.Addr) == "" {
return nil, errors.New("listen addr is required")
}
if cfg.Protocol.Extract == nil {
return nil, errors.New("frame extractor is required")
}
if cfg.Protocol.Parse == nil {
return nil, errors.New("frame parser is required")
}
if cfg.Sink == nil {
return nil, errors.New("sink is required")
}
if cfg.Logger == nil {
cfg.Logger = slog.Default()
}
if cfg.Resolver == nil {
cfg.Resolver = identity.NoopResolver{}
}
if cfg.ReadBufferSize <= 0 {
cfg.ReadBufferSize = 32 * 1024
}
if cfg.IdleTimeout <= 0 {
cfg.IdleTimeout = 2 * time.Minute
}
if cfg.MaxConnections <= 0 {
cfg.MaxConnections = 10_000
}
return &TCPServer{
protocol: cfg.Protocol,
sink: cfg.Sink,
resolver: cfg.Resolver,
logger: cfg.Logger,
readBufferSize: cfg.ReadBufferSize,
idleTimeout: cfg.IdleTimeout,
maxConnections: cfg.MaxConnections,
}, nil
}
func (s *TCPServer) ListenAndServe(ctx context.Context) error {
var lc net.ListenConfig
listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr)
if err != nil {
return err
}
defer listener.Close()
go func() {
<-ctx.Done()
_ = listener.Close()
}()
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
sem := make(chan struct{}, s.maxConnections)
var wg sync.WaitGroup
defer wg.Wait()
for {
conn, err := listener.Accept()
if err != nil {
if ctx.Err() != nil {
return nil
}
s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err)
continue
}
select {
case sem <- struct{}{}:
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sem }()
s.handleConnection(ctx, conn)
}()
default:
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
_ = conn.Close()
}
}
}
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
defer conn.Close()
source := conn.RemoteAddr().String()
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
log.Info("tcp connection opened")
defer log.Info("tcp connection closed")
readBuffer := make([]byte, s.readBufferSize)
var pending []byte
for {
_ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout))
n, err := conn.Read(readBuffer)
if n > 0 {
pending = append(pending, readBuffer[:n]...)
frames, remainder, extractErr := s.protocol.Extract(pending)
if extractErr != nil {
log.Warn("frame extraction failed", "error", extractErr)
return
}
pending = remainder
for _, frame := range frames {
s.handleFrame(ctx, conn, frame, source)
}
}
if err != nil {
if errors.Is(err, io.EOF) {
return
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
log.Warn("tcp connection idle timeout")
return
}
log.Warn("tcp read failed", "error", err)
return
}
if ctx.Err() != nil {
return
}
}
}
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string) {
receivedAtMS := time.Now().UnixMilli()
env, err := s.protocol.Parse(raw, receivedAtMS, source)
if err != nil {
env = envelope.FrameEnvelope{
Protocol: s.protocol.Protocol,
SourceEndpoint: source,
ReceivedAtMS: receivedAtMS,
EventTimeMS: receivedAtMS,
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
ParseStatus: envelope.ParseBadFrame,
ParseError: err.Error(),
}
env.EventID = env.StableEventID()
} else {
resolved, resolveErr := s.resolver.Resolve(ctx, env)
if resolveErr != nil {
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
env.ParseStatus = envelope.ParsePartial
} else {
env = resolved
}
}
if err := s.sink.PublishRaw(ctx, env); err != nil {
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if env.ParseStatus == envelope.ParseBadFrame {
return
}
if err := s.sink.PublishUnified(ctx, env); err != nil {
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if s.protocol.Respond == nil {
return
}
response, ok, err := s.protocol.Respond(raw, env)
if err != nil {
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if !ok || len(response) == 0 {
return
}
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := conn.Write(response); err != nil {
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
}
}
func (p TCPProtocol) String() string {
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
}

View File

@@ -0,0 +1,177 @@
package gateway
import (
"context"
"encoding/hex"
"io"
"log/slog"
"net"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
)
func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
if err != nil {
t.Fatal(err)
}
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolJT808,
Addr: ":0",
Extract: jt808.ExtractFrames,
Parse: jt808.ParseFrame,
}, sink)
client, done := runPipe(t, server)
if _, err := client.Write(frame); err != nil {
t.Fatalf("client.Write() error = %v", err)
}
_ = client.Close()
<-done
if len(sink.raw) != 1 || len(sink.unified) != 1 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].Phone != "013307795425" {
t.Fatalf("phone = %q", sink.raw[0].Phone)
}
if sink.raw[0].Fields[envelope.FieldTotalMileageKM] != 10241.2 {
t.Fatalf("total mileage = %#v", sink.raw[0].Fields[envelope.FieldTotalMileageKM])
}
}
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
good[len(good)-1] ^= 0xff
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolGB32960,
Addr: ":0",
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
}, sink)
client, done := runPipe(t, server)
if _, err := client.Write(good); err != nil {
t.Fatalf("client.Write() error = %v", err)
}
_ = client.Close()
<-done
if len(sink.raw) != 1 || len(sink.unified) != 0 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
}
if sink.raw[0].ParseError == "" {
t.Fatal("parse error should be recorded")
}
}
func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
frame := buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil)
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolGB32960,
Addr: ":0",
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
return []byte("ACK"), true, nil
},
}, sink)
client, done := runPipe(t, server)
if _, err := client.Write(frame); err != nil {
t.Fatalf("client.Write() error = %v", err)
}
buf := make([]byte, 3)
if _, err := io.ReadFull(client, buf); err != nil {
t.Fatalf("read response error = %v", err)
}
if string(buf) != "ACK" {
t.Fatalf("response = %q", string(buf))
}
_ = client.Close()
<-done
}
func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer {
t.Helper()
server, err := NewTCPServer(TCPServerConfig{
Protocol: protocol,
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
ReadBufferSize: 1024,
IdleTimeout: time.Second,
MaxConnections: 1,
})
if err != nil {
t.Fatalf("NewTCPServer() error = %v", err)
}
return server
}
func runPipe(t *testing.T, server *TCPServer) (net.Conn, <-chan struct{}) {
t.Helper()
client, srv := net.Pipe()
done := make(chan struct{})
go func() {
defer close(done)
server.handleConnection(context.Background(), srv)
}()
return client, done
}
type recordingSink struct {
raw []envelope.FrameEnvelope
unified []envelope.FrameEnvelope
}
func (s *recordingSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
s.raw = append(s.raw, env)
return nil
}
func (s *recordingSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
s.unified = append(s.unified, env)
return nil
}
func (s *recordingSink) Close() error {
return nil
}
type testWriter struct {
t *testing.T
}
func (w testWriter) Write(p []byte) (int, error) {
w.t.Log(string(p))
return len(p), nil
}
func buildGBFrame(command byte, response byte, vin string, body []byte) []byte {
frame := []byte{'#', '#', command, response}
vinBytes := []byte(vin)
if len(vinBytes) < 17 {
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
}
frame = append(frame, vinBytes[:17]...)
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
frame = append(frame, body...)
var bcc byte
for _, value := range frame[2:] {
bcc ^= value
}
return append(frame, bcc)
}

View File

@@ -0,0 +1,908 @@
package history
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
)
type Queryer interface {
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
}
type RawFrameQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
MessageID string
OrderBy string
IncludeTotal bool
DateFrom string
DateTo string
Limit int
Offset int
}
type RawFrameRow struct {
TS string `json:"ts"`
FrameID string `json:"frame_id"`
EventID string `json:"event_id"`
MessageID int64 `json:"message_id"`
MessageIDHex string `json:"message_id_hex"`
EventTime string `json:"event_time"`
ReceivedAt string `json:"received_at"`
RawSizeBytes int64 `json:"raw_size_bytes"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
ParsedJSON string `json:"parsed_json,omitempty"`
FieldsJSON string `json:"fields_json,omitempty"`
ParseStatus string `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
SourceEndpoint string `json:"source_endpoint"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type LocationQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
DateFrom string
DateTo string
Limit int
Offset int
}
type LocationRow struct {
TS string `json:"ts"`
EventID string `json:"event_id"`
FrameID string `json:"frame_id"`
ReceivedAt string `json:"received_at"`
Longitude float64 `json:"longitude"`
Latitude float64 `json:"latitude"`
AltitudeM *float64 `json:"altitude_m,omitempty"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
DirectionDeg *int64 `json:"direction_deg,omitempty"`
AlarmFlag *int64 `json:"alarm_flag,omitempty"`
StatusFlag *int64 `json:"status_flag,omitempty"`
TotalMileageKM *float64 `json:"total_mileage_km,omitempty"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type MileagePointQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
DateFrom string
DateTo string
Limit int
Offset int
}
type MileagePointRow struct {
TS string `json:"ts"`
EventID string `json:"event_id"`
FrameID string `json:"frame_id"`
ReceivedAt string `json:"received_at"`
TotalMileageKM float64 `json:"total_mileage_km"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type RawFrameRepository struct {
db Queryer
database string
}
func NewRawFrameRepository(db Queryer, database string) *RawFrameRepository {
if db == nil {
panic("raw frame query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &RawFrameRepository{db: db, database: database}
}
type LocationRepository struct {
db Queryer
database string
}
type MileagePointRepository struct {
db Queryer
database string
}
func NewLocationRepository(db Queryer, database string) *LocationRepository {
if db == nil {
panic("location query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &LocationRepository{db: db, database: database}
}
func NewMileagePointRepository(db Queryer, database string) *MileagePointRepository {
if db == nil {
panic("mileage point query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &MileagePointRepository{db: db, database: database}
}
func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) {
query = normalizeRawFrameQuery(query)
sqlText, args := buildRawFrameSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]RawFrameRow, 0)
for rows.Next() {
var row RawFrameRow
var ts scanDateTime
var eventTime scanDateTime
var receivedAt scanDateTime
if err := rows.Scan(
&ts,
&row.FrameID,
&row.EventID,
&row.MessageID,
&eventTime,
&receivedAt,
&row.RawSizeBytes,
&row.RawHex,
&row.RawText,
&row.ParsedJSON,
&row.FieldsJSON,
&row.ParseStatus,
&row.ParseError,
&row.SourceEndpoint,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.EventTime = eventTime.String
row.ReceivedAt = receivedAt.String
row.MessageIDHex = fmt.Sprintf("0x%04X", row.MessageID)
out = append(out, row)
}
return out, rows.Err()
}
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
query = normalizeRawFrameQuery(query)
sqlText, args := buildRawFrameCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]LocationRow, error) {
query = normalizeLocationQuery(query)
sqlText, args := buildLocationSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]LocationRow, 0)
for rows.Next() {
var row LocationRow
var ts scanDateTime
var receivedAt scanDateTime
var altitude sql.NullFloat64
var speed sql.NullFloat64
var direction sql.NullInt64
var alarm sql.NullInt64
var status sql.NullInt64
var mileage sql.NullFloat64
if err := rows.Scan(
&ts,
&row.EventID,
&row.FrameID,
&receivedAt,
&row.Longitude,
&row.Latitude,
&altitude,
&speed,
&direction,
&alarm,
&status,
&mileage,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.ReceivedAt = receivedAt.String
row.AltitudeM = nullableFloat(altitude)
row.SpeedKMH = nullableFloat(speed)
row.DirectionDeg = nullableInt(direction)
row.AlarmFlag = nullableInt(alarm)
row.StatusFlag = nullableInt(status)
row.TotalMileageKM = nullableFloat(mileage)
out = append(out, row)
}
return out, rows.Err()
}
func (r *LocationRepository) Count(ctx context.Context, query LocationQuery) (int64, error) {
query = normalizeLocationQuery(query)
sqlText, args := buildLocationCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func (r *MileagePointRepository) Query(ctx context.Context, query MileagePointQuery) ([]MileagePointRow, error) {
query = normalizeMileagePointQuery(query)
sqlText, args := buildMileagePointSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]MileagePointRow, 0)
for rows.Next() {
var row MileagePointRow
var ts scanDateTime
var receivedAt scanDateTime
var speed sql.NullFloat64
var longitude sql.NullFloat64
var latitude sql.NullFloat64
if err := rows.Scan(
&ts,
&row.EventID,
&row.FrameID,
&receivedAt,
&row.TotalMileageKM,
&speed,
&longitude,
&latitude,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.ReceivedAt = receivedAt.String
row.SpeedKMH = nullableFloat(speed)
row.Longitude = nullableFloat(longitude)
row.Latitude = nullableFloat(latitude)
out = append(out, row)
}
return out, rows.Err()
}
func (r *MileagePointRepository) Count(ctx context.Context, query MileagePointQuery) (int64, error) {
query = normalizeMileagePointQuery(query)
sqlText, args := buildMileagePointCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func countRows(ctx context.Context, db Queryer, sqlText string, args ...any) (int64, error) {
rows, err := db.QueryContext(ctx, sqlText, args...)
if err != nil {
return 0, err
}
defer rows.Close()
var total int64
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, rows.Err()
}
func (r *RawFrameRepository) tableName() string {
if r.database == "" {
return "raw_frames"
}
return r.database + ".raw_frames"
}
func (r *LocationRepository) tableName() string {
if r.database == "" {
return "vehicle_locations"
}
return r.database + ".vehicle_locations"
}
func (r *MileagePointRepository) tableName() string {
if r.database == "" {
return "vehicle_mileage_points"
}
return r.database + ".vehicle_mileage_points"
}
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.MessageID = strings.TrimSpace(query.MessageID)
query.OrderBy = normalizeRawFrameOrderBy(query.OrderBy)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func normalizeLocationQuery(query LocationQuery) LocationQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func normalizeMileagePointQuery(query MileagePointQuery) MileagePointQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
where := rawFrameWhere(query)
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY " + rawFrameOrderColumn(query.OrderBy) + " DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := rawFrameWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
where := locationWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildLocationCountSQL(table string, query LocationQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := locationWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func buildMileagePointSQL(table string, query MileagePointQuery) (string, []any) {
where := mileagePointWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildMileagePointCountSQL(table string, query MileagePointQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := mileagePointWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func rawFrameWhere(query RawFrameQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.MessageID != "" {
if parsed, ok := parseMessageID(query.MessageID); ok {
add("message_id = " + strconv.FormatInt(parsed, 10))
}
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
func normalizeRawFrameOrderBy(value string) string {
value = strings.ToLower(strings.TrimSpace(value))
switch value {
case "receivedat", "received_at":
return "receivedAt"
default:
return "ts"
}
}
func rawFrameOrderColumn(value string) string {
if normalizeRawFrameOrderBy(value) == "receivedAt" {
return "received_at"
}
return "ts"
}
func locationWhere(query LocationQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
func mileagePointWhere(query MileagePointQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
type RawFrameHandler struct {
repository *RawFrameRepository
}
type LocationHandler struct {
repository *LocationRepository
}
type MileagePointHandler struct {
repository *MileagePointRepository
}
func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler {
if repository == nil {
panic("raw frame repository must not be nil")
}
return &RawFrameHandler{repository: repository}
}
func NewLocationHandler(repository *LocationRepository) *LocationHandler {
if repository == nil {
panic("location repository must not be nil")
}
return &LocationHandler{repository: repository}
}
func NewMileagePointHandler(repository *MileagePointRepository) *MileagePointHandler {
if repository == nil {
panic("mileage point repository must not be nil")
}
return &MileagePointHandler{repository: repository}
}
func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/raw-frames" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseRawFrameQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
var total int64
if query.IncludeTotal {
total, err = h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
if !query.IncludeTotal {
total = int64(len(rows))
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/locations" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseLocationQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func (h *MileagePointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/mileage-points" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseMileagePointQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return RawFrameQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return RawFrameQuery{}, err
}
query := RawFrameQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
MessageID: values.Get("messageId"),
OrderBy: values.Get("orderBy"),
IncludeTotal: values.Get("includeTotal") != "false",
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
if strings.TrimSpace(query.MessageID) != "" {
if _, ok := parseMessageID(query.MessageID); !ok {
return RawFrameQuery{}, errors.New("messageId must be decimal or hex like 0x0200")
}
}
return normalizeRawFrameQuery(query), nil
}
func parseMileagePointQuery(r *http.Request) (MileagePointQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return MileagePointQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return MileagePointQuery{}, err
}
query := MileagePointQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return MileagePointQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
return normalizeMileagePointQuery(query), nil
}
func parseLocationQuery(r *http.Request) (LocationQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return LocationQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return LocationQuery{}, err
}
query := LocationQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
return normalizeLocationQuery(query), nil
}
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return fallback, nil
}
value, err := strconv.Atoi(raw)
if err != nil || value < min || value > max {
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
}
return value, nil
}
func parseMessageID(value string) (int64, bool) {
value = strings.TrimSpace(strings.ToLower(value))
if value == "" {
return 0, false
}
if strings.HasPrefix(value, "0x") {
parsed, err := strconv.ParseInt(strings.TrimPrefix(value, "0x"), 16, 32)
return parsed, err == nil
}
parsed, err := strconv.ParseInt(value, 10, 32)
return parsed, err == nil
}
func validDateTime(value string) bool {
value = normalizeDateTimeLiteral(value)
if value == "" {
return true
}
for _, layout := range []string{"2006-01-02", "2006-01-02 15:04:05", time.RFC3339} {
if _, err := time.Parse(layout, value); err == nil {
return true
}
}
return false
}
func normalizeDateTimeLiteral(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} {
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
}
}
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
}
return value
}
func safeIdentifier(value string) bool {
if value == "" {
return false
}
for _, r := range value {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
continue
}
return false
}
return true
}
type scanDateTime struct {
String string
}
func (s *scanDateTime) Scan(value any) error {
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
return nil
}
func formatSQLTime(value any, layout string) string {
switch typed := value.(type) {
case nil:
return ""
case time.Time:
return typed.Format(layout)
case []byte:
return strings.TrimSpace(string(typed))
case string:
return strings.TrimSpace(typed)
default:
return strings.TrimSpace(fmt.Sprint(typed))
}
}
func nullableFloat(value sql.NullFloat64) *float64 {
if !value.Valid {
return nil
}
return &value.Float64
}
func nullableInt(value sql.NullInt64) *int64 {
if !value.Valid {
return nil
}
return &value.Int64
}
func writeHistoryError(w http.ResponseWriter, status int, message string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
}

View File

@@ -0,0 +1,423 @@
package history
import (
"context"
"database/sql"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
)
func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
"go_frame", "event-1", 0x0200,
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
time.Date(2026, 7, 1, 23, 25, 37, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
64, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "222.66.200.68:29646", "JT808", "013079963379", "LKLG7C4E3NA774736", "013079963379", "9963379",
))
repository := NewRawFrameRepository(db, "lingniu_vehicle_ts")
rows, err := repository.Query(context.Background(), RawFrameQuery{
Protocol: "JT808",
VIN: "LKLG7C4E3NA774736",
DateFrom: "2026-07-01 00:00:00",
DateTo: "2026-07-01 23:59:59",
Limit: 20,
})
if err != nil {
t.Fatalf("Query() error = %v", err)
}
if len(rows) != 1 {
t.Fatalf("row count = %d", len(rows))
}
if rows[0].MessageID != 512 || rows[0].MessageIDHex != "0x0200" || rows[0].ParsedJSON == "" {
t.Fatalf("unexpected row: %#v", rows[0])
}
if rows[0].TS != "2026-07-01 23:25:36" {
t.Fatalf("timestamp = %q", rows[0].TS)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(38))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-01 22:28:25", "go_frame", "event-2", 2, "2026-07-01 22:28:25", "2026-07-01 22:28:25",
128, "2323", "", `{"command":"REALTIME"}`, `{"total_mileage_km":53490.9}`,
"OK", "", "8.134.95.166:53702", "GB32960", "LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"message_id_hex":"0x0002"`, `"total":38`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(11))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "go_frame", "event-3", 0x0200, "2026-07-02 00:18:22", "2026-07-02 00:22:43",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"total_mileage_km":8792.8}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"phone":"013307811350"`, `"total":11`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=NOT_EXISTS&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
t.Fatalf("expected empty items array, got: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestLocationHandlerReturnsLocationsByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(17))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
"JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
8792.8, 8.0, 121.07764, 30.585928,
"JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewMileagePointHandler(NewMileagePointRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/mileage-points?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"total_mileage_km":8792.8`, `"speed_kmh":8`, `"total":19`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
}
func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) {
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?dateFrom=2026-07-01T00:00:00&dateTo=2026-07-02T00:00:00", nil)
query, err := parseRawFrameQuery(request)
if err != nil {
t.Fatalf("parseRawFrameQuery() error = %v", err)
}
if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" {
t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo)
}
}
func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) {
for raw, want := range map[string]string{
"2026-07-01T00:00:00": "2026-06-30 16:00:00",
"2026-07-01 00:00:00": "2026-06-30 16:00:00",
"2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00",
"2026-06-30T16:00:00Z": "2026-06-30 16:00:00",
} {
if got := normalizeDateTimeLiteral(raw); got != want {
t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want)
}
}
}
func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
for raw, want := range map[string]int64{
"512": 512,
"0x0200": 512,
"0X0100": 256,
} {
got, ok := parseMessageID(raw)
if !ok || got != want {
t.Fatalf("parseMessageID(%q) = %d,%v want %d,true", raw, got, ok, want)
}
}
}
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
DateFrom: "2026-07-02 00:00:00",
DateTo: "2026-07-02 23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"ts >= '2026-07-01 16:00:00'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildLocationSQL("lingniu_vehicle_ts.vehicle_locations", LocationQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
DateFrom: "2026-07-02 00:00:00",
DateTo: "2026-07-02 23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"ts >= '2026-07-01 16:00:00'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
VIN: "VIN'1",
MessageID: "0x0200",
DateFrom: "2026-07-01T00:00:00",
DateTo: "2026-07-01T23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"vin = 'VIN''1'",
"message_id = 512",
"ts >= '2026-06-30 16:00:00'",
"ts <= '2026-07-01 15:59:59'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildRawFrameSQLCanOrderByReceivedAt(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
Protocol: "JT808",
OrderBy: "receivedAt",
Limit: 1,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
if !strings.Contains(sqlText, "ORDER BY received_at DESC LIMIT 1 OFFSET 0") {
t.Fatalf("sql should order by received_at: %s", sqlText)
}
}

View File

@@ -0,0 +1,71 @@
package history
const DefaultDatabase = "lingniu_vehicle_ts"
func SchemaStatements(database string) []string {
if database == "" {
database = DefaultDatabase
}
return []string{
"CREATE DATABASE IF NOT EXISTS " + database + " KEEP 7300 DURATION 10 BUFFER 256",
"USE " + database,
`CREATE STABLE IF NOT EXISTS raw_frames (
ts TIMESTAMP,
frame_id NCHAR(64),
event_id NCHAR(64),
message_id INT,
event_time TIMESTAMP,
received_at TIMESTAMP,
raw_size_bytes INT,
raw_hex BINARY(16374),
raw_text BINARY(16374),
parsed_json BINARY(16374),
fields_json BINARY(4096),
parse_status NCHAR(16),
parse_error BINARY(1024),
source_endpoint NCHAR(128)
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
)`,
`CREATE STABLE IF NOT EXISTS vehicle_locations (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
longitude DOUBLE,
latitude DOUBLE,
altitude_m DOUBLE,
speed_kmh DOUBLE,
direction_deg INT,
alarm_flag BIGINT,
status_flag BIGINT,
total_mileage_km DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
)`,
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
total_mileage_km DOUBLE,
speed_kmh DOUBLE,
longitude DOUBLE,
latitude DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
)`,
}
}

View File

@@ -0,0 +1,339 @@
package history
import (
"context"
"crypto/sha1"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Execer interface {
ExecContext(context.Context, string, ...any) (sql.Result, error)
}
type Writer struct {
exec Execer
cache tableCache
}
type tableCache struct {
mu sync.Mutex
seen map[string]struct{}
}
func NewWriter(exec Execer) *Writer {
if exec == nil {
panic("history execer must not be nil")
}
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
}
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
for _, statement := range SchemaStatements(database) {
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
return err
}
}
return nil
}
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
if err := w.AppendRawFrame(ctx, env); err != nil {
return err
}
if err := w.AppendLocation(ctx, env); err != nil {
return err
}
return w.AppendMileagePoint(ctx, env)
}
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
table := tableName("raw", env)
if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
return err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
VALUES (%s)`, table, joinLiterals(rawValues(env))))
return err
}
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
longitude, okLon := floatField(env, envelope.FieldLongitude)
latitude, okLat := floatField(env, envelope.FieldLatitude)
if !okLon || !okLat {
return nil
}
table := tableName("loc", env)
if err := w.ensureChild(ctx, table, "vehicle_locations", env); err != nil {
return err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh,
direction_deg, alarm_flag, status_flag, total_mileage_km)
VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
return err
}
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
if !ok {
return nil
}
table := tableName("mil", env)
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
return err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage))))
return err
}
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
key := stable + "." + table
w.cache.mu.Lock()
_, ok := w.cache.seen[key]
w.cache.mu.Unlock()
if ok {
return nil
}
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
table, stable,
quote(string(env.Protocol)),
quote(env.VehicleKey()),
quote(env.VIN),
quote(env.Phone),
quote(env.DeviceID))
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
return err
}
w.cache.mu.Lock()
w.cache.seen[key] = struct{}{}
w.cache.mu.Unlock()
return nil
}
func rawValues(env envelope.FrameEnvelope) []any {
received := millis(env.ReceivedAtMS)
eventTime := millis(env.EventTimeMS)
return []any{
received,
frameID(env),
env.StableEventID(),
messageIDInt(env.MessageID),
eventTime,
received,
rawSizeBytes(env),
env.RawHex,
env.RawText,
jsonString(env.Parsed),
jsonString(env.Fields),
string(env.ParseStatus),
env.ParseError,
env.SourceEndpoint,
}
}
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
received := millis(env.ReceivedAtMS)
return []any{
eventTimeOrReceived(env),
env.StableEventID(),
frameID(env),
received,
longitude,
latitude,
floatFieldOrNil(env, "altitude_m"),
floatFieldOrNil(env, envelope.FieldSpeedKMH),
intFieldOrNil(env, "direction_deg"),
intFieldOrNil(env, "alarm_flag"),
intFieldOrNil(env, "status_flag"),
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
}
}
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
received := millis(env.ReceivedAtMS)
return []any{
eventTimeOrReceived(env),
env.StableEventID(),
frameID(env),
received,
totalMileage,
floatFieldOrNil(env, envelope.FieldSpeedKMH),
floatFieldOrNil(env, envelope.FieldLongitude),
floatFieldOrNil(env, envelope.FieldLatitude),
}
}
func frameID(env envelope.FrameEnvelope) string {
return "go_" + env.StableEventID()
}
func tableName(prefix string, env envelope.FrameEnvelope) string {
return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
}
func hash16(value string) string {
sum := sha1.Sum([]byte(value))
return hex.EncodeToString(sum[:8])
}
func messageIDInt(value string) int64 {
value = strings.TrimSpace(strings.ToLower(value))
value = strings.TrimPrefix(value, "0x")
parsed, err := strconv.ParseInt(value, 16, 32)
if err == nil {
return parsed
}
parsed, _ = strconv.ParseInt(value, 10, 32)
return parsed
}
func rawSizeBytes(env envelope.FrameEnvelope) int {
rawHex := strings.TrimSpace(env.RawHex)
if len(rawHex)%2 != 0 {
return len(rawHex) / 2
}
if rawHex != "" {
return len(rawHex) / 2
}
return len([]byte(env.RawText))
}
func jsonString(value any) string {
data, err := json.Marshal(value)
if err != nil {
return "{}"
}
return string(data)
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
if env.Fields == nil {
return 0, false
}
value, ok := env.Fields[key]
if !ok || value == nil {
return 0, false
}
switch typed := value.(type) {
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int64:
return float64(typed), true
case uint16:
return float64(typed), true
case uint32:
return float64(typed), true
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return parsed, err == nil
default:
return 0, false
}
}
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
value, ok := floatField(env, key)
if !ok {
return nil
}
return value
}
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
if env.Fields == nil {
return nil
}
value, ok := env.Fields[key]
if !ok || value == nil {
return nil
}
switch typed := value.(type) {
case int:
return typed
case int64:
return typed
case uint16:
return int64(typed)
case uint32:
return int64(typed)
case float64:
return int64(typed)
case string:
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
if err == nil {
return parsed
}
}
return nil
}
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
if env.EventTimeMS > 0 {
return millis(env.EventTimeMS)
}
return millis(env.ReceivedAtMS)
}
func millis(value int64) time.Time {
if value <= 0 {
return time.UnixMilli(time.Now().UnixMilli()).UTC()
}
return time.UnixMilli(value).UTC()
}
func quote(value string) string {
return strings.ReplaceAll(value, "'", "''")
}
func joinLiterals(values []any) string {
out := make([]string, 0, len(values))
for _, value := range values {
out = append(out, literal(value))
}
return strings.Join(out, ", ")
}
func literal(value any) string {
if value == nil {
return "NULL"
}
switch typed := value.(type) {
case time.Time:
return "'" + typed.UTC().Format("2006-01-02 15:04:05.000") + "'"
case string:
return "'" + quote(typed) + "'"
case envelope.ParseStatus:
return "'" + quote(string(typed)) + "'"
case int:
return strconv.FormatInt(int64(typed), 10)
case int64:
return strconv.FormatInt(typed, 10)
case uint16:
return strconv.FormatUint(uint64(typed), 10)
case uint32:
return strconv.FormatUint(uint64(typed), 10)
case float64:
return strconv.FormatFloat(typed, 'f', -1, 64)
case float32:
return strconv.FormatFloat(float64(typed), 'f', -1, 64)
default:
return "'" + quote(fmt.Sprint(typed)) + "'"
}
}

View File

@@ -0,0 +1,146 @@
package history
import (
"context"
"database/sql"
"strings"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
statements := strings.Join(SchemaStatements("test_ts"), "\n")
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} {
if !strings.Contains(statements, want) {
t.Fatalf("schema missing %q:\n%s", want, statements)
}
}
}
func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
}
if got := countSQL(exec.calls, "USING raw_frames"); got != 1 {
t.Fatalf("raw child create count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
t.Fatalf("location child create count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 {
t.Fatalf("mileage child create count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
t.Fatalf("location insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 {
t.Fatalf("mileage insert count = %d", got)
}
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {
t.Fatalf("raw insert should quote string literals: %s", rawInsert)
}
if strings.Contains(rawInsert, "VALUES (?,") {
t.Fatalf("raw insert should not use placeholders for tdengine websocket plain insert: %s", rawInsert)
}
}
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
env.Fields = map[string]any{}
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
t.Fatalf("location insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
t.Fatalf("mileage insert count = %d", got)
}
}
func TestRawSizeBytesUsesRawTextWhenHexIsEmpty(t *testing.T) {
env := envelope.FrameEnvelope{RawText: `{"code":"0F80","data":{"speed":12}}`}
if got, want := rawSizeBytes(env), len([]byte(env.RawText)); got != want {
t.Fatalf("raw text size = %d, want %d", got, want)
}
env.RawHex = "7E0200"
if got, want := rawSizeBytes(env), 3; got != want {
t.Fatalf("raw hex size = %d, want %d", got, want)
}
}
func sampleEnvelope() envelope.FrameEnvelope {
return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
Sequence: 1,
VIN: "LNBVIN00000000001",
Phone: "013307795425",
SourceEndpoint: "127.0.0.1:18080",
EventTimeMS: 1782745114000,
ReceivedAtMS: 1782745114999,
RawHex: "7E0200",
Parsed: map[string]any{"message": "location"},
Fields: map[string]any{
envelope.FieldLongitude: 121.069881,
envelope.FieldLatitude: 30.590151,
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10241.2,
"direction_deg": uint16(79),
"alarm_flag": uint32(0),
"status_flag": uint32(72),
},
ParseStatus: envelope.ParseOK,
}
}
func countSQL(calls []execCall, pattern string) int {
var count int
for _, call := range calls {
if strings.Contains(call.query, pattern) {
count++
}
}
return count
}
func findSQL(calls []execCall, pattern string) string {
for _, call := range calls {
if strings.Contains(call.query, pattern) {
return call.query
}
}
return ""
}
type execCall struct {
query string
args []any
}
type recordingExec struct {
calls []execCall
}
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
e.calls = append(e.calls, execCall{query: query, args: args})
return nil, nil
}

View File

@@ -0,0 +1,237 @@
package identity
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Resolver interface {
Resolve(context.Context, envelope.FrameEnvelope) (envelope.FrameEnvelope, error)
}
type NoopResolver struct{}
func (NoopResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
return env, nil
}
type MySQLResolver struct {
db *sql.DB
table string
}
func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver {
if db == nil {
panic("identity db must not be nil")
}
table = strings.TrimSpace(table)
if table == "" || !safeIdentifier(table) {
table = "vehicle_identity_binding"
}
return &MySQLResolver{db: db, table: table}
}
func safeIdentifier(value string) bool {
if value == "" {
return false
}
for _, r := range value {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
continue
}
return false
}
return true
}
func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
if strings.TrimSpace(env.VIN) != "" {
if err := r.trackJT808(ctx, env); err != nil {
return env, err
}
return env, nil
}
candidates := CandidateKeys(env)
for _, candidate := range candidates {
vin, err := r.lookup(ctx, candidate.Column, candidate.Value)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
continue
}
return env, err
}
if strings.TrimSpace(vin) != "" {
env.VIN = strings.TrimSpace(vin)
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
env.Parsed["identity"] = map[string]any{
"resolved": true,
"source": candidate.Column,
"value": candidate.Value,
}
env.EventID = env.StableEventID()
if err := r.trackJT808(ctx, env); err != nil {
return env, err
}
return env, nil
}
}
if err := r.trackJT808(ctx, env); err != nil {
return env, err
}
return env, nil
}
func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) {
query := "SELECT vin FROM " + r.table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' ORDER BY updated_at DESC LIMIT 1"
var vin string
err := r.db.QueryRowContext(ctx, query, value).Scan(&vin)
return vin, err
}
func (r *MySQLResolver) trackJT808(ctx context.Context, env envelope.FrameEnvelope) error {
if env.Protocol != envelope.ProtocolJT808 || strings.TrimSpace(env.Phone) == "" || strings.TrimSpace(env.MessageID) == "" {
return nil
}
registration := mapValue(env.Parsed, "registration")
authentication := mapValue(env.Parsed, "authentication")
deviceID := firstNonEmpty(env.DeviceID, textValue(registration, "device_id"))
plate := firstNonEmpty(env.Plate, textValue(registration, "plate"))
vin := strings.TrimSpace(env.VIN)
if vin == "" {
vin = "unknown"
}
firstRegisteredAt := nilTime()
latestRegisteredAt := nilTime()
if env.MessageID == "0x0100" {
firstRegisteredAt = "CURRENT_TIMESTAMP"
latestRegisteredAt = "CURRENT_TIMESTAMP"
}
latestAuthenticatedAt := nilTime()
if env.MessageID == "0x0102" {
latestAuthenticatedAt = "CURRENT_TIMESTAMP"
}
query := `INSERT INTO jt808_registration
(phone, device_id, plate, vin, province, city, manufacturer, device_type, plate_color,
auth_token, auth_imei, auth_software_version, source_endpoint,
first_registered_at, latest_registered_at, latest_authenticated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ` + firstRegisteredAt + `, ` + latestRegisteredAt + `, ` + latestAuthenticatedAt + `)
ON DUPLICATE KEY UPDATE
device_id = IF(VALUES(device_id) <> '', VALUES(device_id), device_id),
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
vin = IF(VALUES(vin) <> 'unknown', VALUES(vin), vin),
province = IF(VALUES(province) <> '', VALUES(province), province),
city = IF(VALUES(city) <> '', VALUES(city), city),
manufacturer = IF(VALUES(manufacturer) <> '', VALUES(manufacturer), manufacturer),
device_type = IF(VALUES(device_type) <> '', VALUES(device_type), device_type),
plate_color = IF(VALUES(plate_color) <> '', VALUES(plate_color), plate_color),
auth_token = IF(VALUES(auth_token) <> '', VALUES(auth_token), auth_token),
auth_imei = IF(VALUES(auth_imei) <> '', VALUES(auth_imei), auth_imei),
auth_software_version = IF(VALUES(auth_software_version) <> '', VALUES(auth_software_version), auth_software_version),
source_endpoint = IF(VALUES(source_endpoint) <> '', VALUES(source_endpoint), source_endpoint),
latest_seen_at = CURRENT_TIMESTAMP,
first_registered_at = COALESCE(first_registered_at, VALUES(first_registered_at)),
latest_registered_at = COALESCE(VALUES(latest_registered_at), latest_registered_at),
latest_authenticated_at = COALESCE(VALUES(latest_authenticated_at), latest_authenticated_at)`
_, err := r.db.ExecContext(ctx, query,
env.Phone,
deviceID,
plate,
vin,
textValue(registration, "province"),
textValue(registration, "city"),
textValue(registration, "manufacturer"),
textValue(registration, "device_type"),
textValue(registration, "plate_color"),
textValue(authentication, "token"),
textValue(authentication, "imei"),
textValue(authentication, "software_version"),
env.SourceEndpoint,
)
if err != nil {
return err
}
if vin == "unknown" {
return nil
}
_, err = r.db.ExecContext(ctx, `UPDATE IGNORE `+r.table+`
SET
phone = CASE WHEN (phone IS NULL OR phone = '') AND ? <> '' THEN ? ELSE phone END,
device_id = CASE WHEN (device_id IS NULL OR device_id = '') AND ? <> '' THEN ? ELSE device_id END,
plate = CASE WHEN (plate IS NULL OR plate = '') AND ? <> '' THEN ? ELSE plate END
WHERE vin = ?`, env.Phone, env.Phone, deviceID, deviceID, plate, plate, vin)
return err
}
type CandidateKey struct {
Column string
Value string
}
func CandidateKeys(env envelope.FrameEnvelope) []CandidateKey {
var out []CandidateKey
add := func(column string, value string) {
value = strings.TrimSpace(value)
if value == "" || strings.EqualFold(value, "unknown") {
return
}
for _, existing := range out {
if existing.Column == column && existing.Value == value {
return
}
}
out = append(out, CandidateKey{Column: column, Value: value})
}
add("phone", env.Phone)
if trimmed := strings.TrimLeft(strings.TrimSpace(env.Phone), "0"); trimmed != "" {
add("phone", trimmed)
}
add("device_id", env.DeviceID)
add("plate", env.Plate)
add("vin", env.VehicleKeyHint)
return out
}
func mapValue(values map[string]any, key string) map[string]any {
if values == nil {
return nil
}
nested, _ := values[key].(map[string]any)
return nested
}
func textValue(values map[string]any, key string) string {
if values == nil {
return ""
}
value, ok := values[key]
if !ok || value == nil {
return ""
}
switch typed := value.(type) {
case string:
return strings.TrimSpace(typed)
default:
return strings.TrimSpace(fmt.Sprint(typed))
}
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
}
return ""
}
func nilTime() string {
return "NULL"
}

View File

@@ -0,0 +1,171 @@
package identity
import (
"context"
"database/sql"
"testing"
"github.com/DATA-DOG/go-sqlmock"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestCandidateKeysPrioritizesPhoneDevicePlate(t *testing.T) {
keys := CandidateKeys(envelope.FrameEnvelope{
Phone: "013307795425",
DeviceID: "D1",
Plate: "豫A12345",
VehicleKeyHint: "D1",
})
got := []string{}
for _, key := range keys {
got = append(got, key.Column+"="+key.Value)
}
want := []string{"phone=013307795425", "phone=13307795425", "device_id=D1", "plate=豫A12345", "vin=D1"}
if len(got) != len(want) {
t.Fatalf("candidate count = %d got=%#v", len(got), got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
}
}
}
func TestCandidateKeysAddsTrimmedPhoneVariant(t *testing.T) {
keys := CandidateKeys(envelope.FrameEnvelope{
Phone: "013079963379",
})
got := []string{}
for _, key := range keys {
got = append(got, key.Column+"="+key.Value)
}
want := []string{"phone=013079963379", "phone=13079963379"}
if len(got) != len(want) {
t.Fatalf("candidate count = %d got=%#v", len(got), got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
}
}
}
func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
db, mock := newMockDB(t)
defer db.Close()
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("013307795425").
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001"))
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307795425",
Parsed: map[string]any{},
})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if env.VIN != "LNBVIN00000000001" {
t.Fatalf("vin = %q", env.VIN)
}
identity, ok := env.Parsed["identity"].(map[string]any)
if !ok || identity["source"] != "phone" {
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMySQLResolverFallsBackToDeviceID(t *testing.T) {
db, mock := newMockDB(t)
defer db.Close()
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("013307795425").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("13307795425").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
WithArgs("D1").
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000002"))
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307795425",
DeviceID: "D1",
})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if env.VIN != "LNBVIN00000000002" {
t.Fatalf("vin = %q", env.VIN)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMySQLResolverTracksJT808RegistrationAndBackfillsBinding(t *testing.T) {
db, mock := newMockDB(t)
defer db.Close()
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("013079963379").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("13079963379").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
WithArgs("DEV0001").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
WithArgs("TEST123").
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LKLG7C4E3NA774736"))
mock.ExpectExec("INSERT INTO jt808_registration").
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE IGNORE vehicle_identity_binding").
WithArgs("013079963379", "013079963379", "DEV0001", "DEV0001", "TEST123", "TEST123", "LKLG7C4E3NA774736").
WillReturnResult(sqlmock.NewResult(0, 1))
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0100",
Phone: "013079963379",
DeviceID: "DEV0001",
Plate: "TEST123",
SourceEndpoint: "115.231.168.135:43625",
Parsed: map[string]any{
"registration": map[string]any{
"province": uint16(16),
"city": uint16(32),
"manufacturer": "YUTNG",
"device_type": "ZK6105CHEVNPG4",
"device_id": "DEV0001",
"plate_color": uint8(2),
"plate": "TEST123",
},
},
})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if env.VIN != "LKLG7C4E3NA774736" {
t.Fatalf("vin = %q", env.VIN)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) {
t.Helper()
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
return db, mock
}

View File

@@ -0,0 +1,13 @@
package observability
import (
"log/slog"
"os"
)
// NewLogger returns a JSON logger with a stable service field so ECS logs from
// different Go processes can be filtered without relying on container names.
func NewLogger(service string) *slog.Logger {
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
return slog.New(handler).With("service", service)
}

View File

@@ -0,0 +1,840 @@
package gb32960
import (
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var (
ErrFrameTooShort = errors.New("gb32960 frame too short")
ErrBadStartSymbol = errors.New("gb32960 bad start symbol")
ErrBodyLength = errors.New("gb32960 body length mismatch")
ErrBCC = errors.New("gb32960 bcc mismatch")
)
const (
minFrameLen = 25
headerLen = 24
)
// ExtractFrames splits GB/T 32960 TCP streams using the protocol length field.
// Returned frames include the start symbols and trailing BCC byte.
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
offset := 0
for {
start := indexStart(stream[offset:])
if start < 0 {
return frames, nil, nil
}
offset += start
remaining := stream[offset:]
if len(remaining) < headerLen {
return frames, append([]byte(nil), remaining...), nil
}
bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
frameLen := frameLength(remaining, bodyLen)
if len(remaining) < frameLen {
return frames, append([]byte(nil), remaining...), nil
}
frames = append(frames, append([]byte(nil), remaining[:frameLen]...))
offset += frameLen
if offset >= len(stream) {
return frames, nil, nil
}
}
}
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
if len(raw) < minFrameLen {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
}
version, ok := version(raw[0], raw[1])
if !ok {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
}
bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
actualBodyLen := len(raw) - headerLen - 1
if len(raw) != headerLen+bodyLen+1 && !isExtendedPlatformLogin(raw[2], bodyLen, actualBodyLen) {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
}
if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrBCC, got, want)
}
command := raw[2]
responseFlag := raw[3]
vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
body := raw[headerLen : headerLen+actualBodyLen]
parsed := map[string]any{
"header": map[string]any{
"version": version,
"command": fmt.Sprintf("0x%02X", command),
"response_flag": fmt.Sprintf("0x%02X", responseFlag),
"vin": vin,
"encrypt": raw[21],
"body_length": bodyLen,
"actual_body_length": actualBodyLen,
},
}
fields := map[string]any{}
eventTimeMS := receivedAtMS
if command == 0x02 || command == 0x03 {
eventTime, units := parseDataBody(version, body, fields)
if !eventTime.IsZero() {
eventTimeMS = eventTime.UnixMilli()
fields["device_time"] = eventTime.Format(time.RFC3339)
parsed["device_time"] = eventTime.Format(time.RFC3339)
}
parsed["data_units"] = units
} else if command == 0x05 {
login := parsePlatformLogin(body)
parsed["platform_login"] = login
if deviceTime, ok := login["login_time"].(string); ok && deviceTime != "" {
fields["device_time"] = deviceTime
parsed["device_time"] = deviceTime
}
if account, ok := login["username"].(string); ok && account != "" {
fields["platform_account"] = account
}
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
MessageID: fmt.Sprintf("0x%02X", command),
VIN: vin,
SourceEndpoint: sourceEndpoint,
EventTimeMS: eventTimeMS,
ReceivedAtMS: receivedAtMS,
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
Parsed: parsed,
Fields: fields,
ParseStatus: envelope.ParseOK,
}
env.EventID = env.StableEventID()
return env, nil
}
func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
if len(body) < 6 {
return time.Time{}, nil
}
eventTime := parseGBTime(body[:6])
cursor := 6
var units []map[string]any
for cursor < len(body) {
unitType := body[cursor]
cursor++
switch unitType {
case 0x01:
size := 20
if version == "V2025" {
size = 18
}
if len(body[cursor:]) < size {
units = append(units, map[string]any{"type": "0x01", "error": "truncated"})
return eventTime, units
}
unit := parseVehicleData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x01", "name": "vehicle", "value": unit})
fields["vehicle_status"] = unit["vehicle_status"]
fields["charge_status"] = unit["charge_status"]
fields["running_mode"] = unit["running_mode"]
fields[envelope.FieldSpeedKMH] = unit["speed_kmh"]
fields[envelope.FieldTotalMileageKM] = unit["total_mileage_km"]
fields[envelope.FieldSOCPercent] = unit["soc_percent"]
cursor += size
case 0x05:
if len(body[cursor:]) < 9 {
units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
return eventTime, units
}
unit := parsePositionData(body[cursor : cursor+9])
units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
fields["position_status"] = unit["position_status"]
fields[envelope.FieldLongitude] = unit["longitude"]
fields[envelope.FieldLatitude] = unit["latitude"]
cursor += 9
case 0x02:
if len(body[cursor:]) < 1 {
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
return eventTime, units
}
size := 1 + int(body[cursor])*12
if len(body[cursor:]) < size {
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
return eventTime, units
}
unit := parseDriveMotorData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x02", "name": "drive_motor", "value": unit})
cursor += size
case 0x03:
if len(body[cursor:]) < 8 {
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
return eventTime, units
}
probeCount := int(binary.BigEndian.Uint16(body[cursor+6 : cursor+8]))
size := 8 + probeCount + 10
if len(body[cursor:]) < size {
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
return eventTime, units
}
unit := parseFuelCellData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x03", "name": "fuel_cell", "value": unit})
fields["fuel_cell_hydrogen_consumption_kg_per_100km"] = unit["hydrogen_consumption_kg_per_100km"]
fields["fuel_cell_voltage_v"] = unit["fuel_cell_voltage_v"]
fields["fuel_cell_current_a"] = unit["fuel_cell_current_a"]
cursor += size
case 0x04:
if len(body[cursor:]) < 5 {
units = append(units, map[string]any{"type": "0x04", "error": "truncated"})
return eventTime, units
}
unit := parseEngineData(body[cursor : cursor+5])
units = append(units, map[string]any{"type": "0x04", "name": "engine", "value": unit})
cursor += 5
case 0x06:
if len(body[cursor:]) < 14 {
units = append(units, map[string]any{"type": "0x06", "error": "truncated"})
return eventTime, units
}
unit := parseExtremeData(body[cursor : cursor+14])
units = append(units, map[string]any{"type": "0x06", "name": "extreme", "value": unit})
cursor += 14
case 0x07:
size, ok := alarmDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x07", "error": "truncated"})
return eventTime, units
}
unit := parseAlarmData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x07", "name": "alarm", "value": unit})
cursor += size
case 0x08:
size, ok := voltageDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x08", "error": "truncated"})
return eventTime, units
}
unit := parseVoltageData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x08", "name": "voltage", "value": unit})
cursor += size
case 0x09:
size, ok := temperatureDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x09", "error": "truncated"})
return eventTime, units
}
unit := parseTemperatureData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x09", "name": "temperature", "value": unit})
cursor += size
case 0x30:
size, ok := gdFCStackDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x30", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCStackData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x30", "name": "gd_fc_stack", "value": unit})
if summaries, ok := unit["summaries"].([]map[string]any); ok && len(summaries) > 0 {
fields["gd_fc_stack_hydrogen_inlet_pressure_kpa"] = summaries[0]["hydrogen_inlet_pressure_kpa"]
fields["gd_fc_stack_water_outlet_temp_c"] = summaries[0]["stack_water_outlet_temp_c"]
fields["gd_fc_stack_cell_count"] = summaries[0]["cell_count"]
fields["gd_fc_stack_frame_cell_count"] = summaries[0]["frame_cell_count"]
}
cursor += size
case 0x31:
size, ok := gdFCAuxiliaryDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x31", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCAuxiliaryData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x31", "name": "gd_fc_auxiliary", "value": unit})
cursor += size
case 0x32:
if len(body[cursor:]) < 9 {
units = append(units, map[string]any{"type": "0x32", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCDcDcData(body[cursor : cursor+9])
units = append(units, map[string]any{"type": "0x32", "name": "gd_fc_dcdc", "value": unit})
fields["gd_fc_dcdc_controller_temp_c"] = unit["controller_temp_c"]
cursor += 9
case 0x33:
if len(body[cursor:]) < 5 {
units = append(units, map[string]any{"type": "0x33", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCAirConditionerData(body[cursor : cursor+5])
units = append(units, map[string]any{"type": "0x33", "name": "gd_fc_air_conditioner", "value": unit})
cursor += 5
case 0x34:
if len(body[cursor:]) < 7 {
units = append(units, map[string]any{"type": "0x34", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCVehicleInfoData(body[cursor : cursor+7])
units = append(units, map[string]any{"type": "0x34", "name": "gd_fc_vehicle_info", "value": unit})
fields["gd_fc_vehicle_hydrogen_mass_kg"] = unit["hydrogen_mass_kg"]
cursor += 7
case 0x80:
if len(body[cursor:]) < 11 {
units = append(units, map[string]any{"type": "0x80", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCDemoExtensionData(body[cursor : cursor+11])
units = append(units, map[string]any{"type": "0x80", "name": "gd_fc_demo_extension", "value": unit})
fields["gd_fc_demo_stack_temp_c"] = unit["stack_temp_c"]
cursor += 11
case 0x83:
size, ok := gdFCVendorTLVDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x83", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCVendorTLVData(0x83, body[cursor:cursor+size])
units = append(units, map[string]any{"type": "0x83", "name": "gd_fc_vendor_tlv", "value": unit})
cursor += size
default:
units = append(units, map[string]any{
"type": fmt.Sprintf("0x%02X", unitType),
"raw_tail": strings.ToUpper(hex.EncodeToString(body[cursor:])),
"parse": "unknown_unit",
"byte_size": len(body) - cursor,
})
return eventTime, units
}
}
return eventTime, units
}
func frameLength(frameStart []byte, bodyLen int) int {
standardLen := headerLen + bodyLen + 1
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
return standardLen
}
// Some field platform-login frames declare the standard 41-byte body but
// carry a 32-byte password, making the actual body 53 bytes. Only consume
// the extended form when its own BCC verifies so the splitter does not eat
// bytes from the next frame.
extendedLen := headerLen + 53 + 1
if len(frameStart) < extendedLen {
return standardLen
}
extended := frameStart[:extendedLen]
if bcc(extended[2:len(extended)-1]) == extended[len(extended)-1] {
return extendedLen
}
return standardLen
}
func isExtendedPlatformLogin(command byte, declaredBodyLen int, actualBodyLen int) bool {
return command == 0x05 && declaredBodyLen == 41 && actualBodyLen == 53
}
func parsePlatformLogin(body []byte) map[string]any {
login := map[string]any{
"raw_body_length": len(body),
}
if len(body) < 9 {
login["error"] = "truncated"
return login
}
loginTime := parseGBTime(body[:6])
if !loginTime.IsZero() {
login["login_time"] = loginTime.Format(time.RFC3339)
}
login["serial_no"] = binary.BigEndian.Uint16(body[6:8])
if len(body) >= 21 {
login["username"] = trimASCII(body[8:20])
}
if len(body) > 21 {
passwordEnd := len(body) - 1
login["password"] = trimASCII(body[20:passwordEnd])
login["encrypt_rule"] = body[passwordEnd]
}
return login
}
func parseVehicleData(data []byte) map[string]any {
return map[string]any{
"vehicle_status": int(data[0]),
"charge_status": int(data[1]),
"running_mode": int(data[2]),
"speed_kmh": float64(binary.BigEndian.Uint16(data[3:5])) / 10,
"total_mileage_km": float64(binary.BigEndian.Uint32(data[5:9])) / 10,
"total_voltage_v": float64(binary.BigEndian.Uint16(data[9:11])) / 10,
"total_current_a": float64(binary.BigEndian.Uint16(data[11:13]))/10 - 1000,
"soc_percent": int(data[13]),
"dc_dc_status": int(data[14]),
"gear": int(data[15]),
"insulation_kohm": binary.BigEndian.Uint16(data[16:18]),
"accelerator_pct": int(data[18]),
"brake_pct": int(data[19]),
}
}
func parseDriveMotorData(data []byte) map[string]any {
count := int(data[0])
motors := make([]map[string]any, 0, count)
cursor := 1
for i := 0; i < count; i++ {
motors = append(motors, map[string]any{
"serial_no": int(data[cursor]),
"state": int(data[cursor+1]),
"controller_temperature_c": int(data[cursor+2]) - 40,
"speed_rpm": int(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) - 20000,
"torque_nm": float64(int(binary.BigEndian.Uint16(data[cursor+5:cursor+7]))-20000) / 10,
"motor_temperature_c": int(data[cursor+7]) - 40,
"controller_voltage_v": float64(binary.BigEndian.Uint16(data[cursor+8:cursor+10])) / 10,
"controller_current_a": float64(binary.BigEndian.Uint16(data[cursor+10:cursor+12]))/10 - 1000,
})
cursor += 12
}
return map[string]any{
"count": count,
"motors": motors,
}
}
func parseFuelCellData(data []byte) map[string]any {
probeCount := int(binary.BigEndian.Uint16(data[6:8]))
probes := make([]int, 0, probeCount)
cursor := 8
for i := 0; i < probeCount; i++ {
probes = append(probes, int(data[cursor])-40)
cursor++
}
out := map[string]any{
"fuel_cell_voltage_v": float64(binary.BigEndian.Uint16(data[0:2])) / 10,
"fuel_cell_current_a": float64(binary.BigEndian.Uint16(data[2:4])) / 10,
"hydrogen_consumption_kg_per_100km": float64(binary.BigEndian.Uint16(data[4:6])) / 100,
"temperature_probe_count": probeCount,
"temperature_probe_values_c": probes,
"max_hydrogen_temperature_c": float64(binary.BigEndian.Uint16(data[cursor:cursor+2]))/10 - 40,
"max_hydrogen_temperature_probe_id": int(data[cursor+2]),
"max_hydrogen_concentration_fraction": float64(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) * 0.000001,
"max_hydrogen_concentration_probe_id": int(data[cursor+5]),
"max_hydrogen_pressure_mpa": float64(binary.BigEndian.Uint16(data[cursor+6:cursor+8])) / 10,
"max_hydrogen_pressure_probe_id": int(data[cursor+8]),
"dc_dc_status": int(data[cursor+9]),
}
return out
}
func parseEngineData(data []byte) map[string]any {
return map[string]any{
"engine_status": int(data[0]),
"crank_speed_rpm": binary.BigEndian.Uint16(data[1:3]),
"fuel_rate": binary.BigEndian.Uint16(data[3:5]),
}
}
func parseExtremeData(data []byte) map[string]any {
return map[string]any{
"max_voltage_subsystem_no": int(data[0]),
"max_voltage_cell_no": int(data[1]),
"max_voltage_v": float64(binary.BigEndian.Uint16(data[2:4])) / 1000,
"min_voltage_subsystem_no": int(data[4]),
"min_voltage_cell_no": int(data[5]),
"min_voltage_v": float64(binary.BigEndian.Uint16(data[6:8])) / 1000,
"max_temp_subsystem_no": int(data[8]),
"max_temp_probe_no": int(data[9]),
"max_temp_c": int(data[10]) - 40,
"min_temp_subsystem_no": int(data[11]),
"min_temp_probe_no": int(data[12]),
"min_temp_c": int(data[13]) - 40,
}
}
func alarmDataSize(data []byte) (int, bool) {
if len(data) < 5 {
return 0, false
}
cursor := 5
for i := 0; i < 4; i++ {
if len(data[cursor:]) < 1 {
return 0, false
}
count := int(data[cursor])
cursor++
cursor += count * 4
if cursor > len(data) {
return 0, false
}
}
return cursor, true
}
func parseAlarmData(data []byte) map[string]any {
cursor := 5
readList := func() []string {
count := int(data[cursor])
cursor++
values := make([]string, 0, count)
for i := 0; i < count; i++ {
values = append(values, fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[cursor:cursor+4])))
cursor += 4
}
return values
}
return map[string]any{
"max_alarm_level": int(data[0]),
"general_alarm_flag": fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[1:5])),
"battery_faults": readList(),
"motor_faults": readList(),
"engine_faults": readList(),
"other_faults": readList(),
}
}
func voltageDataSize(data []byte) (int, bool) {
if len(data) < 1 {
return 0, false
}
cursor := 1
for i := 0; i < int(data[0]); i++ {
if len(data[cursor:]) < 10 {
return 0, false
}
frameCellCount := int(data[cursor+9])
cursor += 10 + frameCellCount*2
if cursor > len(data) {
return 0, false
}
}
return cursor, true
}
func parseVoltageData(data []byte) map[string]any {
subCount := int(data[0])
cursor := 1
totalVoltage := 0.0
maxCell := 0.0
minCell := 0.0
cellSeen := false
for i := 0; i < subCount; i++ {
totalVoltage += float64(binary.BigEndian.Uint16(data[cursor+1:cursor+3])) / 10
cellCount := int(binary.BigEndian.Uint16(data[cursor+5 : cursor+7]))
frameCellCount := int(data[cursor+9])
cursor += 10
for c := 0; c < frameCellCount; c++ {
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
if !cellSeen || voltage > maxCell {
maxCell = voltage
}
if !cellSeen || voltage < minCell {
minCell = voltage
}
cellSeen = true
cursor += 2
}
_ = cellCount
}
return map[string]any{
"subsystem_count": subCount,
"total_voltage_v": totalVoltage,
"max_cell_v": maxCell,
"min_cell_v": minCell,
}
}
func temperatureDataSize(data []byte) (int, bool) {
if len(data) < 1 {
return 0, false
}
cursor := 1
for i := 0; i < int(data[0]); i++ {
if len(data[cursor:]) < 3 {
return 0, false
}
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
cursor += 3 + probeCount
if cursor > len(data) {
return 0, false
}
}
return cursor, true
}
func parseTemperatureData(data []byte) map[string]any {
subCount := int(data[0])
cursor := 1
maxTemp := 0
minTemp := 0
seen := false
for i := 0; i < subCount; i++ {
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
cursor += 3
for p := 0; p < probeCount; p++ {
temp := int(data[cursor]) - 40
if !seen || temp > maxTemp {
maxTemp = temp
}
if !seen || temp < minTemp {
minTemp = temp
}
seen = true
cursor++
}
}
return map[string]any{
"subsystem_count": subCount,
"max_temp_c": maxTemp,
"min_temp_c": minTemp,
}
}
func gdFCStackDataSize(data []byte) (int, bool) {
if len(data) < 1 {
return 0, false
}
cursor := 1
for i := 0; i < int(data[0]); i++ {
if len(data[cursor:]) < 22 {
return 0, false
}
frameCellCount := int(data[cursor+21])
cursor += 22 + frameCellCount*2
if cursor > len(data) {
return 0, false
}
}
return cursor, true
}
func parseGdFCStackData(data []byte) map[string]any {
count := int(data[0])
cursor := 1
summaries := make([]map[string]any, 0, count)
for i := 0; i < count; i++ {
frameCellCount := int(data[cursor+21])
summary := map[string]any{
"engine_work_state": int(data[cursor]),
"stack_water_outlet_temp_c": nullableTempOffset40(data[cursor+1]),
"hydrogen_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+2:cursor+4], 0.1, -100),
"air_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+4:cursor+6], 0.1, -100),
"air_inlet_temp_c": nullableTempOffset40(data[cursor+6]),
"max_cell_voltage_id": nullableU16(data[cursor+7 : cursor+9]),
"min_cell_voltage_id": nullableU16(data[cursor+9 : cursor+11]),
"max_cell_voltage_v": nullableScaledU16(data[cursor+11:cursor+13], 0.001),
"min_cell_voltage_v": nullableScaledU16(data[cursor+13:cursor+15], 0.001),
"avg_cell_voltage_v": nullableScaledU16(data[cursor+15:cursor+17], 0.001),
"cell_count": nullableU16(data[cursor+17 : cursor+19]),
"frame_cell_start": nullableU16(data[cursor+19 : cursor+21]),
"frame_cell_count": frameCellCount,
}
cursor += 22
maxCell := 0.0
minCell := 0.0
seen := false
for c := 0; c < frameCellCount; c++ {
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
if !seen || voltage > maxCell {
maxCell = voltage
}
if !seen || voltage < minCell {
minCell = voltage
}
seen = true
cursor += 2
}
if seen {
summary["frame_max_cell_voltage_v"] = maxCell
summary["frame_min_cell_voltage_v"] = minCell
}
summaries = append(summaries, summary)
}
return map[string]any{
"stack_count": count,
"summaries": summaries,
}
}
func gdFCAuxiliaryDataSize(data []byte) (int, bool) {
if len(data) < 1 {
return 0, false
}
size := 1 + int(data[0])*16
return size, len(data) >= size
}
func parseGdFCAuxiliaryData(data []byte) map[string]any {
count := int(data[0])
cursor := 1
subsystems := make([]map[string]any, 0, count)
for i := 0; i < count; i++ {
subsystems = append(subsystems, map[string]any{
"air_compressor_motor_voltage_v": nullableScaledU16(data[cursor:cursor+2], 0.1),
"air_compressor_power_kw": nullableU16WithOffset(data[cursor+2:cursor+4], -5),
"hydrogen_pump_motor_voltage_v": nullableScaledU16(data[cursor+4:cursor+6], 0.2),
"hydrogen_pump_power_kw": nullableU16WithOffset(data[cursor+6:cursor+8], -5),
"water_pump_voltage_v": nullableScaledU16(data[cursor+8:cursor+10], 0.1),
"ptc_voltage_v": nullableScaledU16(data[cursor+10:cursor+12], 0.1),
"ptc_power_kw": nullableU16WithOffset(data[cursor+12:cursor+14], -5),
"low_voltage_battery_voltage_v": nullableScaledU16(data[cursor+14:cursor+16], 0.1),
})
cursor += 16
}
return map[string]any{
"subsystem_count": count,
"subsystems": subsystems,
}
}
func parseGdFCDcDcData(data []byte) map[string]any {
return map[string]any{
"input_voltage_v": nullableScaledU16(data[0:2], 0.1),
"input_current_a": nullableScaledU16(data[2:4], 0.1),
"output_voltage_v": nullableScaledU16(data[4:6], 0.1),
"output_current_a": nullableScaledU16(data[6:8], 0.1),
"controller_temp_c": nullableTempOffset40(data[8]),
}
}
func parseGdFCAirConditionerData(data []byte) map[string]any {
return map[string]any{
"status": int(data[0]),
"power_kw": nullableU16WithOffset(data[1:3], -5),
"compressor_input_voltage_v": nullableScaledU16(data[3:5], 0.1),
}
}
func parseGdFCVehicleInfoData(data []byte) map[string]any {
return map[string]any{
"collision_alarm": int(data[0]),
"ambient_temp_c": nullableU16WithOffset(data[1:3], -40),
"ambient_pressure_kpa": nullableScaledU16WithOffset(data[3:5], 0.1, -100),
"hydrogen_mass_kg": nullableScaledU16(data[5:7], 0.1),
}
}
func parseGdFCDemoExtensionData(data []byte) map[string]any {
return map[string]any{
"declared_length": int(binary.BigEndian.Uint16(data[0:2])),
"stack_temp_c": nullableTempOffset40(data[2]),
"air_compressor_voltage_v": nullableScaledU16(data[3:5], 0.1),
"air_compressor_current_a": nullableScaledU16WithOffset(data[5:7], 0.1, -1000),
"hydrogen_pump_voltage_v": nullableScaledU16(data[7:9], 0.1),
"hydrogen_pump_current_a": nullableScaledU16WithOffset(data[9:11], 0.1, -1000),
}
}
func gdFCVendorTLVDataSize(data []byte) (int, bool) {
if len(data) < 2 {
return 0, false
}
size := 2 + int(binary.BigEndian.Uint16(data[0:2]))
return size, len(data) >= size
}
func parseGdFCVendorTLVData(typeCode byte, data []byte) map[string]any {
length := int(binary.BigEndian.Uint16(data[0:2]))
return map[string]any{
"type": fmt.Sprintf("0x%02X", typeCode),
"declared_length": length,
"payload_hex": strings.ToUpper(hex.EncodeToString(data[2:])),
}
}
func nullableU16(data []byte) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
return nil
}
return int(value)
}
func nullableU16WithOffset(data []byte, offset int) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
return nil
}
return int(value) + offset
}
func nullableScaledU16(data []byte, scale float64) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
return nil
}
return float64(value) * scale
}
func nullableScaledU16WithOffset(data []byte, scale float64, offset float64) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
return nil
}
return float64(value)*scale + offset
}
func nullableTempOffset40(value byte) any {
if value == 0xfe || value == 0xff {
return nil
}
return int(value) - 40
}
func parsePositionData(data []byte) map[string]any {
return map[string]any{
"position_status": int(data[0]),
"longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
"latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
}
}
func indexStart(data []byte) int {
for i := 0; i+1 < len(data); i++ {
if _, ok := version(data[i], data[i+1]); ok {
return i
}
}
return -1
}
func version(a byte, b byte) (string, bool) {
switch {
case a == '#' && b == '#':
return "V2016", true
case a == '$' && b == '$':
return "V2025", true
default:
return "", false
}
}
func parseGBTime(data []byte) time.Time {
if len(data) != 6 {
return time.Time{}
}
year := 2000 + int(data[0])
month := time.Month(data[1])
day := int(data[2])
hour := int(data[3])
minute := int(data[4])
second := int(data[5])
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
return time.Time{}
}
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
}
func trimASCII(data []byte) string {
return strings.TrimRight(string(data), "\x00 ")
}
func bcc(data []byte) byte {
var out byte
for _, value := range data {
out ^= value
}
return out
}

View File

@@ -0,0 +1,258 @@
package gb32960
import (
"encoding/hex"
"fmt"
"math"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39})
second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
stream := append([]byte{0x00, 0x01}, first...)
stream = append(stream, second[:len(second)-3]...)
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 {
t.Fatalf("expected one complete frame, got %d", len(frames))
}
if string(remainder) != string(second[:len(second)-3]) {
t.Fatalf("expected partial second frame as remainder, got %s", hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782745114999, "115.29.187.205:32960")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x02" {
t.Fatalf("unexpected envelope header: %#v", env)
}
if env.VIN != "LNBSCB3D4R1234567" {
t.Fatalf("unexpected vin: %q", env.VIN)
}
}
func TestParseFrameRejectsBadBCC(t *testing.T) {
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
frame[len(frame)-1] ^= 0xff
if _, err := ParseFrame(frame, 1782745114999, ""); err == nil {
t.Fatal("expected bad bcc error")
}
}
func TestExtractFramesSupportsExtendedPlatformLogin(t *testing.T) {
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x1f, 0x00, 0x01}
body = append(body, fixedASCII("Hyundai", 12)...)
body = append(body, fixedASCII("f2e3445d7cda409fb4f278f6fb890734", 32)...)
body = append(body, 0x01)
frame := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
frames, remainder, err := ExtractFrames(append(frame, []byte{0x23, 0x23, 0x05}...))
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 3 {
t.Fatalf("unexpected split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
if len(frames[0]) != headerLen+53+1 {
t.Fatalf("extended frame len = %d", len(frames[0]))
}
env, err := ParseFrame(frames[0], 1782745114999, "8.134.95.166:39376")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x05" || env.ParseStatus != envelope.ParseOK {
t.Fatalf("unexpected env: %#v", env)
}
header := env.Parsed["header"].(map[string]any)
if header["body_length"] != 41 || header["actual_body_length"] != 53 {
t.Fatalf("unexpected length header: %#v", header)
}
login := env.Parsed["platform_login"].(map[string]any)
if login["username"] != "Hyundai" || login["password"] != "f2e3445d7cda409fb4f278f6fb890734" {
t.Fatalf("unexpected platform login: %#v", login)
}
if login["login_time"] != "2026-07-02T00:38:31+08:00" {
t.Fatalf("unexpected login time: %#v", login)
}
if env.Fields["platform_account"] != "Hyundai" {
t.Fatalf("platform account not exposed: %#v", env.Fields)
}
}
func TestAutoResponseEchoesRawVINAndOriginalTime(t *testing.T) {
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x0f, 0x00, 0x01}
body = append(body, fixedASCII("Hyundai", 12)...)
body = append(body, fixedASCII("ack-test-password", 32)...)
body = append(body, 0x01)
request := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
env, err := ParseFrame(request, 1782914969584, "8.134.95.166:39376")
if err != nil {
t.Fatal(err)
}
response, ok, err := AutoResponse(request, env)
if err != nil {
t.Fatalf("AutoResponse() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
if response[2] != 0x05 || response[3] != 0x01 {
t.Fatalf("unexpected response header: %x", response[:4])
}
if hex.EncodeToString(response[4:21]) != "0000000000000000000000000000000000" {
t.Fatalf("raw vin not echoed: %x", response[4:21])
}
if hex.EncodeToString(response[24:30]) != "1a070200260f" {
t.Fatalf("timestamp body not preserved: %x", response[24:30])
}
if got, want := bcc(response[2:len(response)-1]), response[len(response)-1]; got != want {
t.Fatalf("response bcc got=0x%02x want=0x%02x", got, want)
}
}
func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
body := []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39}
body = append(body, 0x01)
body = append(body,
0x01, // vehicle status
0x03, // charge status
0x02, // running mode
0x01, 0x2c, // speed: 30.0km/h
0x00, 0x01, 0x86, 0xa0, // total mileage: 10000.0km
0x15, 0xe5, // total voltage: 560.5V
0x27, 0x10, // total current: 0A after -1000 offset
85, 0x01, 0x00,
0x27, 0x10,
0x00, 0x00)
body = append(body, 0x05)
body = append(body,
0x00,
0x07, 0x36, 0x50, 0x40, // longitude: 121.000000
0x01, 0xd2, 0x4f, 0x00) // latitude: 30.560000
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", body)
env, err := ParseFrame(frame, 1782745114999, "115.29.187.205:32960")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
assertFloatField(t, env, envelope.FieldSpeedKMH, 30.0)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10000.0)
assertFloatField(t, env, envelope.FieldLongitude, 121.0)
assertFloatField(t, env, envelope.FieldLatitude, 30.56)
if env.Fields[envelope.FieldSOCPercent] != 85 {
t.Fatalf("unexpected soc: %#v", env.Fields[envelope.FieldSOCPercent])
}
if env.EventTimeMS == 1782745114999 {
t.Fatal("event time should come from GB32960 device timestamp")
}
}
func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testing.T) {
frame, err := hex.DecodeString(realFuelCellFrame)
if err != nil {
t.Fatal(err)
}
env, err := ParseFrame(frame, 1782914969584, "115.29.187.205:32960")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.VIN != "LB9A32A21R0LS1707" {
t.Fatalf("vin = %q", env.VIN)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 53490.9)
assertFloatField(t, env, "fuel_cell_hydrogen_consumption_kg_per_100km", 1.8)
assertFloatField(t, env, envelope.FieldLongitude, 120.800326)
assertFloatField(t, env, envelope.FieldLatitude, 31.634907)
assertFloatField(t, env, "gd_fc_stack_hydrogen_inlet_pressure_kpa", 97.0)
assertFloatField(t, env, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
assertIntField(t, env, "gd_fc_demo_stack_temp_c", 65)
units, ok := env.Parsed["data_units"].([]map[string]any)
if !ok {
t.Fatalf("data_units missing: %#v", env.Parsed["data_units"])
}
gotTypes := make([]string, 0, len(units))
for _, unit := range units {
gotTypes = append(gotTypes, fmt.Sprint(unit["type"]))
}
wantTypes := []string{"0x01", "0x02", "0x03", "0x04", "0x05", "0x06", "0x07", "0x08", "0x09", "0x30", "0x31", "0x32", "0x33", "0x34", "0x80", "0x83"}
if fmt.Sprint(gotTypes) != fmt.Sprint(wantTypes) {
t.Fatalf("unit types = %v, want %v", gotTypes, wantTypes)
}
if units[1]["name"] != "drive_motor" {
t.Fatalf("unexpected motor unit: %#v", units[1])
}
if units[2]["name"] != "fuel_cell" {
t.Fatalf("unexpected fuel cell unit: %#v", units[2])
}
if units[9]["name"] != "gd_fc_stack" {
t.Fatalf("unexpected gd stack unit: %#v", units[9])
}
if units[10]["name"] != "gd_fc_auxiliary" {
t.Fatalf("unexpected gd auxiliary unit: %#v", units[10])
}
if units[15]["name"] != "gd_fc_vendor_tlv" {
t.Fatalf("unexpected gd vendor tlv unit: %#v", units[15])
}
tlv := units[15]["value"].(map[string]any)
if tlv["declared_length"] != 36 {
t.Fatalf("unexpected vendor tlv: %#v", tlv)
}
}
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
t.Helper()
got, ok := env.Fields[key].(float64)
if !ok {
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
}
if math.Abs(got-want) > 0.000001 {
t.Fatalf("field %s = %v, want %v", key, got, want)
}
}
func assertIntField(t *testing.T, env envelope.FrameEnvelope, key string, want int) {
t.Helper()
got, ok := env.Fields[key].(int)
if !ok {
t.Fatalf("field %s missing or not int: %#v", key, env.Fields[key])
}
if got != want {
t.Fatalf("field %s = %v, want %v", key, got, want)
}
}
func buildFrame(command byte, response byte, vin string, body []byte) []byte {
return buildFrameWithDeclaredLength(command, response, vin, body, len(body))
}
func buildFrameWithDeclaredLength(command byte, response byte, vin string, body []byte, declaredBodyLen int) []byte {
frame := []byte{'#', '#', command, response}
vinBytes := []byte(vin)
if len(vinBytes) < 17 {
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
}
frame = append(frame, vinBytes[:17]...)
frame = append(frame, 0x01, byte(declaredBodyLen>>8), byte(declaredBodyLen))
frame = append(frame, body...)
return append(frame, bcc(frame[2:]))
}
func fixedASCII(value string, size int) []byte {
out := make([]byte, size)
copy(out, []byte(value))
return out
}

View File

@@ -0,0 +1,80 @@
package gb32960
import (
"encoding/binary"
"errors"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrResponseFrameTooShort = errors.New("gb32960 response frame too short")
const (
responseSuccess = byte(0x01)
responseCommand = byte(0xfe)
encryptNone = byte(0x01)
)
// AutoResponse builds the protocol ACK for accepted upstream GB/T 32960 frames.
// The ACK is written only after the caller has durably published the frame.
func AutoResponse(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if env.ParseStatus == envelope.ParseBadFrame {
return nil, false, nil
}
if len(raw) < headerLen+1 {
return nil, false, ErrResponseFrameTooShort
}
if raw[3] != responseCommand {
return nil, false, nil
}
command := raw[2]
if !shouldRespond(command) {
return nil, false, nil
}
body := []byte(nil)
if timestamp := responseTime(raw, command); !timestamp.IsZero() {
body = encodeGBTime(timestamp)
}
return buildResponse(raw[0], command, responseSuccess, raw[4:21], body), true, nil
}
func shouldRespond(command byte) bool {
switch command {
case 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08:
return true
default:
return false
}
}
func responseTime(raw []byte, command byte) time.Time {
bodyLen := len(raw) - headerLen - 1
if bodyLen >= 6 {
return parseGBTime(raw[headerLen : headerLen+6])
}
return time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600))
}
func encodeGBTime(t time.Time) []byte {
local := t.In(time.FixedZone("Asia/Shanghai", 8*3600))
return []byte{
byte(local.Year() - 2000),
byte(local.Month()),
byte(local.Day()),
byte(local.Hour()),
byte(local.Minute()),
byte(local.Second()),
}
}
func buildResponse(start byte, command byte, responseFlag byte, vin17 []byte, body []byte) []byte {
out := make([]byte, 0, headerLen+len(body)+1)
out = append(out, start, start, command, responseFlag)
out = append(out, vin17[:17]...)
out = append(out, encryptNone, 0, 0)
binary.BigEndian.PutUint16(out[22:24], uint16(len(body)))
out = append(out, body...)
return append(out, bcc(out[2:]))
}

View File

@@ -0,0 +1,600 @@
package jt808
import (
"bytes"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"golang.org/x/text/encoding/simplifiedchinese"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var (
ErrFrameTooShort = errors.New("jt808 frame too short")
ErrChecksum = errors.New("jt808 checksum mismatch")
ErrEscape = errors.New("jt808 invalid escape sequence")
)
type Location struct {
AlarmFlag uint32
StatusFlag uint32
Latitude float64
Longitude float64
AltitudeM uint16
SpeedKMH float64
DirectionDeg uint16
Time time.Time
TotalMileageKM *float64
Additional []AdditionalItem
}
type AdditionalItem struct {
ID byte
Length int
ValueHex string
Parsed any
}
// ExtractFrames splits JT/T 808 TCP streams by 0x7e delimiters and unescapes
// complete frame payloads. Returned frames exclude the leading and trailing 0x7e.
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
start := -1
for i, value := range stream {
if value != 0x7e {
continue
}
if start < 0 {
start = i
continue
}
if i == start+1 {
start = i
continue
}
frame, err := unescape(stream[start+1 : i])
if err != nil {
return nil, nil, err
}
frames = append(frames, frame)
start = i
}
if start >= 0 && start < len(stream)-1 {
return frames, append([]byte(nil), stream[start:]...), nil
}
return frames, nil, nil
}
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
if len(raw) < 13 {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
}
if got, want := checksum(raw[:len(raw)-1]), raw[len(raw)-1]; got != want {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrChecksum, got, want)
}
messageID := binary.BigEndian.Uint16(raw[0:2])
props := binary.BigEndian.Uint16(raw[2:4])
bodySize := props & 0x03ff
versioned := props&0x4000 != 0
headerLen := 12
phoneStart := 4
phoneEnd := 10
serialStart := 10
packageInfo := map[string]any(nil)
protocolVersion := byte(0)
if versioned {
headerLen = 17
if len(raw) < headerLen {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: versioned header len=%d", ErrFrameTooShort, len(raw))
}
protocolVersion = raw[4]
phoneStart = 5
phoneEnd = 15
serialStart = 15
}
subpackage := props&0x2000 != 0
if subpackage {
headerLen += 4
}
if len(raw) < headerLen+int(bodySize)+1 {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
}
if subpackage {
packageInfo = map[string]any{
"total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]),
"index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]),
}
}
phoneBCD := raw[phoneStart:phoneEnd]
phone := bcdString(phoneBCD)
phoneTrimZero := strings.TrimLeft(phone, "0")
if versioned && phoneTrimZero != "" {
phone = phoneTrimZero
}
body := raw[headerLen : headerLen+int(bodySize)]
parsed := map[string]any{
"header": map[string]any{
"message_id": fmt.Sprintf("0x%04X", messageID),
"protocol_version": protocolVersion,
"versioned": versioned,
"properties": props,
"body_size": bodySize,
"phone_bcd_hex": strings.ToUpper(hex.EncodeToString(phoneBCD)),
"phone": phone,
"phone_trim_zero": phoneTrimZero,
"sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
"subpackage": subpackage,
"package_info": packageInfo,
"encryption_flags": (props >> 10) & 0x07,
},
}
fields := map[string]any{}
eventTimeMS := receivedAtMS
if messageID == 0x0200 {
location, err := parseLocation(body)
if err != nil {
return envelope.FrameEnvelope{}, err
}
parsed["location"] = locationToMap(location)
fields[envelope.FieldLatitude] = location.Latitude
fields[envelope.FieldLongitude] = location.Longitude
fields[envelope.FieldSpeedKMH] = location.SpeedKMH
fields["altitude_m"] = location.AltitudeM
fields["direction_deg"] = location.DirectionDeg
fields["alarm_flag"] = location.AlarmFlag
fields["status_flag"] = location.StatusFlag
if !location.Time.IsZero() {
eventTimeMS = location.Time.UnixMilli()
fields["device_time"] = location.Time.Format(time.RFC3339)
}
if location.TotalMileageKM != nil {
fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
}
} else if messageID == 0x0100 {
registration, err := parseRegistration(body, versioned)
if err != nil {
return envelope.FrameEnvelope{}, err
}
parsed["registration"] = registration
if deviceID, ok := registration["device_id"].(string); ok {
fields["device_id"] = deviceID
}
if plate, ok := registration["plate"].(string); ok {
fields["plate"] = plate
}
} else if messageID == 0x0102 {
authentication := parseAuthentication(body, versioned)
parsed["authentication"] = authentication
if token, ok := authentication["token"].(string); ok {
fields["auth_token"] = token
}
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: fmt.Sprintf("0x%04X", messageID),
Sequence: binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
Phone: phone,
SourceEndpoint: sourceEndpoint,
EventTimeMS: eventTimeMS,
ReceivedAtMS: receivedAtMS,
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
Parsed: parsed,
Fields: fields,
ParseStatus: envelope.ParseOK,
}
if registration, ok := parsed["registration"].(map[string]any); ok {
env.DeviceID, _ = registration["device_id"].(string)
env.Plate, _ = registration["plate"].(string)
}
env.EventID = env.StableEventID()
return env, nil
}
func parseRegistration(body []byte, versioned bool) (map[string]any, error) {
schemaVersion := "2011_2013"
manufacturerSize := 5
deviceTypeSize := 20
deviceIDSize := 7
if versioned {
schemaVersion = "2019"
manufacturerSize = 11
deviceTypeSize = 30
deviceIDSize = 30
}
minLen := 2 + 2 + manufacturerSize + deviceTypeSize + deviceIDSize + 1
if len(body) < minLen {
return nil, fmt.Errorf("%w: registration body len=%d", ErrFrameTooShort, len(body))
}
manufacturerStart := 4
deviceTypeStart := manufacturerStart + manufacturerSize
deviceIDStart := deviceTypeStart + deviceTypeSize
plateColorStart := deviceIDStart + deviceIDSize
registration := map[string]any{
"schema_version": schemaVersion,
"province": binary.BigEndian.Uint16(body[0:2]),
"city": binary.BigEndian.Uint16(body[2:4]),
"manufacturer": fixedText(body[manufacturerStart:deviceTypeStart]),
"device_type": fixedText(body[deviceTypeStart:deviceIDStart]),
"device_id": fixedText(body[deviceIDStart:plateColorStart]),
"plate_color": body[plateColorStart],
"plate": freeText(body[plateColorStart+1:]),
}
return registration, nil
}
func parseAuthentication(body []byte, versioned bool) map[string]any {
if versioned {
if out, ok := parseVersionedAuthentication(body); ok {
return out
}
if out, ok := parseDelimitedVersionedAuthentication(body); ok {
return out
}
}
return map[string]any{"token": freeText(body)}
}
func parseVersionedAuthentication(body []byte) (map[string]any, bool) {
if len(body) < 17 {
return nil, false
}
tokenLen := int(body[0])
imeiStart := 1 + tokenLen
softwareLenStart := imeiStart + 15
if tokenLen <= 0 || len(body) < softwareLenStart+1 {
return nil, false
}
softwareLen := int(body[softwareLenStart])
softwareStart := softwareLenStart + 1
if len(body) < softwareStart+softwareLen {
return nil, false
}
return map[string]any{
"schema_version": "2019",
"token": freeText(body[1:imeiStart]),
"imei": fixedText(body[imeiStart:softwareLenStart]),
"software_version": freeText(body[softwareStart : softwareStart+softwareLen]),
}, true
}
func parseDelimitedVersionedAuthentication(body []byte) (map[string]any, bool) {
parts := bytes.Split(body, []byte{0xff, 0xff})
if len(parts) < 2 {
return nil, false
}
out := map[string]any{
"schema_version": "2019_delimited",
"token": freeText(parts[0]),
"imei": freeText(parts[1]),
}
if len(parts) > 2 {
out["software_version"] = freeText(parts[2])
}
return out, true
}
func parseLocation(body []byte) (Location, error) {
if len(body) < 28 {
return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
}
status := binary.BigEndian.Uint32(body[4:8])
latitude := float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000
longitude := float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000
if status&(1<<2) != 0 {
latitude = -latitude
}
if status&(1<<3) != 0 {
longitude = -longitude
}
location := Location{
AlarmFlag: binary.BigEndian.Uint32(body[0:4]),
StatusFlag: status,
Latitude: latitude,
Longitude: longitude,
AltitudeM: binary.BigEndian.Uint16(body[16:18]),
SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10,
DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
Time: parseBCDTime(body[22:28]),
}
location.Additional = parseAdditional(body[28:], &location)
return location, nil
}
func parseAdditional(data []byte, location *Location) []AdditionalItem {
var out []AdditionalItem
for len(data) >= 2 {
id := data[0]
size := int(data[1])
data = data[2:]
if len(data) < size {
out = append(out, AdditionalItem{
ID: id,
Length: size,
ValueHex: strings.ToUpper(hex.EncodeToString(data)),
Parsed: "truncated",
})
return out
}
value := data[:size]
item := AdditionalItem{
ID: id,
Length: size,
ValueHex: strings.ToUpper(hex.EncodeToString(value)),
}
switch {
case id == 0x01 && size == 4:
rawMileage := binary.BigEndian.Uint32(value)
mileage := float64(rawMileage) / 10
location.TotalMileageKM = &mileage
item.Parsed = map[string]any{
"name": envelope.FieldTotalMileageKM,
"raw_value_tenth_km": rawMileage,
"value": mileage,
"unit": "km",
}
case id == 0x02 && size == 2:
rawFuel := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "fuel_l",
"raw_value_tenth_l": rawFuel,
"value": float64(rawFuel) / 10,
"unit": "L",
}
case id == 0x03 && size == 2:
rawSpeed := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "recorder_speed_kmh",
"raw_value_tenth_kmh": rawSpeed,
"value": float64(rawSpeed) / 10,
"unit": "km/h",
}
case id == 0x04 && size == 2:
item.Parsed = map[string]any{
"name": "manual_alarm_event_id",
"value": binary.BigEndian.Uint16(value),
}
case id == 0x05 && size == 30:
item.Parsed = map[string]any{
"name": "tire_pressure",
"values": parseTirePressure(value),
}
case id == 0x06 && size == 2:
item.Parsed = map[string]any{
"name": "carriage_temperature_c",
"value": int16(binary.BigEndian.Uint16(value)),
"unit": "C",
}
case id == 0x11 && (size == 1 || size == 5):
parsed := map[string]any{
"name": "overspeed_alarm",
"location_type": value[0],
}
if size == 5 {
parsed["area_id"] = binary.BigEndian.Uint32(value[1:5])
}
item.Parsed = parsed
case id == 0x12 && size == 6:
item.Parsed = map[string]any{
"name": "area_route_alarm",
"location_type": value[0],
"area_id": binary.BigEndian.Uint32(value[1:5]),
"direction": value[5],
}
case id == 0x13 && size == 7:
item.Parsed = map[string]any{
"name": "road_section_driving_time_alarm",
"road_section_id": binary.BigEndian.Uint32(value[0:4]),
"driving_time_seconds": binary.BigEndian.Uint16(value[4:6]),
"result": value[6],
"result_description": roadSectionDrivingTimeResult(value[6]),
"driving_time_result_source": "jt808_0x0200_additional_0x13",
}
case id == 0x25 && size == 4:
rawStatus := binary.BigEndian.Uint32(value)
item.Parsed = map[string]any{
"name": "extended_vehicle_signal_status",
"value": rawStatus,
"bits": parseExtendedVehicleSignal(rawStatus),
}
case id == 0x2a && size == 2:
rawStatus := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "io_status",
"value": rawStatus,
"bits": parseIOStatus(rawStatus),
}
case id == 0x2b && size == 4:
analog := binary.BigEndian.Uint32(value)
item.Parsed = map[string]any{
"name": "analog",
"ad0": analog & 0xffff,
"ad1": analog >> 16,
"value": analog,
}
case id == 0x30 && size == 1:
item.Parsed = map[string]any{
"name": "network_signal_strength",
"value": value[0],
}
case id == 0x31 && size == 1:
item.Parsed = map[string]any{
"name": "gnss_satellite_count",
"value": value[0],
}
}
out = append(out, item)
data = data[size:]
}
return out
}
func locationToMap(location Location) map[string]any {
out := map[string]any{
"alarm_flag": location.AlarmFlag,
"status_flag": location.StatusFlag,
"latitude": location.Latitude,
"longitude": location.Longitude,
"altitude_m": location.AltitudeM,
"speed_kmh": location.SpeedKMH,
"direction_deg": location.DirectionDeg,
"additional": additionalToMaps(location.Additional),
}
if !location.Time.IsZero() {
out["device_time"] = location.Time.Format(time.RFC3339)
}
if location.TotalMileageKM != nil {
out[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
}
return out
}
func additionalToMaps(items []AdditionalItem) []map[string]any {
out := make([]map[string]any, 0, len(items))
for _, item := range items {
value := map[string]any{
"id": fmt.Sprintf("0x%02X", item.ID),
"length": item.Length,
"value_hex": item.ValueHex,
}
if item.Parsed != nil {
value["parsed"] = item.Parsed
}
out = append(out, value)
}
return out
}
func parseTirePressure(data []byte) []map[string]any {
out := make([]map[string]any, 0, len(data))
for index, value := range data {
out = append(out, map[string]any{
"position": index,
"value": value,
"valid": value != 0xff,
})
}
return out
}
func parseExtendedVehicleSignal(value uint32) map[string]bool {
return map[string]bool{
"low_beam": value&0x0001 != 0,
"high_beam": value&0x0002 != 0,
"right_turn": value&0x0004 != 0,
"left_turn": value&0x0008 != 0,
"brake": value&0x0010 != 0,
"reverse_gear": value&0x0020 != 0,
"fog_light": value&0x0040 != 0,
"clearance_light": value&0x0080 != 0,
"horn": value&0x0100 != 0,
"air_conditioner": value&0x0200 != 0,
"neutral": value&0x0400 != 0,
"retarder": value&0x0800 != 0,
"abs": value&0x1000 != 0,
"heater": value&0x2000 != 0,
"clutch": value&0x4000 != 0,
}
}
func parseIOStatus(value uint16) map[string]bool {
return map[string]bool{
"deep_sleep": value&0x0001 != 0,
"sleep": value&0x0002 != 0,
}
}
func roadSectionDrivingTimeResult(value byte) string {
if value == 0 {
return "insufficient"
}
if value == 1 {
return "too_long"
}
return "unknown"
}
func bcdString(data []byte) string {
out := make([]byte, 0, len(data)*2)
for _, value := range data {
out = append(out, '0'+((value>>4)&0x0f), '0'+(value&0x0f))
}
return string(out)
}
func fixedText(data []byte) string {
return strings.Trim(freeText(data), "\x00 ")
}
func freeText(data []byte) string {
data = bytes.Trim(data, "\x00 ")
if len(data) == 0 {
return ""
}
if utf8.Valid(data) {
return string(data)
}
decoded, err := simplifiedchinese.GBK.NewDecoder().Bytes(data)
if err == nil && utf8.Valid(decoded) {
return string(decoded)
}
return strings.ToValidUTF8(string(data), "")
}
func parseBCDTime(data []byte) time.Time {
if len(data) != 6 {
return time.Time{}
}
year := 2000 + bcdByte(data[0])
month := time.Month(bcdByte(data[1]))
day := bcdByte(data[2])
hour := bcdByte(data[3])
minute := bcdByte(data[4])
second := bcdByte(data[5])
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
return time.Time{}
}
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
}
func bcdByte(value byte) int {
return int(value>>4)*10 + int(value&0x0f)
}
func unescape(data []byte) ([]byte, error) {
out := make([]byte, 0, len(data))
for i := 0; i < len(data); i++ {
if data[i] != 0x7d {
out = append(out, data[i])
continue
}
if i+1 >= len(data) {
return nil, ErrEscape
}
i++
switch data[i] {
case 0x01:
out = append(out, 0x7d)
case 0x02:
out = append(out, 0x7e)
default:
return nil, ErrEscape
}
}
return out, nil
}
func checksum(data []byte) byte {
var out byte
for _, value := range data {
out ^= value
}
return out
}

View File

@@ -0,0 +1,503 @@
package jt808
import (
"encoding/binary"
"encoding/hex"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E"
const fullAdditionalLocationFrame = "7e020000800123456789017fff000004000000080006eeb6ad02633df701380003006320070719235901040000000b02020016030200210402002c051e3737370000000000000000000000000000000000000000000000000000001105420000004212064d0000004d4d1307000000580058582504000000632a02000a2b040000001430011e31012806020001927e"
func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
if err != nil {
t.Fatal(err)
}
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 {
t.Fatalf("expected one complete frame, got %d", len(frames))
}
if hex.EncodeToString(remainder) != "7e02000000" {
t.Fatalf("expected partial remainder, got %s", hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782745114999, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Protocol != envelope.ProtocolJT808 {
t.Fatalf("protocol = %q", env.Protocol)
}
if env.MessageID != "0x0200" {
t.Fatalf("message id = %q", env.MessageID)
}
if env.Phone != "013307795425" {
t.Fatalf("phone = %q", env.Phone)
}
if env.Sequence != 1 {
t.Fatalf("sequence = %d", env.Sequence)
}
assertFloatField(t, env, envelope.FieldLatitude, 30.590151)
assertFloatField(t, env, envelope.FieldLongitude, 121.069881)
assertFloatField(t, env, envelope.FieldSpeedKMH, 23.0)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10241.2)
if env.Fields["direction_deg"] != uint16(79) {
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
}
location, ok := env.Parsed["location"].(map[string]any)
if !ok {
t.Fatalf("parsed location missing: %#v", env.Parsed)
}
additional, ok := location["additional"].([]map[string]any)
if !ok || len(additional) == 0 {
t.Fatalf("additional fields missing: %#v", location["additional"])
}
if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
t.Fatalf("unexpected first additional item: %#v", additional[0])
}
parsed, ok := additional[0]["parsed"].(map[string]any)
if !ok {
t.Fatalf("mileage parsed field missing: %#v", additional[0])
}
if parsed["raw_value_tenth_km"] != uint32(102412) || parsed["value"] != 10241.2 {
t.Fatalf("unexpected mileage parsed value: %#v", parsed)
}
}
func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
stream, err := hex.DecodeString(real2019LocationFrame)
if err != nil {
t.Fatal(err)
}
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782914967713, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x0200" {
t.Fatalf("message id = %q", env.MessageID)
}
if env.Phone != "14894135060" {
t.Fatalf("phone = %q", env.Phone)
}
if env.Sequence != 1995 {
t.Fatalf("sequence = %d", env.Sequence)
}
assertFloatField(t, env, envelope.FieldLatitude, 30.570155)
assertFloatField(t, env, envelope.FieldLongitude, 117.178187)
assertFloatField(t, env, envelope.FieldSpeedKMH, 33.4)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 7868.9)
if env.Fields["altitude_m"] != uint16(13) {
t.Fatalf("altitude = %#v", env.Fields["altitude_m"])
}
if env.Fields["direction_deg"] != uint16(232) {
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
}
header, ok := env.Parsed["header"].(map[string]any)
if !ok {
t.Fatalf("parsed header missing: %#v", env.Parsed)
}
if header["versioned"] != true || header["protocol_version"] != byte(1) {
t.Fatalf("unexpected versioned header: %#v", header)
}
if header["phone_bcd_hex"] != "00000000014894135060" {
t.Fatalf("phone bcd = %#v", header["phone_bcd_hex"])
}
}
func TestParseFrameParsesCommonLocationAdditionalItems(t *testing.T) {
stream, err := hex.DecodeString(fullAdditionalLocationFrame)
if err != nil {
t.Fatal(err)
}
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782914967713, "127.0.0.1:808")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Phone != "012345678901" {
t.Fatalf("phone = %q", env.Phone)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.1)
location := env.Parsed["location"].(map[string]any)
additional := location["additional"].([]map[string]any)
byID := map[string]map[string]any{}
for _, item := range additional {
byID[item["id"].(string)] = item["parsed"].(map[string]any)
}
if byID["0x01"]["raw_value_tenth_km"] != uint32(11) || byID["0x01"]["value"] != 1.1 {
t.Fatalf("mileage parsed = %#v", byID["0x01"])
}
if byID["0x05"]["name"] != "tire_pressure" {
t.Fatalf("tire pressure missing: %#v", byID["0x05"])
}
tireValues := byID["0x05"]["values"].([]map[string]any)
if tireValues[0]["value"] != byte(0x37) || tireValues[0]["valid"] != true {
t.Fatalf("tire pressure first value = %#v", tireValues[0])
}
if byID["0x11"]["location_type"] != byte(0x42) || byID["0x11"]["area_id"] != uint32(0x42) {
t.Fatalf("overspeed parsed = %#v", byID["0x11"])
}
if byID["0x12"]["location_type"] != byte(0x4d) || byID["0x12"]["area_id"] != uint32(0x4d) || byID["0x12"]["direction"] != byte(0x4d) {
t.Fatalf("area route parsed = %#v", byID["0x12"])
}
if byID["0x13"]["road_section_id"] != uint32(0x58) || byID["0x13"]["driving_time_seconds"] != uint16(0x58) {
t.Fatalf("road section parsed = %#v", byID["0x13"])
}
ioBits := byID["0x2A"]["bits"].(map[string]bool)
if ioBits["deep_sleep"] || !ioBits["sleep"] {
t.Fatalf("io bits = %#v", ioBits)
}
}
func TestParseFrameRejectsTruncatedSubpackageHeader(t *testing.T) {
payload := []byte{0x02, 0x00, 0x20, 0x00}
payload = append(payload, encodeBCD("013307795425", 6)...)
payload = binary.BigEndian.AppendUint16(payload, 1)
payload = append(payload, checksum(payload))
_, err := ParseFrame(payload, 1782914967713, "115.231.168.135:43625")
if err == nil {
t.Fatal("expected truncated subpackage header to fail")
}
}
func TestParseLocationAppliesSouthWestStatusBits(t *testing.T) {
body := make([]byte, 28)
binary.BigEndian.PutUint32(body[4:8], (1<<2)|(1<<3))
binary.BigEndian.PutUint32(body[8:12], 12345678)
binary.BigEndian.PutUint32(body[12:16], 98765432)
location, err := parseLocation(body)
if err != nil {
t.Fatalf("parseLocation() error = %v", err)
}
if location.Latitude != -12.345678 {
t.Fatalf("latitude = %v", location.Latitude)
}
if location.Longitude != -98.765432 {
t.Fatalf("longitude = %v", location.Longitude)
}
}
func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTNG", 5)
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
body = appendFixedASCII(body, "DEV0001", 7)
body = append(body, 2)
body = append(body, []byte("TEST123")...)
env, err := ParseFrame(buildFrame(0x0100, "013079963379", 7, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x0100" {
t.Fatalf("message id = %q", env.MessageID)
}
if env.Phone != "013079963379" {
t.Fatalf("phone = %q", env.Phone)
}
if env.DeviceID != "DEV0001" {
t.Fatalf("device id = %q", env.DeviceID)
}
if env.Plate != "TEST123" {
t.Fatalf("plate = %q", env.Plate)
}
registration, ok := env.Parsed["registration"].(map[string]any)
if !ok {
t.Fatalf("registration missing: %#v", env.Parsed)
}
if registration["manufacturer"] != "YUTNG" || registration["device_type"] != "ZK6105CHEVNPG4" {
t.Fatalf("registration fields = %#v", registration)
}
if registration["plate_color"] != uint8(2) {
t.Fatalf("plate color = %#v", registration["plate_color"])
}
}
func TestParseFrameParsesJT8082019RegistrationIdentity(t *testing.T) {
body := make([]byte, 0, 84)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTONG00001", 11)
body = appendFixedASCII(body, "ZK6105CHEVNPG4-2019", 30)
body = appendFixedASCII(body, "DEV201900000000000000000000001", 30)
body = append(body, 2)
body = append(body, []byte("TEST2019")...)
env, err := ParseFrame(buildVersionedFrame(0x0100, "14894135060", 11, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Phone != "14894135060" {
t.Fatalf("phone = %q", env.Phone)
}
if env.DeviceID != "DEV201900000000000000000000001" {
t.Fatalf("device id = %q", env.DeviceID)
}
if env.Plate != "TEST2019" {
t.Fatalf("plate = %q", env.Plate)
}
registration, ok := env.Parsed["registration"].(map[string]any)
if !ok {
t.Fatalf("registration missing: %#v", env.Parsed)
}
if registration["schema_version"] != "2019" || registration["manufacturer"] != "YUTONG00001" {
t.Fatalf("registration fields = %#v", registration)
}
}
func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTNG", 5)
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
body = appendFixedASCII(body, "DEV0002", 7)
body = append(body, 2)
body = append(body, []byte{0xBB, 0xA6, 'A', '5', '3', '3', '0', '1'}...)
env, err := ParseFrame(buildFrame(0x0100, "013079963380", 9, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Plate != "沪A53301" {
t.Fatalf("plate = %q", env.Plate)
}
}
func TestParseFrameParsesJT8082019Authentication(t *testing.T) {
body := []byte{5}
body = append(body, []byte("TOKEN")...)
body = appendFixedASCII(body, "123456789012345", 15)
body = append(body, 4)
body = append(body, []byte("v1.2")...)
env, err := ParseFrame(buildVersionedFrame(0x0102, "14894135060", 12, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
auth, ok := env.Parsed["authentication"].(map[string]any)
if !ok {
t.Fatalf("authentication missing: %#v", env.Parsed)
}
if auth["token"] != "TOKEN" || auth["imei"] != "123456789012345" || auth["software_version"] != "v1.2" {
t.Fatalf("authentication = %#v", auth)
}
}
func TestParseFrameParsesAuthenticationToken(t *testing.T) {
env, err := ParseFrame(buildFrame(0x0102, "064646848757", 8, []byte("g7gps")), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x0102" {
t.Fatalf("message id = %q", env.MessageID)
}
auth, ok := env.Parsed["authentication"].(map[string]any)
if !ok {
t.Fatalf("authentication missing: %#v", env.Parsed)
}
if auth["token"] != "g7gps" {
t.Fatalf("token = %#v", auth["token"])
}
}
func TestAutoResponderBuildsRegisterAck(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTNG", 5)
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
body = appendFixedASCII(body, "DEV0001", 7)
body = append(body, 2)
body = append(body, []byte("TEST123")...)
request := buildFrame(0x0100, "013079963379", 9, body)
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatal(err)
}
response, ok, err := NewAutoResponder("g7gps").Respond(request, env)
if err != nil {
t.Fatalf("Respond() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
frames, remainder, err := ExtractFrames(response)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
payload := frames[0]
if binary.BigEndian.Uint16(payload[0:2]) != 0x8100 {
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
}
if hex.EncodeToString(payload[4:10]) != "013079963379" {
t.Fatalf("phone bcd = %x", payload[4:10])
}
if binary.BigEndian.Uint16(payload[12:14]) != 9 || payload[14] != 0 || string(payload[15:20]) != "g7gps" {
t.Fatalf("unexpected register ack body: %x", payload[12:len(payload)-1])
}
}
func TestAutoResponderBuildsVersionedGeneralAck(t *testing.T) {
request := buildVersionedFrame(0x0200, "14894135060", 12, make([]byte, 28))
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatal(err)
}
response, ok, err := NewAutoResponder("").Respond(request, env)
if err != nil {
t.Fatalf("Respond() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
frames, remainder, err := ExtractFrames(response)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
payload := frames[0]
if binary.BigEndian.Uint16(payload[0:2]) != 0x8001 {
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
}
if binary.BigEndian.Uint16(payload[2:4])&0x4000 == 0 || payload[4] != 1 {
t.Fatalf("versioned header not preserved: %x", payload[:5])
}
if hex.EncodeToString(payload[5:15]) != "00000000014894135060" {
t.Fatalf("phone bcd = %x", payload[5:15])
}
bodyStart := 17
if binary.BigEndian.Uint16(payload[bodyStart:bodyStart+2]) != 12 ||
binary.BigEndian.Uint16(payload[bodyStart+2:bodyStart+4]) != 0x0200 ||
payload[bodyStart+4] != 0 {
t.Fatalf("unexpected general ack body: %x", payload[bodyStart:len(payload)-1])
}
}
func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
frame = append(frame, 0x7e)
frames, remainder, err := ExtractFrames(frame)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
if hex.EncodeToString(frames[0][:len(payload)]) != hex.EncodeToString(payload) {
t.Fatalf("frame was not unescaped: %s", hex.EncodeToString(frames[0]))
}
}
func buildFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
payload := make([]byte, 0, 12+len(body)+1)
payload = binary.BigEndian.AppendUint16(payload, messageID)
payload = binary.BigEndian.AppendUint16(payload, uint16(len(body)))
payload = append(payload, encodeBCD(phone, 6)...)
payload = binary.BigEndian.AppendUint16(payload, sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return payload
}
func buildVersionedFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
payload := make([]byte, 0, 17+len(body)+1)
payload = binary.BigEndian.AppendUint16(payload, messageID)
payload = binary.BigEndian.AppendUint16(payload, 0x4000|uint16(len(body)))
payload = append(payload, 1)
payload = append(payload, encodeBCD(phone, 10)...)
payload = binary.BigEndian.AppendUint16(payload, sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return payload
}
func appendFixedASCII(out []byte, value string, size int) []byte {
start := len(out)
out = append(out, make([]byte, size)...)
copy(out[start:], []byte(value))
return out
}
func encodeBCD(value string, size int) []byte {
out := make([]byte, size)
if len(value)%2 == 1 {
value = "0" + value
}
if len(value) > size*2 {
value = value[len(value)-size*2:]
}
for len(value) < size*2 {
value = "0" + value
}
for i := 0; i < size; i++ {
out[i] = (value[i*2]-'0')<<4 | (value[i*2+1] - '0')
}
return out
}
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
t.Helper()
got, ok := env.Fields[key].(float64)
if !ok {
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
}
if got != want {
t.Fatalf("field %s = %v, want %v", key, got, want)
}
}
func escape(payload []byte) []byte {
var out []byte
for _, value := range payload {
switch value {
case 0x7e:
out = append(out, 0x7d, 0x02)
case 0x7d:
out = append(out, 0x7d, 0x01)
default:
out = append(out, value)
}
}
return out
}

View File

@@ -0,0 +1,130 @@
package jt808
import (
"encoding/binary"
"errors"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrResponseHeaderTooShort = errors.New("jt808 response header too short")
const (
msgTerminalGeneralResponse = uint16(0x0001)
msgTerminalRegister = uint16(0x0100)
msgPlatformGeneralResponse = uint16(0x8001)
msgPlatformRegisterAck = uint16(0x8100)
)
type AutoResponder struct {
authCode string
}
func NewAutoResponder(authCode string) AutoResponder {
if authCode == "" {
authCode = "g7gps"
}
return AutoResponder{authCode: authCode}
}
func (r AutoResponder) Respond(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if env.ParseStatus == envelope.ParseBadFrame {
return nil, false, nil
}
header, err := parseResponseHeader(raw)
if err != nil {
return nil, false, err
}
if header.messageID == msgTerminalGeneralResponse {
return nil, false, nil
}
if header.messageID == msgTerminalRegister {
body := make([]byte, 3, 3+len(r.authCode))
binary.BigEndian.PutUint16(body[0:2], header.sequence)
body[2] = 0
body = append(body, []byte(r.authCode)...)
return encodeResponse(msgPlatformRegisterAck, header, body), true, nil
}
body := make([]byte, 5)
binary.BigEndian.PutUint16(body[0:2], header.sequence)
binary.BigEndian.PutUint16(body[2:4], header.messageID)
body[4] = 0
return encodeResponse(msgPlatformGeneralResponse, header, body), true, nil
}
type responseHeader struct {
messageID uint16
versioned bool
protocolVersion byte
phoneBCD []byte
sequence uint16
}
func parseResponseHeader(raw []byte) (responseHeader, error) {
if len(raw) < 12 {
return responseHeader{}, ErrResponseHeaderTooShort
}
props := binary.BigEndian.Uint16(raw[2:4])
versioned := props&0x4000 != 0
phoneStart := 4
phoneLen := 6
protocolVersion := byte(0)
if versioned {
if len(raw) < 17 {
return responseHeader{}, ErrResponseHeaderTooShort
}
protocolVersion = raw[4]
phoneStart = 5
phoneLen = 10
}
sequenceStart := phoneStart + phoneLen
if len(raw) < sequenceStart+2 {
return responseHeader{}, ErrResponseHeaderTooShort
}
phone := append([]byte(nil), raw[phoneStart:sequenceStart]...)
return responseHeader{
messageID: binary.BigEndian.Uint16(raw[0:2]),
versioned: versioned,
protocolVersion: protocolVersion,
phoneBCD: phone,
sequence: binary.BigEndian.Uint16(raw[sequenceStart : sequenceStart+2]),
}, nil
}
func encodeResponse(messageID uint16, header responseHeader, body []byte) []byte {
props := uint16(len(body)) & 0x03ff
if header.versioned {
props |= 0x4000
}
payload := make([]byte, 0, 4+1+len(header.phoneBCD)+2+len(body)+1)
payload = appendU16(payload, messageID)
payload = appendU16(payload, props)
if header.versioned {
payload = append(payload, header.protocolVersion)
}
payload = append(payload, header.phoneBCD...)
payload = appendU16(payload, header.sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return frame(payload)
}
func appendU16(out []byte, value uint16) []byte {
return append(out, byte(value>>8), byte(value))
}
func frame(payload []byte) []byte {
out := []byte{0x7e}
for _, value := range payload {
switch value {
case 0x7e:
out = append(out, 0x7d, 0x02)
case 0x7d:
out = append(out, 0x7d, 0x01)
default:
out = append(out, value)
}
}
out = append(out, 0x7e)
return out
}

View File

@@ -0,0 +1,201 @@
package yutongmqtt
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrInvalidJSON = errors.New("yutong mqtt invalid json")
func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS int64) (envelope.FrameEnvelope, error) {
root := map[string]any{}
decoder := json.NewDecoder(bytes.NewReader(payload))
decoder.UseNumber()
if err := decoder.Decode(&root); err != nil {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %v", ErrInvalidJSON, err)
}
data := objectValue(root, "data")
if data == nil {
data = root
}
deviceID := firstText(root, "device", "deviceId", "terminalId", "terminalID", "imei", "IMEI", "mqttDeviceId")
externalVIN := firstText(root, "vin", "VIN", "externalVin", "vehicleVin")
vin := externalVIN
if vin == "" && looksLikeVIN(deviceID) {
vin = deviceID
}
plate := firstText(root, "plateNo", "carNo", "plate", "vehicleNo")
phone := firstText(root, "sim", "phone", "mobile")
eventTimeMS := parseTimeMS(firstText(root, "time", "deviceTime", "eventTime", "gpsTime"), receivedAtMS)
fields := fieldsFromData(data)
parsed := map[string]any{
"endpoint": endpoint,
"topic": topic,
"root": root,
"data": data,
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
MessageID: "MQTT",
VIN: vin,
VehicleKeyHint: deviceID,
Phone: phone,
DeviceID: deviceID,
Plate: plate,
SourceEndpoint: "mqtt://" + endpoint + topic,
EventTimeMS: eventTimeMS,
ReceivedAtMS: receivedAtMS,
RawText: string(payload),
Parsed: parsed,
Fields: fields,
ParseStatus: envelope.ParseOK,
}
env.EventID = env.StableEventID()
return env, nil
}
func fieldsFromData(data map[string]any) map[string]any {
fields := map[string]any{}
if speed, ok := firstFloat(data, "METER_SPEED", "speed", "speed_kmh"); ok {
fields[envelope.FieldSpeedKMH] = speed
}
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
fields[envelope.FieldTotalMileageKM] = normalizeTotalMileageKM(mileage)
}
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
fields[envelope.FieldSOCPercent] = soc
}
if longitude, ok := firstCoord(data, "LONGITUDE", "longitude", "lng"); ok {
fields[envelope.FieldLongitude] = longitude
}
if latitude, ok := firstCoord(data, "LATITUDE", "latitude", "lat"); ok {
fields[envelope.FieldLatitude] = latitude
}
if direction, ok := firstFloat(data, "GPSDirection", "direction", "direction_deg"); ok {
fields["direction_deg"] = direction
}
if gear, ok := firstFloat(data, "ON_GEAR", "gear"); ok {
fields["gear"] = gear
}
return fields
}
func objectValue(root map[string]any, key string) map[string]any {
value, ok := root[key]
if !ok {
return nil
}
typed, ok := value.(map[string]any)
if !ok {
return nil
}
return typed
}
func firstText(root map[string]any, keys ...string) string {
for _, key := range keys {
if value, ok := root[key]; ok && value != nil {
text := strings.TrimSpace(fmt.Sprint(value))
if text != "" && text != "<nil>" {
return text
}
}
}
return ""
}
func firstFloat(root map[string]any, keys ...string) (float64, bool) {
for _, key := range keys {
value, ok := root[key]
if !ok || value == nil {
continue
}
if parsed, ok := toFloat(value); ok {
return parsed, true
}
}
return 0, false
}
func firstCoord(root map[string]any, keys ...string) (float64, bool) {
value, ok := firstFloat(root, keys...)
if !ok {
return 0, false
}
if value == 0 || int64(value) == 999999 {
return 0, false
}
if value > 1000 || value < -1000 {
value = value / 1_000_000
}
return value, true
}
func toFloat(value any) (float64, bool) {
switch typed := value.(type) {
case json.Number:
parsed, err := typed.Float64()
return parsed, err == nil
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int64:
return float64(typed), true
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return parsed, err == nil
default:
return 0, false
}
}
func parseTimeMS(raw string, fallback int64) int64 {
raw = strings.TrimSpace(raw)
if raw == "" {
return fallback
}
for _, layout := range []string{
"2006-01-02 15:04:05.000",
"2006-01-02 15:04:05",
} {
if parsed, err := time.ParseInLocation(layout, raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
return parsed.UnixMilli()
}
}
if len(raw) == 14 {
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
return parsed.UnixMilli()
}
}
if parsed, err := time.Parse(time.RFC3339, raw); err == nil {
return parsed.UnixMilli()
}
if epoch, err := strconv.ParseInt(raw, 10, 64); err == nil {
if epoch > 10_000_000_000 {
return epoch
}
return epoch * 1000
}
return fallback
}
func looksLikeVIN(value string) bool {
value = strings.TrimSpace(value)
return len(value) == 17
}
func normalizeTotalMileageKM(value float64) float64 {
return value / 1000
}

View File

@@ -0,0 +1,112 @@
package yutongmqtt
import (
"errors"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
payload := []byte(`{
"device": "LTEST000000000001",
"time": "20260413100000",
"plateNo": "豫A12345",
"data": {
"METER_SPEED": 52.3,
"TOTAL_MILEAGE": 123456.7,
"BATTERY_CAPACITY_SOC": 70,
"LONGITUDE": 116397128,
"LATITUDE": 39916527,
"GPSDirection": 88
}
}`)
env, err := ParseMessage("endpoint-a", "/ytforward/shln/dev1", payload, 1782745114999)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if env.Protocol != envelope.ProtocolYutongMQTT {
t.Fatalf("protocol = %q", env.Protocol)
}
if env.VIN != "LTEST000000000001" || env.DeviceID != "LTEST000000000001" {
t.Fatalf("identity fields = vin:%q device:%q", env.VIN, env.DeviceID)
}
if env.Plate != "豫A12345" {
t.Fatalf("plate = %q", env.Plate)
}
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123.4567)
assertFloatField(t, env, envelope.FieldSOCPercent, 70)
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
assertFloatField(t, env, "direction_deg", 88)
if env.EventTimeMS == 1782745114999 {
t.Fatal("event time should come from device time")
}
if env.RawText == "" || env.Parsed["data"] == nil {
t.Fatalf("raw/parsed missing: %#v", env)
}
}
func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
payload := []byte(`{"device":"YT-DEVICE-001","time":"20260413100000","data":{"METER_SPEED":"12.3"}}`)
env, err := ParseMessage("endpoint-a", "/ytforward/shln/YT-DEVICE-001", payload, 1782745114999)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if env.VIN != "" {
t.Fatalf("non-vin device should not be treated as vin: %q", env.VIN)
}
if env.VehicleKey() != "YT-DEVICE-001" {
t.Fatalf("vehicle key = %q", env.VehicleKey())
}
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
}
func TestParseMessageNormalizesProductionMileageMeters(t *testing.T) {
payload := []byte(`{
"device":"LMRKH9AC3R1004101",
"time":"2026-07-02 01:02:04.085",
"data":{"TOTAL_MILEAGE":56905000,"METER_SPEED":19}
}`)
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 56905)
if env.EventTimeMS != 1782925324085 {
t.Fatalf("event time = %d, want 1782925324085", env.EventTimeMS)
}
}
func TestParseMessageTreatsSmallYutongMileageAsMeters(t *testing.T) {
payload := []byte(`{
"device":"LMRKH9AC3R1004101",
"time":"2026-07-02 01:02:04.085",
"data":{"TOTAL_MILEAGE":1024,"METER_SPEED":19}
}`)
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.024)
}
func TestParseMessageRejectsMalformedJSON(t *testing.T) {
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
if !errors.Is(err, ErrInvalidJSON) {
t.Fatalf("error = %v", err)
}
}
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
t.Helper()
got, ok := env.Fields[key].(float64)
if !ok {
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
}
if got != want {
t.Fatalf("field %s = %v, want %v", key, got, want)
}
}

View File

@@ -0,0 +1,69 @@
package realtime
import (
"encoding/json"
"errors"
"net/http"
"strings"
"github.com/redis/go-redis/v9"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Handler struct {
repository *Repository
}
func NewHandler(repository *Repository) *Handler {
if repository == nil {
panic("realtime repository must not be nil")
}
return &Handler{repository: repository}
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
path := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/realtime/vehicles/"), "/")
parts := strings.Split(path, "/")
if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" {
writeError(w, http.StatusNotFound, "vehicle not found")
return
}
vin := parts[0]
switch {
case len(parts) == 1:
snapshot, err := h.repository.GetMerged(r.Context(), vin)
writeResult(w, snapshot, err)
case len(parts) == 2 && parts[1] == "online":
status, err := h.repository.IsOnline(r.Context(), vin)
writeResult(w, status, err)
case len(parts) == 3 && parts[1] == "protocols":
snapshot, err := h.repository.GetProtocol(r.Context(), vin, envelope.Protocol(strings.ToUpper(parts[2])))
writeResult(w, snapshot, err)
default:
writeError(w, http.StatusNotFound, "route not found")
}
}
func writeResult(w http.ResponseWriter, value any, err error) {
if err != nil {
if errors.Is(err, redis.Nil) {
writeError(w, http.StatusNotFound, "not found")
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(value)
}
func writeError(w http.ResponseWriter, status int, message string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
}

View File

@@ -0,0 +1,41 @@
package realtime
import (
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Snapshot struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Protocol envelope.Protocol `json:"protocol,omitempty"`
Protocols []envelope.Protocol `json:"protocols,omitempty"`
EventID string `json:"event_id,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
UpdatedAtMS int64 `json:"updated_at_ms"`
}
type OnlineStatus struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Online bool `json:"online"`
LastSeenMS int64 `json:"last_seen_ms"`
Protocols []envelope.Protocol `json:"protocols,omitempty"`
TTLSeconds int64 `json:"ttl_seconds"`
}
type Config struct {
OnlineTTL time.Duration
}
func (c Config) ttl() time.Duration {
if c.OnlineTTL <= 0 {
return 10 * time.Minute
}
return c.OnlineTTL
}

View File

@@ -0,0 +1,232 @@
package realtime
import (
"context"
"encoding/json"
"errors"
"sort"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Repository struct {
client *redis.Client
cfg Config
}
func NewRepository(client *redis.Client, cfg Config) *Repository {
if client == nil {
panic("redis client must not be nil")
}
return &Repository{client: client, cfg: cfg}
}
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
vin := strings.TrimSpace(env.VIN)
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil
}
nowMS := time.Now().UnixMilli()
eventMS := env.EventTimeMS
if eventMS <= 0 {
eventMS = env.ReceivedAtMS
}
protocolSnapshot := Snapshot{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
EventID: env.StableEventID(),
EventTimeMS: eventMS,
ReceivedAtMS: env.ReceivedAtMS,
SourceEndpoint: env.SourceEndpoint,
Fields: cloneFields(env.Fields),
FieldTimesMS: fieldTimes(env.Fields, eventMS),
UpdatedAtMS: nowMS,
}
if err := r.setJSON(ctx, protocolKey(vehicleKey, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
return err
}
protocols, err := r.addProtocol(ctx, vehicleKey, env.Protocol)
if err != nil {
return err
}
merged, err := r.GetMerged(ctx, vehicleKey)
if err != nil && !errors.Is(err, redis.Nil) {
return err
}
if merged.VehicleKey == "" {
merged = Snapshot{VehicleKey: vehicleKey, VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
} else if merged.VIN == "" && vin != "" {
merged.VIN = vin
}
mergeFields(&merged, env.Fields, eventMS)
if eventMS >= merged.EventTimeMS {
merged.EventTimeMS = eventMS
merged.EventID = env.StableEventID()
merged.ReceivedAtMS = env.ReceivedAtMS
merged.SourceEndpoint = env.SourceEndpoint
}
merged.Protocols = protocols
merged.UpdatedAtMS = nowMS
if err := r.setJSON(ctx, mergedKey(vehicleKey), merged, r.cfg.ttl()); err != nil {
return err
}
online := OnlineStatus{
VehicleKey: vehicleKey,
VIN: vin,
Online: true,
LastSeenMS: env.ReceivedAtMS,
Protocols: protocols,
TTLSeconds: int64(r.cfg.ttl().Seconds()),
}
if err := r.setJSON(ctx, onlineKey(vehicleKey), online, r.cfg.ttl()); err != nil {
return err
}
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vehicleKey}).Err()
}
func (r *Repository) GetMerged(ctx context.Context, vehicleKey string) (Snapshot, error) {
return r.getSnapshot(ctx, mergedKey(vehicleKey))
}
func (r *Repository) GetProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) (Snapshot, error) {
return r.getSnapshot(ctx, protocolKey(vehicleKey, protocol))
}
func (r *Repository) IsOnline(ctx context.Context, vehicleKey string) (OnlineStatus, error) {
vehicleKey = strings.TrimSpace(vehicleKey)
var status OnlineStatus
payload, err := r.client.Get(ctx, onlineKey(vehicleKey)).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}, nil
}
return status, err
}
if err := json.Unmarshal(payload, &status); err != nil {
return status, err
}
ttl := r.client.TTL(ctx, onlineKey(vehicleKey)).Val()
status.Online = ttl > 0
status.TTLSeconds = int64(ttl.Seconds())
return status, nil
}
func (r *Repository) getSnapshot(ctx context.Context, key string) (Snapshot, error) {
var snapshot Snapshot
payload, err := r.client.Get(ctx, key).Bytes()
if err != nil {
return snapshot, err
}
return snapshot, json.Unmarshal(payload, &snapshot)
}
func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl time.Duration) error {
payload, err := json.Marshal(value)
if err != nil {
return err
}
return r.client.Set(ctx, key, payload, ttl).Err()
}
func (r *Repository) addProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
key := protocolsKey(vehicleKey)
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
return nil, err
}
_ = r.client.Expire(ctx, key, r.cfg.ttl()).Err()
values, err := r.client.SMembers(ctx, key).Result()
if err != nil {
return nil, err
}
sort.Strings(values)
out := make([]envelope.Protocol, 0, len(values))
for _, value := range values {
out = append(out, envelope.Protocol(value))
}
return out, nil
}
func mergeFields(snapshot *Snapshot, fields map[string]any, eventMS int64) {
if snapshot.Fields == nil {
snapshot.Fields = map[string]any{}
}
if snapshot.FieldTimesMS == nil {
snapshot.FieldTimesMS = map[string]int64{}
}
for key, value := range fields {
if key == envelope.FieldTotalMileageKM && !positiveNumber(value) {
continue
}
if eventMS >= snapshot.FieldTimesMS[key] {
snapshot.Fields[key] = value
snapshot.FieldTimesMS[key] = eventMS
}
}
}
func positiveNumber(value any) bool {
switch typed := value.(type) {
case float64:
return typed > 0
case float32:
return typed > 0
case int:
return typed > 0
case int64:
return typed > 0
case uint16:
return typed > 0
case uint32:
return typed > 0
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return err == nil && parsed > 0
default:
return false
}
}
func cloneFields(fields map[string]any) map[string]any {
if len(fields) == 0 {
return map[string]any{}
}
out := make(map[string]any, len(fields))
for key, value := range fields {
out[key] = value
}
return out
}
func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
out := make(map[string]int64, len(fields))
for key := range fields {
out[key] = eventMS
}
return out
}
func mergedKey(vehicleKey string) string {
return "vehicle:latest:" + strings.TrimSpace(vehicleKey)
}
func protocolKey(vehicleKey string, protocol envelope.Protocol) string {
return "vehicle:latest:" + strings.TrimSpace(vehicleKey) + ":" + string(protocol)
}
func onlineKey(vehicleKey string) string {
return "vehicle:online:" + strings.TrimSpace(vehicleKey)
}
func protocolsKey(vehicleKey string) string {
return "vehicle:protocols:" + strings.TrimSpace(vehicleKey)
}

View File

@@ -0,0 +1,250 @@
package realtime
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{
envelope.FieldLongitude: 121.1,
envelope.FieldTotalMileageKM: 10.5,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "VIN001",
EventTimeMS: 900,
ReceivedAtMS: 1200,
Fields: map[string]any{
envelope.FieldLongitude: 120.0,
envelope.FieldLatitude: 30.5,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
merged, err := repo.GetMerged(ctx, "VIN001")
if err != nil {
t.Fatalf("GetMerged() error = %v", err)
}
if merged.Fields[envelope.FieldLongitude] != 121.1 {
t.Fatalf("older longitude overwrote newer value: %#v", merged.Fields)
}
if merged.Fields[envelope.FieldLatitude] != 30.5 {
t.Fatalf("new latitude missing: %#v", merged.Fields)
}
if got := protocolNames(merged.Protocols); strings.Join(got, ",") != "GB32960,JT808" {
t.Fatalf("merged protocols = %#v", got)
}
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
if err != nil {
t.Fatalf("GetProtocol() error = %v", err)
}
if protocol.Fields[envelope.FieldTotalMileageKM] != 10.5 {
t.Fatalf("protocol snapshot = %#v", protocol)
}
}
func TestRepositoryOnlineStatus(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
status, err := repo.IsOnline(ctx, "VIN001")
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if status.Online {
t.Fatal("empty vin should be offline")
}
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
status, err = repo.IsOnline(ctx, "VIN001")
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if !status.Online || status.LastSeenMS != 1100 {
t.Fatalf("online status = %#v", status)
}
}
func TestRepositoryUpdatesPhoneOnlyVehicleKey(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811170",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10003.7,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
vehicleKey := "JT808:013307811170"
merged, err := repo.GetMerged(ctx, vehicleKey)
if err != nil {
t.Fatalf("GetMerged() error = %v", err)
}
if merged.VehicleKey != vehicleKey || merged.VIN != "" {
t.Fatalf("unexpected identity: %#v", merged)
}
if merged.Fields[envelope.FieldTotalMileageKM] != 10003.7 {
t.Fatalf("merged fields = %#v", merged.Fields)
}
status, err := repo.IsOnline(ctx, vehicleKey)
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if !status.Online || status.VehicleKey != vehicleKey {
t.Fatalf("online status = %#v", status)
}
}
func TestRepositoryDoesNotOverwritePositiveMileageWithZero(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{
envelope.FieldTotalMileageKM: 12345.6,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 2000,
ReceivedAtMS: 2100,
Fields: map[string]any{
envelope.FieldSpeedKMH: 22.0,
envelope.FieldTotalMileageKM: 0,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
merged, err := repo.GetMerged(ctx, "VIN001")
if err != nil {
t.Fatalf("GetMerged() error = %v", err)
}
if merged.Fields[envelope.FieldTotalMileageKM] != 12345.6 {
t.Fatalf("zero mileage overwrote positive value: %#v", merged.Fields)
}
if merged.Fields[envelope.FieldSpeedKMH] != 22.0 {
t.Fatalf("new speed should still merge: %#v", merged.Fields)
}
}
func TestHandlerReturnsMergedSnapshot(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/VIN001", nil)
rec := httptest.NewRecorder()
NewHandler(repo).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
if !stringsContains(rec.Body.String(), `"vin":"VIN001"`) {
t.Fatalf("unexpected body: %s", rec.Body.String())
}
}
func TestHandlerReturnsPhoneOnlyVehicleKeySnapshot(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811170",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/JT808:013307811170", nil)
rec := httptest.NewRecorder()
NewHandler(repo).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
if !stringsContains(rec.Body.String(), `"vehicle_key":"JT808:013307811170"`) {
t.Fatalf("unexpected body: %s", rec.Body.String())
}
}
func newTestRepository(t *testing.T) (*Repository, func()) {
t.Helper()
server, err := miniredis.Run()
if err != nil {
t.Fatalf("miniredis.Run() error = %v", err)
}
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
return NewRepository(client, Config{OnlineTTL: time.Minute}), func() {
_ = client.Close()
server.Close()
}
}
func stringsContains(value string, pattern string) bool {
return strings.Contains(value, pattern)
}
func protocolNames(protocols []envelope.Protocol) []string {
names := make([]string, 0, len(protocols))
for _, protocol := range protocols {
names = append(names, string(protocol))
}
return names
}

View File

@@ -0,0 +1,275 @@
package stats
import (
"context"
"database/sql"
"errors"
"strconv"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
const (
MetricDailyMileageKM = "daily_mileage_km"
MetricDailyTotalMileageKM = "daily_total_mileage_km"
)
type Execer interface {
ExecContext(context.Context, string, ...any) (sql.Result, error)
}
type Writer struct {
exec Execer
loc *time.Location
}
type MetricSample struct {
VehicleKey string
VIN string
Protocol envelope.Protocol
StatDate string
MetricKey string
MetricValue float64
TotalMileageKM float64
}
func NewWriter(exec Execer, loc *time.Location) *Writer {
if exec == nil {
panic("stats execer must not be nil")
}
if loc == nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
return &Writer{exec: exec, loc: loc}
}
func (w *Writer) EnsureSchema(ctx context.Context) error {
if _, err := w.exec.ExecContext(ctx, DailyMetricTableSQL); err != nil {
return err
}
if db, ok := w.exec.(metadataDB); ok {
return migrateDailyMetricTable(ctx, db)
}
return nil
}
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
samples, err := SamplesFromEnvelope(env, w.loc)
if err != nil {
return err
}
for _, sample := range samples {
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL,
sample.VehicleKey,
sample.VIN,
sample.StatDate,
string(sample.Protocol),
sample.MetricKey,
sample.MetricValue,
sample.TotalMileageKM,
sample.TotalMileageKM); err != nil {
return err
}
}
return nil
}
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
vin := strings.TrimSpace(env.VIN)
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil, nil
}
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
if !ok {
return nil, nil
}
if totalMileage <= 0 {
return nil, nil
}
if loc == nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
eventMS := env.EventTimeMS
if eventMS <= 0 {
eventMS = env.ReceivedAtMS
}
if eventMS <= 0 {
return nil, errors.New("event or received time is required")
}
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
return []MetricSample{
{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
StatDate: statDate,
MetricKey: MetricDailyMileageKM,
MetricValue: 0,
TotalMileageKM: totalMileage,
},
{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
StatDate: statDate,
MetricKey: MetricDailyTotalMileageKM,
MetricValue: totalMileage,
TotalMileageKM: totalMileage,
},
}, nil
}
const upsertDailyMetricSQL = `
INSERT INTO vehicle_daily_metric
(vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit,
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
VALUES (?, ?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
ON DUPLICATE KEY UPDATE
vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
first_total_mileage_km = CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END,
latest_total_mileage_km = CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
END,
metric_value = CASE
WHEN metric_key = 'daily_mileage_km'
THEN GREATEST(
CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE latest_total_mileage_km
END,
VALUES(latest_total_mileage_km)
)
- CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END
WHEN metric_key = 'daily_total_mileage_km'
THEN GREATEST(
CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE latest_total_mileage_km
END,
VALUES(latest_total_mileage_km)
)
ELSE VALUES(metric_value)
END,
sample_count = sample_count + 1,
updated_at = CURRENT_TIMESTAMP
`
type metadataDB interface {
Execer
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func migrateDailyMetricTable(ctx context.Context, db metadataDB) error {
columnExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND column_name = 'vehicle_key'`)
if err != nil {
return err
}
if !columnExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD COLUMN vehicle_key VARCHAR(96) NOT NULL DEFAULT '' AFTER id`); err != nil {
return err
}
}
if _, err := db.ExecContext(ctx, `UPDATE vehicle_daily_metric SET vehicle_key = vin WHERE vehicle_key = ''`); err != nil {
return err
}
vehicleIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric_vehicle'`)
if err != nil {
return err
}
if !vehicleIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key)`); err != nil {
return err
}
}
oldIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric'`)
if err != nil {
return err
}
if oldIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric DROP INDEX uk_daily_metric`); err != nil {
return err
}
}
vinIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'idx_vin'`)
if err != nil {
return err
}
if !vinIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD KEY idx_vin (vin)`); err != nil {
return err
}
}
return nil
}
func informationSchemaExists(ctx context.Context, db metadataDB, query string) (bool, error) {
var count int
if err := db.QueryRowContext(ctx, query).Scan(&count); err != nil {
return false, err
}
return count > 0, nil
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
if env.Fields == nil {
return 0, false
}
value, ok := env.Fields[key]
if !ok || value == nil {
return 0, false
}
switch typed := value.(type) {
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int64:
return float64(typed), true
case uint16:
return float64(typed), true
case uint32:
return float64(typed), true
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return parsed, err == nil
default:
return 0, false
}
}

View File

@@ -0,0 +1,162 @@
package stats
import (
"context"
"database/sql"
"strings"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "LNBVIN00000000001",
EventTimeMS: time.Date(2026, 7, 1, 8, 30, 0, 0, loc).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: 10241.2,
},
}
samples, err := SamplesFromEnvelope(env, loc)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 2 {
t.Fatalf("sample count = %d", len(samples))
}
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
}
if samples[0].VehicleKey != "LNBVIN00000000001" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 {
t.Fatalf("unexpected daily total sample: %#v", samples[1])
}
if samples[0].StatDate != "2026-07-01" {
t.Fatalf("stat date = %q", samples[0].StatDate)
}
}
func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811254",
EventTimeMS: time.Date(2026, 7, 2, 0, 3, 0, 0, loc).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: 10985.7,
},
}, loc)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 2 {
t.Fatalf("sample count = %d", len(samples))
}
if samples[0].VehicleKey != "JT808:013307811254" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[0].VIN != "" {
t.Fatalf("vin should stay empty for unresolved JT808 identity, got %q", samples[0].VIN)
}
}
func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
}, nil)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 0 {
t.Fatalf("expected no samples without vehicle key, got %#v", samples)
}
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "LNBVIN00000000002",
Fields: map[string]any{},
}, nil)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 0 {
t.Fatalf("expected no samples without mileage, got %#v", samples)
}
}
func TestSamplesFromEnvelopeSkipsNonPositiveMileage(t *testing.T) {
for _, value := range []any{0, 0.0, -1.0, "0"} {
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "LNBVIN00000000001",
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: value,
},
}, nil)
if err != nil {
t.Fatalf("SamplesFromEnvelope(%#v) error = %v", value, err)
}
if len(samples) != 0 {
t.Fatalf("expected no samples for non-positive mileage %#v, got %#v", value, samples)
}
}
}
func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
if err := writer.EnsureSchema(context.Background()); err != nil {
t.Fatalf("EnsureSchema() error = %v", err)
}
if err := writer.Append(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "LNBVIN00000000002",
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: "10000.0",
},
}); err != nil {
t.Fatalf("Append() error = %v", err)
}
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") {
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
}
if !strings.Contains(exec.calls[0].query, "vehicle_key") {
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
}
if len(exec.calls) != 3 {
t.Fatalf("exec calls = %d", len(exec.calls))
}
if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query)
}
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
}
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
t.Fatalf("first upsert arg should be vehicle_key, got %#v", got)
}
}
type execCall struct {
query string
args []any
}
type recordingExec struct {
calls []execCall
}
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
e.calls = append(e.calls, execCall{query: query, args: args})
return nil, nil
}

View File

@@ -0,0 +1,326 @@
package stats
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
)
type Queryer interface {
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
}
type MetricQuery struct {
VehicleKey string
VIN string
Protocol string
MetricKey string
DateFrom string
DateTo string
Limit int
Offset int
}
type MetricRow struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
MetricKey string `json:"metric_key"`
MetricValue float64 `json:"metric_value"`
MetricUnit string `json:"metric_unit"`
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
SampleCount int64 `json:"sample_count"`
CalculationMethod string `json:"calculation_method"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type MetricRepository struct {
db Queryer
}
func NewMetricRepository(db Queryer) *MetricRepository {
if db == nil {
panic("metric query db must not be nil")
}
return &MetricRepository{db: db}
}
func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]MetricRow, error) {
query = normalizeMetricQuery(query)
sqlText, args := buildMetricSQL(query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]MetricRow, 0)
for rows.Next() {
var row MetricRow
var statDate scanDate
var createdAt scanDateTime
var updatedAt scanDateTime
var first sql.NullFloat64
var latest sql.NullFloat64
if err := rows.Scan(
&row.VehicleKey,
&row.VIN,
&statDate,
&row.Protocol,
&row.MetricKey,
&row.MetricValue,
&row.MetricUnit,
&first,
&latest,
&row.SampleCount,
&row.CalculationMethod,
&createdAt,
&updatedAt,
); err != nil {
return nil, err
}
row.StatDate = statDate.String
row.CreatedAt = createdAt.String
row.UpdatedAt = updatedAt.String
if first.Valid {
row.FirstTotalMileageKM = &first.Float64
}
if latest.Valid {
row.LatestTotalMileageKM = &latest.Float64
}
out = append(out, row)
}
return out, rows.Err()
}
func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, error) {
query = normalizeMetricQuery(query)
sqlText, args := buildMetricCountSQL(query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return 0, err
}
defer rows.Close()
var total int64
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, rows.Err()
}
func normalizeMetricQuery(query MetricQuery) MetricQuery {
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.MetricKey = strings.TrimSpace(query.MetricKey)
query.DateFrom = strings.TrimSpace(query.DateFrom)
query.DateTo = strings.TrimSpace(query.DateTo)
if query.Limit <= 0 {
query.Limit = 50
}
return query
}
func buildMetricSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
args = append(args, query.Limit, query.Offset)
return sqlText, args
}
func buildMetricCountSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT COUNT(*) FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, args
}
func buildMetricWhere(query MetricQuery) ([]string, []any) {
var where []string
var args []any
add := func(clause string, value any) {
where = append(where, clause)
args = append(args, value)
}
if query.VIN != "" {
add("vin = ?", query.VIN)
}
if query.VehicleKey != "" {
add("vehicle_key = ?", query.VehicleKey)
}
if query.Protocol != "" {
add("protocol = ?", query.Protocol)
}
if query.MetricKey != "" {
add("metric_key = ?", query.MetricKey)
}
if query.DateFrom != "" {
add("stat_date >= ?", query.DateFrom)
}
if query.DateTo != "" {
add("stat_date <= ?", query.DateTo)
}
return where, args
}
type MetricHandler struct {
repository *MetricRepository
}
func NewMetricHandler(repository *MetricRepository) *MetricHandler {
if repository == nil {
panic("metric repository must not be nil")
}
return &MetricHandler{repository: repository}
}
func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeMetricError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/stats/daily-metrics" {
writeMetricError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseMetricQuery(r)
if err != nil {
writeMetricError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeMetricError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeMetricError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func parseMetricQuery(r *http.Request) (MetricQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit")
if err != nil {
return MetricQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return MetricQuery{}, err
}
query := MetricQuery{
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Protocol: values.Get("protocol"),
MetricKey: values.Get("metricKey"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDate(query.DateFrom) || !validDate(query.DateTo) {
return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD")
}
return normalizeMetricQuery(query), nil
}
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return fallback, nil
}
value, err := strconv.Atoi(raw)
if err != nil || value < min || value > max {
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
}
return value, nil
}
func validDate(value string) bool {
value = strings.TrimSpace(value)
if value == "" {
return true
}
if len(value) != len("2006-01-02") {
return false
}
for i, r := range value {
switch i {
case 4, 7:
if r != '-' {
return false
}
default:
if r < '0' || r > '9' {
return false
}
}
}
return true
}
func writeMetricError(w http.ResponseWriter, status int, message string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
}
type scanDate struct {
String string
}
func (s *scanDate) Scan(value any) error {
s.String = formatSQLTime(value, "2006-01-02")
return nil
}
type scanDateTime struct {
String string
}
func (s *scanDateTime) Scan(value any) error {
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
return nil
}
func formatSQLTime(value any, layout string) string {
switch typed := value.(type) {
case nil:
return ""
case time.Time:
return typed.Format(layout)
case []byte:
return strings.TrimSpace(string(typed))
case string:
return strings.TrimSpace(typed)
default:
return strings.TrimSpace(fmt.Sprint(typed))
}
}

View File

@@ -0,0 +1,185 @@
package stats
import (
"context"
"database/sql"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
)
func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km",
12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
))
repository := NewMetricRepository(db)
rows, err := repository.Query(context.Background(), MetricQuery{
VIN: "LKLG7C4E3NA774736",
Protocol: "JT808",
DateFrom: "2026-07-01",
DateTo: "2026-07-01",
Limit: 20,
})
if err != nil {
t.Fatalf("Query() error = %v", err)
}
if len(rows) != 1 {
t.Fatalf("row count = %d", len(rows))
}
if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 {
t.Fatalf("unexpected row: %#v", rows[0])
}
if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
t.Fatalf("vehicle key = %q", rows[0].VehicleKey)
}
if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" {
t.Fatalf("unexpected time formatting: %#v", rows[0])
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km",
53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25",
))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":42`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("JT808:013307811254", "JT808").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("JT808:013307811254", "JT808", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"JT808:013307811254", "", "2026-07-02", "JT808", MetricDailyTotalMileageKM, 10985.7, "km",
10985.7, 10985.7, 1, "TOTAL_MILEAGE_DIFF", "2026-07-02 00:03:00", "2026-07-02 00:03:00",
))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vehicleKey=JT808:013307811254&protocol=JT808", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"metric_value":10985.7`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("YUTONG_MQTT").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("YUTONG_MQTT", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?protocol=YUTONG_MQTT", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
t.Fatalf("expected empty items array, got: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerRejectsInvalidPagination(t *testing.T) {
handler := NewMetricHandler(NewMetricRepository(&sql.DB{}))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?limit=2001", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
}

View File

@@ -0,0 +1,22 @@
package stats
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vehicle_key VARCHAR(96) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '',
stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL,
metric_key VARCHAR(64) NOT NULL,
metric_value DECIMAL(18,3) NOT NULL,
metric_unit VARCHAR(16) NOT NULL,
first_total_mileage_km DECIMAL(18,3) NULL,
latest_total_mileage_km DECIMAL(18,3) NULL,
sample_count BIGINT NOT NULL DEFAULT 0,
calculation_method VARCHAR(64) NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key),
KEY idx_vin (vin),
KEY idx_stat_date (stat_date),
KEY idx_protocol_metric (protocol, metric_key)
)`

162
tools/go_kafka_prod_smoke.py Executable file
View File

@@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""Kafka topic and consumer lag smoke checks for the Go native production stack."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
DEFAULT_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_BOOTSTRAP = "127.0.0.1:9092"
DEFAULT_KAFKA_BIN = "/opt/kafka/current/bin"
DEFAULT_TOPICS = [
"vehicle.raw.go.gb32960.v1",
"vehicle.raw.go.jt808.v1",
"vehicle.raw.go.yutong-mqtt.v1",
"vehicle.event.go.unified.v1",
]
DEFAULT_GROUPS = [
"go-history-writer",
"go-stat-writer",
"go-realtime-api",
]
@dataclass(frozen=True)
class Check:
name: str
status: str
message: str
@dataclass(frozen=True)
class ConsumerLag:
group: str
topic: str
partition: int
current_offset: int | None
log_end_offset: int
lag: int | None
def parse_topic_list(output: str) -> set[str]:
return {line.strip() for line in output.splitlines() if line.strip()}
def topic_checks(existing: set[str], required: list[str]) -> list[Check]:
checks: list[Check] = []
for topic in required:
if topic in existing:
checks.append(Check("topic." + topic, "pass", "present"))
else:
checks.append(Check("topic." + topic, "fail", "missing"))
return checks
def parse_consumer_group_describe(output: str) -> list[ConsumerLag]:
rows: list[ConsumerLag] = []
for line in output.splitlines():
parts = line.split()
if len(parts) < 6 or parts[0] == "GROUP":
continue
group, topic = parts[0], parts[1]
try:
partition = int(parts[2])
current = parse_optional_int(parts[3])
log_end = int(parts[4])
lag = parse_optional_int(parts[5])
except ValueError:
continue
rows.append(ConsumerLag(group, topic, partition, current, log_end, lag))
return rows
def parse_optional_int(value: str) -> int | None:
value = value.strip()
if value == "-":
return None
return int(value)
def group_lag_check(group: str, rows: list[ConsumerLag], max_lag: int) -> Check:
group_rows = [row for row in rows if row.group == group]
if not group_rows:
return Check("group." + group, "fail", "no rows")
concrete_lags = [row.lag for row in group_rows if row.lag is not None]
if not concrete_lags:
return Check("group." + group, "fail", "no concrete lag rows")
max_seen = max(concrete_lags)
topics = sorted({row.topic for row in group_rows})
status = "pass" if max_seen <= max_lag else "fail"
return Check(
"group." + group,
status,
"max_lag=" + str(max_seen) + "; threshold=" + str(max_lag) + "; topics=" + ",".join(topics),
)
def ssh(host: str, user: str, command: str, timeout: float) -> str:
target = user + "@" + host if user else host
completed = subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", target, command],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
return completed.stdout
def remote_topic_command(kafka_bin: str, bootstrap: str) -> str:
return kafka_bin.rstrip("/") + "/kafka-topics.sh --bootstrap-server " + bootstrap + " --list"
def remote_group_command(kafka_bin: str, bootstrap: str, group: str) -> str:
return kafka_bin.rstrip("/") + "/kafka-consumer-groups.sh --bootstrap-server " + bootstrap + " --describe --group " + group
def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
topic_output = ssh(args.host, args.user, remote_topic_command(args.kafka_bin, args.bootstrap), args.timeout)
checks = topic_checks(parse_topic_list(topic_output), DEFAULT_TOPICS)
for group in DEFAULT_GROUPS:
group_output = ssh(args.host, args.user, remote_group_command(args.kafka_bin, args.bootstrap, group), args.timeout)
checks.append(group_lag_check(group, parse_consumer_group_describe(group_output), args.max_lag))
status = "fail" if any(check.status == "fail" for check in checks) else "pass"
return status, checks
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--bootstrap", default=DEFAULT_BOOTSTRAP)
parser.add_argument("--kafka-bin", default=DEFAULT_KAFKA_BIN)
parser.add_argument("--max-lag", type=int, default=100)
parser.add_argument("--timeout", type=float, default=20.0)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
status, checks = run(args)
except Exception as exc:
status = "fail"
checks = [Check("kafka.ssh", "fail", str(exc))]
print(json.dumps({
"status": status,
"host": args.host,
"bootstrap": args.bootstrap,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

253
tools/go_native_deploy.py Normal file
View File

@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Build and deploy the Go native vehicle gateway release to ECS."""
from __future__ import annotations
import argparse
import os
import pathlib
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
from dataclasses import dataclass
DEFAULT_APP_HOST = "115.29.187.205"
DEFAULT_KAFKA_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_RELEASE_ROOT = "/opt/lingniu-go-native"
DEFAULT_GO_DIR = "go/vehicle-gateway"
BINARIES = [
("gateway", "./cmd/gateway"),
("history-writer", "./cmd/history-writer"),
("stat-writer", "./cmd/stat-writer"),
("realtime-api", "./cmd/realtime-api"),
]
SERVICES = [
"lingniu-go-gateway",
"lingniu-go-history-writer",
"lingniu-go-stat-writer",
"lingniu-go-realtime-api",
]
@dataclass(frozen=True)
class BuildCommand:
output_name: str
argv: list[str]
env: list[str]
cwd: str
@dataclass(frozen=True)
class SpoolStatus:
files: int
recent: int
def git_short_sha(cwd: str) -> str:
return run_capture(["git", "rev-parse", "--short=7", "HEAD"], cwd=cwd).strip()
def build_commands(go_dir: str, output_dir: str) -> list[BuildCommand]:
commands: list[BuildCommand] = []
env = ["CGO_ENABLED=0", "GOOS=linux", "GOARCH=amd64"]
for output_name, package in BINARIES:
commands.append(BuildCommand(
output_name=output_name,
argv=[
"go",
"build",
"-trimpath",
"-ldflags=-s -w",
"-o",
str(pathlib.Path(output_dir) / output_name),
package,
],
env=env,
cwd=go_dir,
))
return commands
def build_release(go_dir: str, output_dir: str) -> None:
for command in build_commands(go_dir, output_dir):
env = os.environ.copy()
for item in command.env:
key, value = item.split("=", 1)
env[key] = value
run(command.argv, cwd=command.cwd, env=env)
def create_archive(source_dir: str, archive_path: str) -> None:
with tarfile.open(archive_path, "w:gz") as archive:
for binary, _ in BINARIES:
archive.add(pathlib.Path(source_dir) / binary, arcname=binary)
def remote_switch_command(release: str, release_root: str, remote_archive: str) -> str:
if not safe_release_name(release):
raise ValueError("release must contain only letters, numbers, dot, underscore, or dash")
release_dir = release_root.rstrip("/") + "/releases/" + release
root = release_root.rstrip("/")
commands = [
"set -euo pipefail",
"mkdir -p " + shell_quote(release_dir),
"tar -xzf " + shell_quote(remote_archive) + " -C " + shell_quote(release_dir),
"chmod +x " + shell_quote(release_dir) + "/*",
"ln -sfn " + shell_quote(release_dir) + " " + shell_quote(root + "/current"),
"systemctl daemon-reload",
]
commands.extend("systemctl restart " + service for service in SERVICES)
commands.append("rm -f " + shell_quote(remote_archive))
return " && ".join(commands)
def acceptance_command(app_host: str, kafka_host: str, date: str, timeout: float) -> list[str]:
command = [
"python3",
"tools/go_prod_acceptance.py",
"--app-host",
app_host,
"--kafka-host",
kafka_host,
"--timeout",
str(int(timeout) if timeout == int(timeout) else timeout),
]
if date:
command.extend(["--date", date])
return command
def deploy(args: argparse.Namespace) -> None:
repo = pathlib.Path(args.repo).resolve()
go_dir = str(repo / args.go_dir)
release = args.release or git_short_sha(str(repo))
with tempfile.TemporaryDirectory(prefix="lingniu-go-native-") as tmp:
output_dir = pathlib.Path(tmp) / "out"
output_dir.mkdir()
archive_path = pathlib.Path(tmp) / ("lingniu-go-native-" + release + ".tar.gz")
build_release(go_dir, str(output_dir))
create_archive(str(output_dir), str(archive_path))
remote_archive = "/tmp/" + archive_path.name
target = ssh_target(args.user, args.app_host)
run(["scp", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", str(archive_path), target + ":" + remote_archive])
run([
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
target,
remote_switch_command(release, args.release_root, remote_archive),
])
wait_for_spool_drain(target, args.release_root, args.spool_drain_timeout, args.spool_poll_interval)
if not args.skip_acceptance:
run(acceptance_command(args.app_host, args.kafka_host, args.date, args.timeout), cwd=str(repo))
def wait_for_spool_drain(target: str, release_root: str, timeout: float, poll_interval: float) -> None:
deadline = time.monotonic() + timeout
last = SpoolStatus(files=-1, recent=-1)
while time.monotonic() <= deadline:
output = run_capture([
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
target,
spool_status_command(release_root),
])
last = parse_spool_status(output)
print(f"spool files={last.files} recent={last.recent}", flush=True)
if last.files == 0:
return
time.sleep(max(1.0, poll_interval))
raise RuntimeError(f"gateway spool did not drain: files={last.files} recent={last.recent}")
def spool_status_command(release_root: str) -> str:
spool_dir = release_root.rstrip("/") + "/spool/gateway"
return (
"spool=" + shell_quote(spool_dir) + "; "
"files=$(find \"$spool\" -type f 2>/dev/null | wc -l); "
"recent=$(find \"$spool\" -type f -mmin -5 2>/dev/null | wc -l); "
"printf 'files=%s recent=%s\\n' \"$files\" \"$recent\""
)
def parse_spool_status(output: str) -> SpoolStatus:
values: dict[str, int] = {}
for part in output.split():
if "=" not in part:
continue
key, value = part.split("=", 1)
try:
values[key] = int(value)
except ValueError:
continue
return SpoolStatus(files=values.get("files", -1), recent=values.get("recent", -1))
def ssh_target(user: str, host: str) -> str:
return user + "@" + host if user else host
def shell_quote(value: str) -> str:
return "'" + value.replace("'", "'\"'\"'") + "'"
def safe_release_name(value: str) -> bool:
value = value.strip()
if not value:
return False
return all(r.isalnum() or r in "._-" for r in value)
def run(argv: list[str], cwd: str | None = None, env: dict[str, str] | None = None) -> None:
subprocess.run(argv, cwd=cwd, env=env, check=True)
def run_capture(argv: list[str], cwd: str | None = None) -> str:
return subprocess.run(argv, cwd=cwd, check=True, stdout=subprocess.PIPE, text=True).stdout
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", default=".")
parser.add_argument("--go-dir", default=DEFAULT_GO_DIR)
parser.add_argument("--app-host", default=DEFAULT_APP_HOST)
parser.add_argument("--kafka-host", default=DEFAULT_KAFKA_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--release-root", default=DEFAULT_RELEASE_ROOT)
parser.add_argument("--release", default="")
parser.add_argument("--date", default="")
parser.add_argument("--timeout", type=float, default=20.0)
parser.add_argument("--spool-drain-timeout", type=float, default=900.0)
parser.add_argument("--spool-poll-interval", type=float, default=10.0)
parser.add_argument("--skip-acceptance", action="store_true")
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
deploy(args)
except subprocess.CalledProcessError as exc:
print("command failed: " + " ".join(exc.cmd), file=sys.stderr)
return exc.returncode or 1
except (OSError, shutil.Error) as exc:
print("deploy failed: " + str(exc), file=sys.stderr)
return 1
except RuntimeError as exc:
print("deploy failed: " + str(exc), file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

544
tools/go_native_prod_smoke.py Executable file
View File

@@ -0,0 +1,544 @@
#!/usr/bin/env python3
"""HTTP smoke checks for the Go native production vehicle ingest stack."""
from __future__ import annotations
import argparse
import datetime as dt
import json
import sys
import urllib.parse
import urllib.request
from dataclasses import asdict, dataclass
from typing import Any
SHANGHAI = dt.timezone(dt.timedelta(hours=8))
DEFAULT_BASE_URL = "http://115.29.187.205:20210"
DEFAULT_REALTIME_MAX_AGE_MINUTES = 15.0
@dataclass(frozen=True)
class CheckSpec:
name: str
path: str
params: dict[str, Any]
minimum: int
kind: str = "total"
max_age_minutes: float | None = None
require_parsed_json: bool = False
require_frame_id: bool = False
require_metric_formula: bool = False
@dataclass(frozen=True)
class Check:
name: str
status: str
count: int
minimum: int
message: str
@dataclass(frozen=True)
class CheckWindow:
date_from: str
date_to: str
stat_date: str
max_raw_age_minutes: float | None
def shanghai_day_window(now: dt.datetime | None = None) -> tuple[str, str]:
now = now or dt.datetime.now(tz=SHANGHAI)
local = now.astimezone(SHANGHAI)
start = dt.datetime(local.year, local.month, local.day, tzinfo=SHANGHAI)
end = start + dt.timedelta(days=1)
return start.isoformat(timespec="seconds"), end.isoformat(timespec="seconds")
def resolve_check_window(
raw_date: str | None,
max_raw_age_minutes: float,
now: dt.datetime | None = None,
) -> CheckWindow:
current = (now or dt.datetime.now(tz=SHANGHAI)).astimezone(SHANGHAI)
today = current.date()
if raw_date:
day = dt.date.fromisoformat(raw_date)
start = dt.datetime(day.year, day.month, day.day, tzinfo=SHANGHAI)
date_from, date_to = shanghai_day_window(start)
freshness = max_raw_age_minutes if day == today else None
return CheckWindow(date_from, date_to, raw_date, freshness)
date_from, date_to = shanghai_day_window(current)
return CheckWindow(date_from, date_to, date_from[:10], max_raw_age_minutes)
def api_url(base_url: str, path: str, params: dict[str, Any]) -> str:
url = base_url.rstrip("/") + path
if not params:
return url
return url + "?" + urllib.parse.urlencode(params)
def query_json(base_url: str, path: str, params: dict[str, Any], timeout: float) -> dict[str, Any]:
request = urllib.request.Request(
api_url(base_url, path, params),
headers={"Accept": "application/json"},
)
with urllib.request.urlopen(request, timeout=timeout) as response:
return json.loads(response.read().decode("utf-8"))
def check_total(
name: str,
payload: dict[str, Any],
minimum: int,
max_age_minutes: float | None = None,
now: dt.datetime | None = None,
require_parsed_json: bool = False,
require_frame_id: bool = False,
require_metric_formula: bool = False,
) -> Check:
count = int(payload.get("total") or 0)
items = payload.get("items") or []
sample = sample_summary(items)
age_message = latest_age_message(items, now)
if count >= minimum:
parsed_message = ""
if max_age_minutes is not None:
latest_age = latest_age_minutes(items, now)
if latest_age is None:
return Check(name, "fail", count, minimum, f"count={count}; missing latest ts; sample={sample}")
if latest_age > max_age_minutes:
return Check(
name,
"fail",
count,
minimum,
f"count={count}; latest_age_minutes={latest_age:.1f}; "
f"expected <= {max_age_minutes:.1f}; sample={sample}",
)
if require_parsed_json:
parsed_message = parsed_json_message(items)
if not parsed_message.startswith("parsed_json=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {parsed_message}; sample={sample}")
frame_message = ""
if require_frame_id:
frame_message = frame_id_message(items)
if not frame_message.startswith("frame_id=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {frame_message}; sample={sample}")
metric_message = ""
if require_metric_formula:
metric_message = metric_formula_message(items)
if not metric_message.startswith("metric_formula=ok"):
return Check(name, "fail", count, minimum, f"count={count}; {metric_message}; sample={sample}")
details = [f"count={count}", age_message]
if parsed_message:
details.append(parsed_message)
if frame_message:
details.append(frame_message)
if metric_message:
details.append(metric_message)
return Check(name, "pass", count, minimum, "; ".join(details) + f"; sample={sample}")
return Check(name, "fail", count, minimum, f"count={count}; expected >= {minimum}; {age_message}; sample={sample}")
def parse_tdengine_utc_timestamp(value: str) -> dt.datetime:
value = str(value or "").strip()
for layout in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S"):
try:
return dt.datetime.strptime(value[:19], layout).replace(tzinfo=dt.timezone.utc)
except ValueError:
continue
raise ValueError(f"unsupported timestamp: {value}")
def latest_age_minutes(items: list[Any], now: dt.datetime | None = None) -> float | None:
if not items or not isinstance(items[0], dict):
return None
timestamp = items[0].get("received_at") or items[0].get("ts")
if not timestamp:
return None
current = now or dt.datetime.now(tz=dt.timezone.utc)
if current.tzinfo is None:
current = current.replace(tzinfo=dt.timezone.utc)
latest = parse_tdengine_utc_timestamp(str(timestamp))
return max(0.0, (current.astimezone(dt.timezone.utc) - latest).total_seconds() / 60)
def latest_age_message(items: list[Any], now: dt.datetime | None = None) -> str:
latest_age = latest_age_minutes(items, now)
if latest_age is None:
return "latest_age_minutes=unknown"
return f"latest_age_minutes={latest_age:.1f}"
def parsed_json_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing parsed_json"
raw = items[0].get("parsed_json")
if isinstance(raw, dict) and raw:
return "parsed_json=ok"
if not isinstance(raw, str) or not raw.strip() or raw.strip().lower() == "null":
return "missing parsed_json"
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return "invalid parsed_json"
if isinstance(parsed, dict) and parsed:
return "parsed_json=ok"
return "invalid parsed_json"
def frame_id_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing frame_id"
frame_id = str(items[0].get("frame_id") or "").strip()
if frame_id:
return "frame_id=ok"
return "missing frame_id"
def metric_formula_message(items: list[Any]) -> str:
if not items or not isinstance(items[0], dict):
return "missing metric row"
row = items[0]
try:
metric_value = float(row.get("metric_value"))
first_total = float(row.get("first_total_mileage_km"))
latest_total = float(row.get("latest_total_mileage_km"))
except (TypeError, ValueError):
return "missing metric formula fields"
metric_key = str(row.get("metric_key") or "")
if metric_key == "daily_mileage_km":
expected = max(0.0, latest_total - first_total)
elif metric_key == "daily_total_mileage_km":
expected = latest_total
else:
return "unsupported metric_key=" + metric_key
if abs(metric_value - expected) <= 0.001:
return "metric_formula=ok"
return f"metric_formula mismatch value={metric_value:.3f} expected={expected:.3f}"
def check_realtime(
name: str,
payload: dict[str, Any],
max_age_minutes: float | None = None,
now: dt.datetime | None = None,
) -> Check:
if payload.get("online") is False:
return Check(name, "fail", 0, 1, "online=false")
updated_message = ""
if max_age_minutes is not None:
updated_age = realtime_updated_age_minutes(payload, now)
if updated_age is None:
return Check(name, "fail", 0, 1, "missing updated_at_ms")
updated_message = f"updated_age_minutes={updated_age:.1f}"
if updated_age > max_age_minutes:
return Check(
name,
"fail",
0,
1,
updated_message + f"; expected <= {max_age_minutes:.1f}",
)
identifier = payload.get("vin") or payload.get("vehicle_key") or ""
protocols = payload.get("protocols") or []
protocol = payload.get("protocol") or ""
fields = payload.get("fields") or {}
message = json.dumps(
{
key: value
for key, value in {
"identifier": identifier,
"protocol": protocol,
"protocols": protocols,
"field_count": len(fields) if isinstance(fields, dict) else 0,
"online": payload.get("online"),
}.items()
if value not in (None, "", [])
},
ensure_ascii=False,
sort_keys=True,
)
if updated_message:
message = updated_message + "; " + message
return Check(name, "pass", 1, 1, message)
def realtime_updated_age_minutes(payload: dict[str, Any], now: dt.datetime | None = None) -> float | None:
try:
updated_ms = int(payload.get("updated_at_ms"))
except (TypeError, ValueError):
return None
current = now or dt.datetime.now(tz=dt.timezone.utc)
if current.tzinfo is None:
current = current.replace(tzinfo=dt.timezone.utc)
updated = dt.datetime.fromtimestamp(updated_ms / 1000, tz=dt.timezone.utc)
return max(0.0, (current.astimezone(dt.timezone.utc) - updated).total_seconds() / 60)
def sample_summary(items: list[Any]) -> str:
if not items:
return "{}"
first = items[0]
if not isinstance(first, dict):
return json.dumps(first, ensure_ascii=False, sort_keys=True)
keep = {
key: first.get(key)
for key in (
"ts",
"stat_date",
"protocol",
"vehicle_key",
"vin",
"phone",
"message_id_hex",
"metric_key",
"metric_value",
"total_mileage_km",
"latest_total_mileage_km",
)
if first.get(key) not in (None, "")
}
return json.dumps(keep, ensure_ascii=False, sort_keys=True)
def overall_status(checks: list[Check]) -> str:
return "fail" if any(check.status == "fail" for check in checks) else "pass"
def build_check_specs(
*,
date_from: str,
date_to: str,
stat_date: str,
min_raw: int,
min_history: int,
min_stat: int,
max_raw_age_minutes: float | None = None,
) -> list[CheckSpec]:
raw_params = {"limit": 1, "orderBy": "receivedAt", "includeTotal": "false"}
gb32960_history_params = {"protocol": "GB32960", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
jt808_history_params = {"protocol": "JT808", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
yutong_mqtt_history_params = {"protocol": "YUTONG_MQTT", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
return [
CheckSpec(
"gb32960.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "GB32960"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec(
"jt808.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "JT808"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec(
"yutong_mqtt.raw",
"/api/history/raw-frames",
{**raw_params, "protocol": "YUTONG_MQTT"},
min_raw,
max_age_minutes=max_raw_age_minutes,
require_parsed_json=True,
),
CheckSpec("gb32960.locations", "/api/history/locations", gb32960_history_params, min_history, require_frame_id=True),
CheckSpec("gb32960.mileage_points", "/api/history/mileage-points", gb32960_history_params, min_history, require_frame_id=True),
CheckSpec("jt808.locations", "/api/history/locations", jt808_history_params, min_history, require_frame_id=True),
CheckSpec("jt808.mileage_points", "/api/history/mileage-points", jt808_history_params, min_history, require_frame_id=True),
CheckSpec("yutong_mqtt.locations", "/api/history/locations", yutong_mqtt_history_params, min_history, require_frame_id=True),
CheckSpec("yutong_mqtt.mileage_points", "/api/history/mileage-points", yutong_mqtt_history_params, min_history, require_frame_id=True),
CheckSpec(
"gb32960.daily_mileage",
"/api/stats/daily-metrics",
{
"protocol": "GB32960",
"metricKey": "daily_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"gb32960.daily_total_mileage",
"/api/stats/daily-metrics",
{
"protocol": "GB32960",
"metricKey": "daily_total_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"jt808.daily_mileage",
"/api/stats/daily-metrics",
{
"protocol": "JT808",
"metricKey": "daily_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
CheckSpec(
"jt808.daily_total_mileage",
"/api/stats/daily-metrics",
{
"protocol": "JT808",
"metricKey": "daily_total_mileage_km",
"dateFrom": stat_date,
"dateTo": stat_date,
"limit": 1,
},
min_stat,
require_metric_formula=True,
),
]
def vehicle_identifier(item: dict[str, Any]) -> str:
for key in ("vin", "vehicle_key"):
value = str(item.get(key) or "").strip()
if value and value.lower() != "unknown":
return value
return ""
def build_realtime_specs(payloads: dict[str, dict[str, Any]]) -> list[CheckSpec]:
configs = [
("gb32960", "gb32960.raw", "GB32960"),
("jt808", "jt808.raw", "JT808"),
("yutong_mqtt", "yutong_mqtt.raw", "YUTONG_MQTT"),
]
specs: list[CheckSpec] = []
for prefix, source_name, protocol in configs:
items = payloads.get(source_name, {}).get("items") or []
if not items or not isinstance(items[0], dict):
continue
identifier = vehicle_identifier(items[0])
if not identifier:
continue
encoded = urllib.parse.quote(identifier, safe="")
base_path = "/api/realtime/vehicles/" + encoded
specs.extend([
CheckSpec(
prefix + ".realtime",
base_path,
{},
1,
"realtime",
max_age_minutes=DEFAULT_REALTIME_MAX_AGE_MINUTES,
),
CheckSpec(prefix + ".realtime_online", base_path + "/online", {}, 1, "realtime"),
CheckSpec(
prefix + ".realtime_protocol",
base_path + "/protocols/" + protocol,
{},
1,
"realtime",
max_age_minutes=DEFAULT_REALTIME_MAX_AGE_MINUTES,
),
])
return specs
def run_checks(base_url: str, specs: list[CheckSpec], timeout: float) -> list[Check]:
checks: list[Check] = []
for spec in specs:
try:
payload = query_json(base_url, spec.path, spec.params, timeout)
if spec.kind == "realtime":
checks.append(check_realtime(spec.name, payload, spec.max_age_minutes))
else:
checks.append(check_total(
spec.name,
payload,
spec.minimum,
spec.max_age_minutes,
require_parsed_json=spec.require_parsed_json,
require_frame_id=spec.require_frame_id,
require_metric_formula=spec.require_metric_formula,
))
except Exception as exc: # pragma: no cover - exercised by live failures.
checks.append(Check(spec.name, "fail", 0, spec.minimum, f"request failed: {exc}"))
return checks
def query_payloads(base_url: str, specs: list[CheckSpec], timeout: float) -> tuple[list[Check], dict[str, dict[str, Any]]]:
checks: list[Check] = []
payloads: dict[str, dict[str, Any]] = {}
for spec in specs:
try:
payload = query_json(base_url, spec.path, spec.params, timeout)
payloads[spec.name] = payload
checks.append(check_total(
spec.name,
payload,
spec.minimum,
spec.max_age_minutes,
require_parsed_json=spec.require_parsed_json,
require_frame_id=spec.require_frame_id,
require_metric_formula=spec.require_metric_formula,
))
except Exception as exc:
checks.append(Check(spec.name, "fail", 0, spec.minimum, f"request failed: {exc}"))
return checks, payloads
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--date", help="Shanghai local date, YYYY-MM-DD. Defaults to today.")
parser.add_argument("--timeout", type=float, default=5.0)
parser.add_argument("--min-raw", type=int, default=1)
parser.add_argument("--min-history", type=int, default=1)
parser.add_argument("--min-stat", type=int, default=1)
parser.add_argument(
"--max-raw-age-minutes",
type=float,
default=15.0,
help="Require latest RAW sample age to be at most this many minutes when checking today's date.",
)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
window = resolve_check_window(args.date, args.max_raw_age_minutes)
specs = build_check_specs(
date_from=window.date_from,
date_to=window.date_to,
stat_date=window.stat_date,
min_raw=args.min_raw,
min_history=args.min_history,
min_stat=args.min_stat,
max_raw_age_minutes=window.max_raw_age_minutes,
)
checks, payloads = query_payloads(args.base_url, specs, args.timeout)
checks.extend(run_checks(args.base_url, build_realtime_specs(payloads), args.timeout))
status = overall_status(checks)
print(json.dumps({
"status": status,
"baseUrl": args.base_url,
"dateFrom": window.date_from,
"dateTo": window.date_to,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

136
tools/go_prod_acceptance.py Executable file
View File

@@ -0,0 +1,136 @@
#!/usr/bin/env python3
"""Run the Go native production acceptance smoke suite."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
from typing import Any
DEFAULT_APP_HOST = "115.29.187.205"
DEFAULT_KAFKA_HOST = "114.55.58.251"
DEFAULT_USER = "root"
DEFAULT_BASE_URL = "http://115.29.187.205:20210"
@dataclass(frozen=True)
class ChildCommand:
name: str
argv: list[str]
@dataclass(frozen=True)
class ChildResult:
name: str
status: str
exit_code: int
payload: dict[str, Any]
def build_commands(args: argparse.Namespace) -> list[ChildCommand]:
timeout = str(args.timeout)
return [
ChildCommand("systemd", [
sys.executable,
"tools/go_systemd_prod_smoke.py",
"--host",
args.app_host,
"--user",
args.ssh_user,
"--timeout",
timeout,
]),
ChildCommand("kafka", [
sys.executable,
"tools/go_kafka_prod_smoke.py",
"--host",
args.kafka_host,
"--user",
args.ssh_user,
"--max-lag",
str(args.max_lag),
"--timeout",
timeout,
]),
ChildCommand("http", [
sys.executable,
"tools/go_native_prod_smoke.py",
"--base-url",
args.base_url,
"--timeout",
timeout,
] + (["--date", args.date] if args.date else [])),
]
def run_child(command: ChildCommand, timeout: float) -> ChildResult:
completed = subprocess.run(
command.argv,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
payload = parse_json_output(completed.stdout)
if completed.stderr.strip():
payload.setdefault("stderr", completed.stderr.strip())
status = payload.get("status")
if status not in {"pass", "fail"}:
status = "pass" if completed.returncode == 0 else "fail"
return ChildResult(command.name, status, completed.returncode, payload)
def parse_json_output(output: str) -> dict[str, Any]:
output = output.strip()
if not output:
return {}
start = output.find("{")
end = output.rfind("}")
if start < 0 or end < start:
return {"rawOutput": output}
try:
return json.loads(output[start:end + 1])
except json.JSONDecodeError:
return {"rawOutput": output}
def overall_status(results: list[ChildResult]) -> str:
if not results:
return "fail"
return "fail" if any(result.status != "pass" or result.exit_code != 0 for result in results) else "pass"
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--app-host", default=DEFAULT_APP_HOST)
parser.add_argument("--kafka-host", default=DEFAULT_KAFKA_HOST)
parser.add_argument("--ssh-user", default=DEFAULT_USER)
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--date", help="Shanghai local date, YYYY-MM-DD. Defaults to today in go_native_prod_smoke.py.")
parser.add_argument("--timeout", type=float, default=20.0)
parser.add_argument("--max-lag", type=int, default=100)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
results: list[ChildResult] = []
for command in build_commands(args):
try:
results.append(run_child(command, args.timeout + 10))
except Exception as exc:
results.append(ChildResult(command.name, "fail", 1, {"status": "fail", "error": str(exc)}))
status = overall_status(results)
print(json.dumps({
"status": status,
"checks": [asdict(result) for result in results],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

210
tools/go_systemd_prod_smoke.py Executable file
View File

@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""Systemd and port smoke checks for the Go native production ECS."""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from dataclasses import asdict, dataclass
DEFAULT_HOST = "115.29.187.205"
DEFAULT_USER = "root"
DEFAULT_SERVICES = [
"lingniu-go-gateway.service",
"lingniu-go-history-writer.service",
"lingniu-go-stat-writer.service",
"lingniu-go-realtime-api.service",
]
DEFAULT_PORTS = {
808: "gateway",
32960: "gateway",
20210: "realtime-api",
}
DEFAULT_RELEASE_ROOT = "/opt/lingniu-go-native"
DEFAULT_SPOOL_DIR = "/opt/lingniu-go-native/spool/gateway"
DEFAULT_BINARIES = ["gateway", "history-writer", "stat-writer", "realtime-api"]
@dataclass(frozen=True)
class Check:
name: str
status: str
message: str
@dataclass(frozen=True)
class ServiceState:
active: str
enabled: str
def parse_service_states(output: str) -> dict[str, ServiceState]:
states: dict[str, ServiceState] = {}
for line in output.splitlines():
parts = line.split()
if len(parts) >= 2:
enabled = parts[2] if len(parts) >= 3 else "enabled"
states[parts[0]] = ServiceState(parts[1], enabled)
return states
def service_checks(output: str, services: list[str]) -> list[Check]:
states = parse_service_states(output)
checks: list[Check] = []
for service in services:
state = states.get(service, ServiceState("missing", "missing"))
status = "pass" if state.active == "active" and state.enabled == "enabled" else "fail"
checks.append(Check(
"service." + service,
status,
"state=" + state.active + "; enabled=" + state.enabled,
))
return checks
def port_checks(output: str, ports: dict[int, str]) -> list[Check]:
lines = output.splitlines()
checks: list[Check] = []
for port, process in sorted(ports.items()):
matching = [line.strip() for line in lines if has_port(line, port)]
if not matching:
checks.append(Check("port." + str(port), "fail", "not listening"))
continue
owned = [line for line in matching if process in line]
if owned:
checks.append(Check("port." + str(port), "pass", owned[0]))
else:
checks.append(Check("port." + str(port), "fail", matching[0]))
return checks
def has_port(line: str, port: int) -> bool:
needle = ":" + str(port)
return needle + " " in line or needle + "\t" in line
def spool_check(output: str) -> Check:
values = parse_key_values(output)
files = int(values.get("files", "-1"))
recent = int(values.get("recent", "-1"))
status = "pass" if files == 0 and recent == 0 else "fail"
return Check("spool.gateway", status, output.strip() or "missing")
def release_check(output: str) -> Check:
values = parse_key_values(output)
current = values.get("current", "")
binaries_present = [values.get(binary, "0") == "1" for binary in DEFAULT_BINARIES]
status = "pass" if current and all(binaries_present) else "fail"
return Check("release.current", status, output.strip() or "missing")
def parse_key_values(output: str) -> dict[str, str]:
values: dict[str, str] = {}
for part in output.split():
if "=" not in part:
continue
key, value = part.split("=", 1)
values[key] = value
return values
def ssh(host: str, user: str, command: str, timeout: float) -> str:
target = user + "@" + host if user else host
completed = subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", target, command],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=timeout,
)
return completed.stdout
def remote_command(services: list[str]) -> str:
quoted = " ".join(services)
return (
"for svc in " + quoted + "; do "
"printf '%s ' \"$svc\"; "
"printf '%s ' \"$(systemctl is-active \"$svc\" 2>/dev/null || true)\"; "
"systemctl is-enabled \"$svc\" 2>/dev/null || true; "
"done; "
"printf '\\n--PORTS--\\n'; "
"ss -lntp; "
"printf '\\n--SPOOL--\\n'; "
"spool='" + DEFAULT_SPOOL_DIR + "'; "
"if test -d \"$spool\"; then "
"files=$(find \"$spool\" -type f | wc -l); "
"bytes=$(du -sb \"$spool\" 2>/dev/null | awk '{print $1}'); "
"recent=$(find \"$spool\" -type f -mmin -5 | wc -l); "
"printf 'files=%s bytes=%s recent=%s\\n' \"$files\" \"${bytes:-0}\" \"$recent\"; "
"else printf 'files=-1 bytes=0 recent=-1 missing=%s\\n' \"$spool\"; fi; "
"printf '\\n--RELEASE--\\n'; "
"root='" + DEFAULT_RELEASE_ROOT + "'; "
"current=$(readlink -f \"$root/current\" 2>/dev/null || true); "
"printf 'current=%s' \"$current\"; "
"for bin in " + " ".join(DEFAULT_BINARIES) + "; do "
"if test -x \"$root/current/$bin\"; then present=1; else present=0; fi; "
"printf ' %s=%s' \"$bin\" \"$present\"; "
"done; printf '\\n'"
)
def split_remote_output(output: str) -> tuple[str, str, str, str]:
ports_marker = "\n--PORTS--\n"
spool_marker = "\n--SPOOL--\n"
release_marker = "\n--RELEASE--\n"
if ports_marker not in output:
return output, "", "", ""
service_output, rest = output.split(ports_marker, 1)
if spool_marker not in rest:
return service_output, rest, "", ""
port_output, spool_output = rest.split(spool_marker, 1)
if release_marker not in spool_output:
return service_output, port_output, spool_output, ""
spool_output, release_output = spool_output.split(release_marker, 1)
return service_output, port_output, spool_output, release_output
def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
output = ssh(args.host, args.user, remote_command(DEFAULT_SERVICES), args.timeout)
service_output, port_output, spool_output, release_output = split_remote_output(output)
checks = (
service_checks(service_output, DEFAULT_SERVICES)
+ port_checks(port_output, DEFAULT_PORTS)
+ [spool_check(spool_output)]
+ [release_check(release_output)]
)
status = "fail" if any(check.status == "fail" for check in checks) else "pass"
return status, checks
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--user", default=DEFAULT_USER)
parser.add_argument("--timeout", type=float, default=20.0)
return parser.parse_args(argv)
def main(argv: list[str]) -> int:
args = parse_args(argv)
try:
status, checks = run(args)
except Exception as exc:
status = "fail"
checks = [Check("systemd.ssh", "fail", str(exc))]
print(json.dumps({
"status": status,
"host": args.host,
"checks": [asdict(check) for check in checks],
}, ensure_ascii=False, indent=2))
return 0 if status == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

View File

@@ -0,0 +1,62 @@
import unittest
from tools import go_kafka_prod_smoke as smoke
class GoKafkaProdSmokeTest(unittest.TestCase):
def test_parse_topic_list_marks_required_topics_present(self):
topics = smoke.parse_topic_list("""
vehicle.raw.go.gb32960.v1
vehicle.raw.go.jt808.v1
vehicle.raw.go.yutong-mqtt.v1
vehicle.event.go.unified.v1
""")
checks = smoke.topic_checks(topics, smoke.DEFAULT_TOPICS)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_topic_checks_fail_when_required_topic_missing(self):
topics = {"vehicle.raw.go.gb32960.v1"}
checks = smoke.topic_checks(topics, ["vehicle.raw.go.gb32960.v1", "vehicle.event.go.unified.v1"])
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["topic.vehicle.event.go.unified.v1"].status, "fail")
def test_parse_consumer_group_describe_reads_lag_rows(self):
rows = smoke.parse_consumer_group_describe("""
GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG
go-realtime-api vehicle.event.go.unified.v1 0 15992 15992 0
go-realtime-api vehicle.event.go.unified.v1 1 2036 2036 3
""")
self.assertEqual(rows[0].group, "go-realtime-api")
self.assertEqual(rows[1].topic, "vehicle.event.go.unified.v1")
self.assertEqual(rows[1].lag, 3)
def test_lag_checks_fail_when_lag_exceeds_threshold(self):
rows = [
smoke.ConsumerLag("go-history-writer", "vehicle.raw.go.jt808.v1", 0, 100, 100, 0),
smoke.ConsumerLag("go-history-writer", "vehicle.raw.go.jt808.v1", 1, 100, 110, 10),
]
check = smoke.group_lag_check("go-history-writer", rows, max_lag=5)
self.assertEqual(check.status, "fail")
self.assertIn("max_lag=10", check.message)
def test_lag_checks_pass_when_group_has_rows_and_lag_is_small(self):
rows = [
smoke.ConsumerLag("go-stat-writer", "vehicle.raw.go.gb32960.v1", 0, 100, 100, 0),
smoke.ConsumerLag("go-stat-writer", "vehicle.raw.go.jt808.v1", 1, 100, 101, 1),
]
check = smoke.group_lag_check("go-stat-writer", rows, max_lag=5)
self.assertEqual(check.status, "pass")
self.assertIn("max_lag=1", check.message)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,63 @@
import unittest
from tools import go_native_deploy as deploy
class GoNativeDeployTest(unittest.TestCase):
def test_build_commands_cover_four_production_binaries(self):
commands = deploy.build_commands("/repo/go/vehicle-gateway", "/tmp/out")
self.assertEqual([command.output_name for command in commands], [
"gateway",
"history-writer",
"stat-writer",
"realtime-api",
])
for command in commands:
self.assertEqual(command.argv[:3], ["go", "build", "-trimpath"])
self.assertIn("GOOS=linux", command.env)
self.assertIn("GOARCH=amd64", command.env)
self.assertIn("CGO_ENABLED=0", command.env)
def test_remote_switch_command_creates_release_and_restarts_all_services(self):
command = deploy.remote_switch_command("409f55b", "/opt/lingniu-go-native", "/tmp/lingniu-go-native-409f55b.tar.gz")
self.assertIn("/opt/lingniu-go-native/releases/409f55b", command)
self.assertIn("ln -sfn", command)
self.assertIn("systemctl restart lingniu-go-gateway", command)
self.assertIn("systemctl restart lingniu-go-history-writer", command)
self.assertIn("systemctl restart lingniu-go-stat-writer", command)
self.assertIn("systemctl restart lingniu-go-realtime-api", command)
def test_acceptance_command_uses_deployed_hosts_and_date(self):
command = deploy.acceptance_command(
app_host="115.29.187.205",
kafka_host="114.55.58.251",
date="2026-07-02",
timeout=20,
)
self.assertEqual(command[:2], ["python3", "tools/go_prod_acceptance.py"])
self.assertIn("--app-host", command)
self.assertIn("115.29.187.205", command)
self.assertIn("--kafka-host", command)
self.assertIn("114.55.58.251", command)
self.assertIn("--date", command)
self.assertIn("2026-07-02", command)
def test_parse_spool_status_reads_file_and_recent_counts(self):
status = deploy.parse_spool_status("files=42 recent=3")
self.assertEqual(status.files, 42)
self.assertEqual(status.recent, 3)
def test_spool_status_command_targets_gateway_spool_directory(self):
command = deploy.spool_status_command("/opt/lingniu-go-native")
self.assertIn("/opt/lingniu-go-native/spool/gateway", command)
self.assertIn("files=", command)
self.assertIn("recent=", command)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,358 @@
import datetime as dt
import unittest
from tools import go_native_prod_smoke as smoke
class GoNativeProdSmokeTest(unittest.TestCase):
def test_shanghai_day_window_uses_plus_eight_bounds(self):
now = dt.datetime(2026, 7, 2, 1, 35, tzinfo=smoke.SHANGHAI)
date_from, date_to = smoke.shanghai_day_window(now)
self.assertEqual(date_from, "2026-07-02T00:00:00+08:00")
self.assertEqual(date_to, "2026-07-03T00:00:00+08:00")
def test_api_url_encodes_plus_eight_date_range(self):
url = smoke.api_url(
"http://example.test/base/",
"/api/history/raw-frames",
{
"protocol": "JT808",
"dateFrom": "2026-07-02T00:00:00+08:00",
"dateTo": "2026-07-03T00:00:00+08:00",
"limit": 1,
},
)
self.assertEqual(
url,
"http://example.test/base/api/history/raw-frames?"
"protocol=JT808&dateFrom=2026-07-02T00%3A00%3A00%2B08%3A00"
"&dateTo=2026-07-03T00%3A00%3A00%2B08%3A00&limit=1",
)
def test_explicit_today_date_still_enforces_raw_freshness(self):
now = dt.datetime(2026, 7, 2, 9, 30, tzinfo=smoke.SHANGHAI)
window = smoke.resolve_check_window("2026-07-02", 15.0, now)
self.assertEqual(window.date_from, "2026-07-02T00:00:00+08:00")
self.assertEqual(window.date_to, "2026-07-03T00:00:00+08:00")
self.assertEqual(window.stat_date, "2026-07-02")
self.assertEqual(window.max_raw_age_minutes, 15.0)
def test_explicit_historical_date_disables_raw_freshness(self):
now = dt.datetime(2026, 7, 2, 9, 30, tzinfo=smoke.SHANGHAI)
window = smoke.resolve_check_window("2026-07-01", 15.0, now)
self.assertEqual(window.date_from, "2026-07-01T00:00:00+08:00")
self.assertEqual(window.date_to, "2026-07-02T00:00:00+08:00")
self.assertEqual(window.stat_date, "2026-07-01")
self.assertIsNone(window.max_raw_age_minutes)
def test_total_check_passes_when_total_reaches_minimum(self):
check = smoke.check_total(
"jt808.raw",
{"total": 2, "items": [{"vehicle_key": "JT808:013307811170"}]},
minimum=1,
)
self.assertEqual(check.status, "pass")
self.assertEqual(check.count, 2)
self.assertIn("JT808:013307811170", check.message)
def test_total_check_fails_when_total_is_below_minimum(self):
check = smoke.check_total("gb32960.raw", {"total": 0, "items": []}, minimum=1)
self.assertEqual(check.status, "fail")
self.assertEqual(check.count, 0)
self.assertIn("expected >= 1", check.message)
def test_overall_status_fails_if_any_check_fails(self):
checks = [
smoke.Check("a", "pass", 1, 1, "ok"),
smoke.Check("b", "fail", 0, 1, "bad"),
]
self.assertEqual(smoke.overall_status(checks), "fail")
def test_build_checks_cover_three_raw_protocols_and_two_daily_metric_protocols(self):
specs = smoke.build_check_specs(
date_from="2026-07-02T00:00:00+08:00",
date_to="2026-07-03T00:00:00+08:00",
stat_date="2026-07-02",
min_raw=1,
min_history=1,
min_stat=1,
)
names = [spec.name for spec in specs]
self.assertIn("gb32960.raw", names)
self.assertIn("jt808.raw", names)
self.assertIn("yutong_mqtt.raw", names)
self.assertIn("gb32960.locations", names)
self.assertIn("gb32960.mileage_points", names)
self.assertIn("jt808.locations", names)
self.assertIn("jt808.mileage_points", names)
self.assertIn("yutong_mqtt.locations", names)
self.assertIn("yutong_mqtt.mileage_points", names)
self.assertIn("gb32960.daily_mileage", names)
self.assertIn("jt808.daily_total_mileage", names)
def test_raw_checks_order_by_received_at_for_ingest_freshness(self):
specs = smoke.build_check_specs(
date_from="2026-07-02T00:00:00+08:00",
date_to="2026-07-03T00:00:00+08:00",
stat_date="2026-07-02",
min_raw=1,
min_history=1,
min_stat=1,
max_raw_age_minutes=15,
)
raw_specs = [spec for spec in specs if spec.name.endswith(".raw")]
self.assertTrue(raw_specs)
for spec in raw_specs:
self.assertEqual(spec.params.get("orderBy"), "receivedAt")
self.assertEqual(spec.params.get("includeTotal"), "false")
self.assertNotIn("dateFrom", spec.params)
self.assertNotIn("dateTo", spec.params)
def test_vehicle_identifier_prefers_vin_over_vehicle_key(self):
identifier = smoke.vehicle_identifier({
"vin": "LB9A32A20R0LS1343",
"vehicle_key": "GB32960:ignored",
})
self.assertEqual(identifier, "LB9A32A20R0LS1343")
def test_vehicle_identifier_falls_back_to_vehicle_key(self):
identifier = smoke.vehicle_identifier({
"vin": "",
"vehicle_key": "JT808:013307811170",
})
self.assertEqual(identifier, "JT808:013307811170")
def test_realtime_specs_are_built_from_raw_samples_with_vin(self):
payloads = {
"gb32960.raw": {"items": [{"vin": "LB9A32A20R0LS1343", "vehicle_key": "LB9A32A20R0LS1343"}]},
"jt808.raw": {"items": [{"vin": "", "vehicle_key": "JT808:013307811170"}]},
"yutong_mqtt.raw": {"items": [{"vin": "LMRKH9AC6R1004108"}]},
}
specs = smoke.build_realtime_specs(payloads)
self.assertEqual(
[(spec.name, spec.path) for spec in specs],
[
("gb32960.realtime", "/api/realtime/vehicles/LB9A32A20R0LS1343"),
("gb32960.realtime_online", "/api/realtime/vehicles/LB9A32A20R0LS1343/online"),
("gb32960.realtime_protocol", "/api/realtime/vehicles/LB9A32A20R0LS1343/protocols/GB32960"),
("jt808.realtime", "/api/realtime/vehicles/JT808%3A013307811170"),
("jt808.realtime_online", "/api/realtime/vehicles/JT808%3A013307811170/online"),
("jt808.realtime_protocol", "/api/realtime/vehicles/JT808%3A013307811170/protocols/JT808"),
("yutong_mqtt.realtime", "/api/realtime/vehicles/LMRKH9AC6R1004108"),
("yutong_mqtt.realtime_online", "/api/realtime/vehicles/LMRKH9AC6R1004108/online"),
("yutong_mqtt.realtime_protocol", "/api/realtime/vehicles/LMRKH9AC6R1004108/protocols/YUTONG_MQTT"),
],
)
def test_realtime_check_requires_online_true_when_field_exists(self):
check = smoke.check_realtime(
"gb32960.realtime_online",
{"vin": "LB9A32A20R0LS1343", "online": True, "protocols": ["GB32960"]},
)
self.assertEqual(check.status, "pass")
self.assertEqual(check.count, 1)
def test_realtime_check_fails_when_online_false(self):
check = smoke.check_realtime(
"gb32960.realtime_online",
{"vin": "LB9A32A20R0LS1343", "online": False},
)
self.assertEqual(check.status, "fail")
self.assertIn("online=false", check.message)
def test_realtime_check_fails_when_snapshot_is_stale(self):
now = dt.datetime(2026, 7, 2, 0, 30, 0, tzinfo=dt.timezone.utc)
check = smoke.check_realtime(
"jt808.realtime",
{"vehicle_key": "JT808:013307811170", "updated_at_ms": 1782950400000},
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "fail")
self.assertIn("updated_age_minutes=30.0", check.message)
def test_realtime_check_passes_when_snapshot_is_fresh(self):
now = dt.datetime(2026, 7, 2, 0, 30, 0, tzinfo=dt.timezone.utc)
check = smoke.check_realtime(
"jt808.realtime",
{"vehicle_key": "JT808:013307811170", "updated_at_ms": 1782951600000},
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("updated_age_minutes=10.0", check.message)
def test_parse_tdengine_utc_timestamp_as_aware_utc(self):
parsed = smoke.parse_tdengine_utc_timestamp("2026-07-01 17:36:02")
self.assertEqual(parsed, dt.datetime(2026, 7, 1, 17, 36, 2, tzinfo=dt.timezone.utc))
def test_total_check_fails_when_latest_sample_is_stale(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{"total": 1, "items": [{"ts": "2026-07-01 17:00:00"}]},
minimum=1,
max_age_minutes=30,
now=now,
)
self.assertEqual(check.status, "fail")
self.assertIn("latest_age_minutes=60.0", check.message)
def test_total_check_uses_received_at_for_raw_freshness_when_present(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:00:00",
"received_at": "2026-07-01 17:58:00",
}],
},
minimum=1,
max_age_minutes=15,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("latest_age_minutes=2.0", check.message)
def test_total_check_passes_when_latest_sample_is_fresh(self):
now = dt.datetime(2026, 7, 1, 18, 0, 0, tzinfo=dt.timezone.utc)
check = smoke.check_total(
"gb32960.raw",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00"}]},
minimum=1,
max_age_minutes=30,
now=now,
)
self.assertEqual(check.status, "pass")
self.assertIn("latest_age_minutes=15.0", check.message)
def test_raw_check_requires_structured_parsed_json(self):
check = smoke.check_total(
"jt808.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:45:00",
"parsed_json": "",
}],
},
minimum=1,
require_parsed_json=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("missing parsed_json", check.message)
def test_raw_check_accepts_structured_parsed_json(self):
check = smoke.check_total(
"gb32960.raw",
{
"total": 1,
"items": [{
"ts": "2026-07-01 17:45:00",
"parsed_json": "{\"header\":{\"vin\":\"LTEST\"}}",
}],
},
minimum=1,
require_parsed_json=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("parsed_json=ok", check.message)
def test_history_check_requires_frame_id_backlink(self):
check = smoke.check_total(
"jt808.locations",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00", "frame_id": ""}]},
minimum=1,
require_frame_id=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("missing frame_id", check.message)
def test_history_check_accepts_frame_id_backlink(self):
check = smoke.check_total(
"jt808.mileage_points",
{"total": 1, "items": [{"ts": "2026-07-01 17:45:00", "frame_id": "go_abc"}]},
minimum=1,
require_frame_id=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("frame_id=ok", check.message)
def test_daily_mileage_formula_requires_latest_minus_first(self):
check = smoke.check_total(
"jt808.daily_mileage",
{
"total": 1,
"items": [{
"metric_key": "daily_mileage_km",
"metric_value": 40.0,
"first_total_mileage_km": 4434.9,
"latest_total_mileage_km": 4481.0,
}],
},
minimum=1,
require_metric_formula=True,
)
self.assertEqual(check.status, "fail")
self.assertIn("metric_formula mismatch", check.message)
def test_daily_total_formula_requires_latest_total(self):
check = smoke.check_total(
"jt808.daily_total_mileage",
{
"total": 1,
"items": [{
"metric_key": "daily_total_mileage_km",
"metric_value": 4481.0,
"first_total_mileage_km": 4434.9,
"latest_total_mileage_km": 4481.0,
}],
},
minimum=1,
require_metric_formula=True,
)
self.assertEqual(check.status, "pass")
self.assertIn("metric_formula=ok", check.message)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,53 @@
import argparse
import sys
import unittest
from tools import go_prod_acceptance as acceptance
class GoProdAcceptanceTest(unittest.TestCase):
def test_build_commands_runs_systemd_kafka_and_http_smokes(self):
args = argparse.Namespace(
app_host="115.29.187.205",
kafka_host="114.55.58.251",
ssh_user="root",
base_url="http://115.29.187.205:20210",
date="2026-07-02",
timeout=8.0,
max_lag=100,
)
commands = acceptance.build_commands(args)
self.assertEqual(len(commands), 3)
self.assertEqual(commands[0].name, "systemd")
self.assertEqual(commands[0].argv[:2], [sys.executable, "tools/go_systemd_prod_smoke.py"])
self.assertIn("--host", commands[0].argv)
self.assertIn("115.29.187.205", commands[0].argv)
self.assertEqual(commands[1].name, "kafka")
self.assertIn("tools/go_kafka_prod_smoke.py", commands[1].argv)
self.assertIn("--max-lag", commands[1].argv)
self.assertEqual(commands[2].name, "http")
self.assertIn("tools/go_native_prod_smoke.py", commands[2].argv)
self.assertIn("--base-url", commands[2].argv)
def test_overall_status_fails_when_any_child_fails(self):
results = [
acceptance.ChildResult("systemd", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("kafka", "fail", 1, {"status": "fail"}),
]
self.assertEqual(acceptance.overall_status(results), "fail")
def test_overall_status_passes_when_all_children_pass(self):
results = [
acceptance.ChildResult("systemd", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("kafka", "pass", 0, {"status": "pass"}),
acceptance.ChildResult("http", "pass", 0, {"status": "pass"}),
]
self.assertEqual(acceptance.overall_status(results), "pass")
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,101 @@
import unittest
from tools import go_systemd_prod_smoke as smoke
class GoSystemdProdSmokeTest(unittest.TestCase):
def test_service_checks_require_all_go_units_active(self):
output = """
lingniu-go-gateway.service active enabled
lingniu-go-history-writer.service active enabled
lingniu-go-stat-writer.service active enabled
lingniu-go-realtime-api.service active enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_service_checks_fail_when_unit_is_missing_or_inactive(self):
output = """
lingniu-go-gateway.service active enabled
lingniu-go-history-writer.service failed enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["service.lingniu-go-history-writer.service"].status, "fail")
self.assertEqual(by_name["service.lingniu-go-stat-writer.service"].status, "fail")
def test_service_checks_fail_when_unit_is_not_enabled(self):
output = """
lingniu-go-gateway.service active disabled
lingniu-go-history-writer.service active enabled
lingniu-go-stat-writer.service active enabled
lingniu-go-realtime-api.service active enabled
"""
checks = smoke.service_checks(output, smoke.DEFAULT_SERVICES)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["service.lingniu-go-gateway.service"].status, "fail")
self.assertIn("enabled=disabled", by_name["service.lingniu-go-gateway.service"].message)
def test_port_checks_require_expected_go_processes(self):
output = """
LISTEN 0 4096 *:32960 *:* users:(("gateway",pid=10,fd=3))
LISTEN 0 4096 *:808 *:* users:(("gateway",pid=10,fd=4))
LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
"""
checks = smoke.port_checks(output, smoke.DEFAULT_PORTS)
self.assertTrue(all(check.status == "pass" for check in checks))
def test_port_checks_fail_when_java_owns_ingest_port(self):
output = """
LISTEN 0 4096 *:808 *:* users:(("java",pid=20,fd=4))
LISTEN 0 4096 *:32960 *:* users:(("gateway",pid=10,fd=3))
LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
"""
checks = smoke.port_checks(output, smoke.DEFAULT_PORTS)
by_name = {check.name: check for check in checks}
self.assertEqual(by_name["port.808"].status, "fail")
self.assertIn("java", by_name["port.808"].message)
def test_spool_check_passes_when_no_pending_files(self):
check = smoke.spool_check("files=0 bytes=208896 recent=0")
self.assertEqual(check.status, "pass")
self.assertIn("files=0", check.message)
def test_spool_check_fails_when_pending_or_recent_files_exist(self):
for output in ["files=2 bytes=300 recent=0", "files=0 bytes=208896 recent=1"]:
check = smoke.spool_check(output)
self.assertEqual(check.status, "fail")
def test_release_check_passes_when_current_release_has_all_binaries(self):
check = smoke.release_check(
"current=/opt/lingniu-go-native/releases/8def635 "
"gateway=1 history-writer=1 stat-writer=1 realtime-api=1"
)
self.assertEqual(check.status, "pass")
self.assertIn("8def635", check.message)
def test_release_check_fails_when_binary_is_missing(self):
check = smoke.release_check(
"current=/opt/lingniu-go-native/releases/8def635 "
"gateway=1 history-writer=0 stat-writer=1 realtime-api=1"
)
self.assertEqual(check.status, "fail")
self.assertIn("history-writer=0", check.message)
if __name__ == "__main__":
unittest.main()