228 Commits

Author SHA1 Message Date
lingniu
cbb6f3b741 perf: skip raw freshness count scan 2026-07-02 09:30:15 +08:00
lingniu
42de422935 test: check raw freshness without event date filter 2026-07-02 09:27:22 +08:00
lingniu
318486b1e9 fix: verify raw freshness by received time 2026-07-02 09:16:45 +08:00
lingniu
0a024f9a76 chore: wait for gateway spool before acceptance 2026-07-02 09:11:50 +08:00
lingniu
ea9d9f5d35 chore: add go native ecs deploy script 2026-07-02 08:50:19 +08:00
lingniu
409f55b5ae test: verify go native release layout 2026-07-02 08:47:05 +08:00
lingniu
8def635bf5 test: verify realtime snapshots are fresh 2026-07-02 08:43:27 +08:00
lingniu
c211447747 test: verify daily metric formulas 2026-07-02 08:38:32 +08:00
lingniu
1741aacaaa test: verify history rows link raw frames 2026-07-02 08:35:34 +08:00
lingniu
27f4ad3468 test: verify raw parsed json in production smoke 2026-07-02 02:00:50 +08:00
lingniu
601150b364 test: verify gateway spool is drained 2026-07-02 01:57:53 +08:00
lingniu
5b7f386c5a test: verify go services are enabled 2026-07-02 01:55:39 +08:00
lingniu
f4d9c9ef5a test: add go production acceptance smoke 2026-07-02 01:53:43 +08:00
lingniu
96514898eb test: add go systemd production smoke 2026-07-02 01:51:41 +08:00
lingniu
0ef88e1a4c test: add kafka production smoke 2026-07-02 01:49:02 +08:00
lingniu
0fd4d319f9 docs: record go kafka production topics 2026-07-02 01:44:27 +08:00
lingniu
4e8434c511 test: verify mqtt history tables in production smoke 2026-07-02 01:41:08 +08:00
lingniu
576644eb65 test: verify gb32960 history tables in production smoke 2026-07-02 01:39:45 +08:00
lingniu
9bd48c5781 test: require fresh raw data in production smoke 2026-07-02 01:38:16 +08:00
lingniu
2214abe9c5 test: include realtime in production smoke 2026-07-02 01:36:14 +08:00
lingniu
1477b997c9 test: add go native production smoke 2026-07-02 01:34:28 +08:00
lingniu
4ea8d18972 docs: document go native ecs deployment 2026-07-02 01:31:58 +08:00
lingniu
947e645090 fix: query history with utc tdengine filters 2026-07-02 01:28:45 +08:00
lingniu
78b805e3a2 fix: normalize timezone history filters 2026-07-02 01:24:51 +08:00
lingniu
a7b24f4c0b feat: expose realtime snapshot protocols 2026-07-02 01:23:03 +08:00
lingniu
eb124d929b fix: accept datetime-local history filters 2026-07-02 01:19:19 +08:00
lingniu
4711287655 fix: return empty arrays for query APIs 2026-07-02 01:16:48 +08:00
lingniu
2c224396c9 fix: return real pagination totals 2026-07-02 01:14:06 +08:00
lingniu
4f2cded760 fix: treat yutong mqtt mileage as meters 2026-07-02 01:08:59 +08:00
lingniu
7ad1a25e8e fix: normalize yutong mqtt mileage units 2026-07-02 01:04:14 +08:00
lingniu
3ee1d6f52b fix: stabilize yutong mqtt ingestion 2026-07-02 01:01:33 +08:00
lingniu
6d1d0aa85e feat: write protocol acknowledgements from go gateway 2026-07-02 00:54:31 +08:00
lingniu
055373c405 fix: support extended gb32960 platform login 2026-07-02 00:47:01 +08:00
lingniu
e37dd39c89 fix: enrich jt808 location additional parsing 2026-07-02 00:36:51 +08:00
lingniu
dd73fdbb5b feat: add vehicle mileage history query 2026-07-02 00:29:49 +08:00
lingniu
fb9ab8525a feat: add vehicle location history query 2026-07-02 00:27:20 +08:00
lingniu
cc89e0d537 fix: filter raw frames by vehicle key 2026-07-02 00:24:53 +08:00
lingniu
f3ecf430cc fix: cache realtime snapshots by vehicle key 2026-07-02 00:21:50 +08:00
lingniu
bcf7cdae2e fix: key daily metrics by vehicle identity 2026-07-02 00:18:35 +08:00
lingniu
9045b871d6 fix: align jt808 parser with protocol variants 2026-07-02 00:14:02 +08:00
lingniu
b0150a5503 docs: record yutong mqtt eof diagnosis 2026-07-02 00:04:23 +08:00
lingniu
8f3b337fd6 docs: add native go deployment runbook 2026-07-02 00:02:21 +08:00
lingniu
594ab2d9c7 feat: support yutong mqtt tls config 2026-07-01 23:50:30 +08:00
lingniu
229ffcf61f docs: record go production port verification 2026-07-01 23:46:33 +08:00
lingniu
593713283c fix: query tdengine raw frames with literals 2026-07-01 23:42:09 +08:00
lingniu
6bee0499ae feat: expose raw frame query api 2026-07-01 23:39:10 +08:00
lingniu
59c1c9df0f docs: record daily metric api verification 2026-07-01 23:35:50 +08:00
lingniu
56f4811c68 fix: format daily metric query timestamps 2026-07-01 23:33:03 +08:00
lingniu
1a53011408 feat: expose daily metric query api 2026-07-01 23:30:54 +08:00
lingniu
c2d70dad86 docs: record production chain verification 2026-07-01 23:28:27 +08:00
lingniu
91db6c959e docs: record jt808 identity tracking verification 2026-07-01 23:23:59 +08:00
lingniu
7b44f97ddb fix: decode jt808 registration plate text 2026-07-01 23:21:53 +08:00
lingniu
a72977cd79 feat: track jt808 registration identity 2026-07-01 23:17:29 +08:00
lingniu
dd95101499 docs: record mileage metric guard verification 2026-07-01 23:11:35 +08:00
lingniu
d347525a75 fix: keep valid realtime mileage 2026-07-01 23:08:31 +08:00
lingniu
19008e840c fix: ignore invalid zero mileage metrics 2026-07-01 23:06:06 +08:00
lingniu
c803a19bf5 docs: record go production port switch 2026-07-01 23:03:33 +08:00
lingniu
243de5c2b2 feat: spool gateway publishes to disk 2026-07-01 23:00:06 +08:00
lingniu
4a32e2181a docs: record gateway publish retry deployment 2026-07-01 22:56:55 +08:00
lingniu
7e7066dd86 feat: retry gateway kafka publishes 2026-07-01 22:54:47 +08:00
lingniu
e7c8c57296 docs: record jt808 identity verification 2026-07-01 22:51:18 +08:00
lingniu
6b55694c43 fix: resolve jt808 phone without leading zero 2026-07-01 22:34:47 +08:00
lingniu
eb48d16181 docs: record gb32960 vendor verification 2026-07-01 22:29:46 +08:00
lingniu
a15de91912 feat: parse gb32960 guangdong fuel cell blocks 2026-07-01 22:27:34 +08:00
lingniu
796d79dd30 docs: record go gateway real frame verification 2026-07-01 22:23:44 +08:00
lingniu
5f4f4fd124 feat: parse gb32960 realtime data units 2026-07-01 22:21:21 +08:00
lingniu
e0068a750d fix: support jt808 versioned header 2026-07-01 22:12:47 +08:00
lingniu
665122b23b docs: record go gateway ecs verification 2026-07-01 22:08:45 +08:00
lingniu
60ca42e5d5 fix: quote tdengine history inserts 2026-07-01 22:06:12 +08:00
lingniu
80474eeccb feat: add go vehicle identity resolver 2026-07-01 21:57:18 +08:00
lingniu
17024651c8 chore: add go gateway deployment stack 2026-07-01 21:55:27 +08:00
lingniu
d8c77d7882 feat: add yutong mqtt ingestion 2026-07-01 21:53:39 +08:00
lingniu
b55b075a97 feat: add go realtime redis api 2026-07-01 21:51:13 +08:00
lingniu
bac8864e79 feat: add go daily metric writer 2026-07-01 21:48:52 +08:00
lingniu
5d844701ff feat: add go tdengine history writer 2026-07-01 21:47:38 +08:00
lingniu
967981c040 feat: add go tcp gateway runtime 2026-07-01 21:45:28 +08:00
lingniu
1fdbc29e27 feat: scaffold go vehicle gateway core 2026-07-01 21:43:16 +08:00
lingniu
d148a4e3af docs: design go ingest redesign phase one 2026-07-01 21:36:07 +08:00
lingniu
cde55e1ae6 docs: mark superseded implementation plans 2026-07-01 18:36:03 +08:00
lingniu
e06534d6a5 test: centralize vehicle identity fixtures 2026-07-01 18:32:09 +08:00
lingniu
4eaaa88e9c chore: remove xinda ingestion modules 2026-07-01 18:22:51 +08:00
lingniu
f89ac1c23f refactor: centralize raw frame identity derivation 2026-07-01 18:12:04 +08:00
lingniu
c8abc721e1 fix: link derived events to raw frames 2026-07-01 18:08:11 +08:00
lingniu
c8dac638b2 chore: align boot plugin jvm defaults 2026-07-01 17:48:54 +08:00
lingniu
4591919a31 docs: remove local service run instructions 2026-07-01 17:46:28 +08:00
lingniu
6e24bf615d fix: disable netty native transport for redis clients 2026-07-01 17:43:29 +08:00
lingniu
96b8474182 chore: align analytics port with ecs runtime 2026-07-01 17:31:47 +08:00
lingniu
fe4ed5e88f chore: align observability and redis defaults 2026-07-01 17:26:10 +08:00
lingniu
71353a3cb4 fix: order redis session auto configuration 2026-07-01 17:14:52 +08:00
lingniu
37844e4425 chore: remove local service runners 2026-07-01 16:56:46 +08:00
lingniu
ec4aed3ca9 chore: expose history dlq in launchctl 2026-07-01 16:43:10 +08:00
lingniu
6ac77b2f7d chore: separate kafka consumer dlq topic 2026-07-01 16:30:51 +08:00
lingniu
5deadd06af chore: trim history kafka topic defaults 2026-07-01 16:22:11 +08:00
lingniu
ccf77b4596 docs: clarify default data flow 2026-07-01 16:19:03 +08:00
lingniu
f4ce4557c0 docs: mark old deployment adr superseded 2026-07-01 16:16:16 +08:00
lingniu
2e6a982af5 chore: decouple portainer consumer startup 2026-07-01 16:13:24 +08:00
lingniu
68253901cf chore: decouple history gb32960 decoding 2026-07-01 16:10:31 +08:00
lingniu
fc81693285 chore: trim active app store toggles 2026-07-01 16:01:54 +08:00
lingniu
61cc15ab07 docs: remove legacy bootstrap app reference 2026-07-01 15:56:09 +08:00
lingniu
b7bb93de8e chore: align readme concurrency model 2026-07-01 15:53:31 +08:00
lingniu
5971d1a8c3 chore: document local archive storage only 2026-07-01 15:48:15 +08:00
lingniu
c359a0683e chore: align readme build tooling 2026-07-01 15:44:56 +08:00
lingniu
bf6f94afbb chore: trim observability surface docs 2026-07-01 15:42:55 +08:00
lingniu
4d9779e088 chore: clarify redis-only session store 2026-07-01 15:39:24 +08:00
lingniu
80c051e056 chore: enforce kafka-only messaging surface 2026-07-01 15:36:44 +08:00
lingniu
53f8f79c3e chore: keep swagger on api apps only 2026-07-01 15:32:17 +08:00
lingniu
dc56a664f4 chore: limit observability to ingest apps 2026-07-01 15:25:57 +08:00
lingniu
9cd5c5a35d chore: derive history kafka groups in app 2026-07-01 15:22:14 +08:00
lingniu
192c5eff00 chore: hide history specialized api switch 2026-07-01 15:18:33 +08:00
lingniu
329f106e3c chore: simplify analytics stat binding 2026-07-01 15:16:12 +08:00
lingniu
15c57c33b9 chore: keep legacy xinda kafka only 2026-07-01 15:12:56 +08:00
lingniu
664a0c801b chore: scope kafka topics in portainer stack 2026-07-01 15:08:57 +08:00
lingniu
4ea4469640 chore: drop unused tdengine launchctl flag 2026-07-01 15:06:00 +08:00
lingniu
a6d0236fc4 chore: scope mysql env in portainer stack 2026-07-01 15:03:59 +08:00
lingniu
3161e98eff chore: scope kafka consumer env to consumers 2026-07-01 15:01:38 +08:00
lingniu
5910d23dd9 chore: remove in-memory identity store 2026-07-01 14:58:39 +08:00
lingniu
faeee0c2a8 chore: keep active kafka path always on 2026-07-01 14:55:56 +08:00
lingniu
90791d28bd chore: fix active identity store to mysql 2026-07-01 14:51:46 +08:00
lingniu
3795e43a6a chore: fix ingest session store to redis 2026-07-01 14:47:12 +08:00
lingniu
aa30a413c5 docs: document yutong launchctl mqtt tuning 2026-07-01 14:44:24 +08:00
lingniu
0d77f7777f chore: align yutong launchctl mqtt tuning 2026-07-01 14:41:52 +08:00
lingniu
b979d7a020 chore: expose yutong mqtt portainer tuning 2026-07-01 14:38:43 +08:00
lingniu
f7bb5b1554 chore: enforce kafka-only stat path 2026-07-01 14:35:58 +08:00
lingniu
cd12a84e51 chore: trim inactive app toggles 2026-07-01 14:28:08 +08:00
lingniu
1eed7623d1 chore: simplify analytics app defaults 2026-07-01 14:24:33 +08:00
lingniu
1cca142550 chore: decouple analytics portainer startup 2026-07-01 14:21:41 +08:00
lingniu
a6784bc8c3 fix: reject removed telemetry fields switch 2026-07-01 14:17:51 +08:00
lingniu
eeba8fb754 docs: align history and analytics production surface 2026-07-01 14:13:14 +08:00
lingniu
476d5a7a14 build: centralize active app boot packaging 2026-07-01 14:08:52 +08:00
lingniu
c3d67e4339 refactor: remove telemetry fields from tdengine history 2026-07-01 14:03:32 +08:00
lingniu
68f45ee79c refactor: remove telemetry field api from history app 2026-07-01 13:53:19 +08:00
lingniu
a5cb0c9705 docs: clarify kafka-only sink semantics 2026-07-01 13:44:23 +08:00
lingniu
932b386c69 fix: use frame protocol for metrics timing 2026-07-01 13:40:25 +08:00
lingniu
981879136f fix: tighten ingest metrics protocol labels 2026-07-01 13:38:10 +08:00
lingniu
27f9cbf0d2 build: remove raw archive store prototype 2026-07-01 13:33:24 +08:00
lingniu
cea8ceeabe docs: simplify jt808 mileage runtime state 2026-07-01 13:29:08 +08:00
lingniu
97ea77ed27 build: scope private repository to legacy xinda 2026-07-01 13:25:21 +08:00
lingniu
85013a0ebe build: remove unused testcontainers dependency 2026-07-01 13:18:07 +08:00
lingniu
57ab57c42b build: remove unused archunit dependency 2026-07-01 13:15:05 +08:00
lingniu
28bde233a1 build: remove unused lombok dependency 2026-07-01 13:11:55 +08:00
lingniu
4134824caf build: remove unused opentelemetry bom 2026-07-01 13:09:22 +08:00
lingniu
422a24684d build: remove unused mqtt client dependency 2026-07-01 13:06:39 +08:00
lingniu
78c39036cd deploy: render launchctl templates safely 2026-07-01 13:03:17 +08:00
lingniu
495ded0547 deploy: add launchctl templates for active apps 2026-07-01 12:56:41 +08:00
lingniu
252aff984c docs: align tdengine verification with active apps 2026-07-01 12:46:24 +08:00
lingniu
dc481e8653 ci: parameterize docker image prefix 2026-07-01 12:43:13 +08:00
lingniu
9fb77a1aa2 deploy: parameterize portainer image prefix 2026-07-01 12:40:25 +08:00
lingniu
afed6cd4f3 docs: state kafka-only production channel 2026-07-01 12:37:17 +08:00
lingniu
fd76bfdf61 docs: align readme quick start with production apps 2026-07-01 12:34:07 +08:00
lingniu
3185696b6a docs: align tdengine build skip flag 2026-07-01 12:31:43 +08:00
lingniu
641114696c ci: skip test compilation in woodpecker build 2026-07-01 12:29:29 +08:00
lingniu
2deea2d072 ci: reuse active app list in woodpecker 2026-07-01 12:27:21 +08:00
lingniu
3d9a520c3c docs: clarify kafka-only production channel 2026-07-01 12:24:25 +08:00
lingniu
e81d8dcab5 docs: align launchctl health probe checks 2026-07-01 12:19:26 +08:00
lingniu
a5b39e22b6 deploy: share portainer readiness defaults 2026-07-01 12:17:23 +08:00
lingniu
bc60017c30 deploy: add portainer readiness healthchecks 2026-07-01 12:14:45 +08:00
lingniu
5ff7cfb9bb docs: add health probe checks to runbook 2026-07-01 12:11:14 +08:00
lingniu
59edc7a8d4 config: expose health probes for active apps 2026-07-01 12:08:54 +08:00
lingniu
a76cd3a8d6 deploy: exec java in container entrypoint 2026-07-01 12:04:04 +08:00
lingniu
2d65a2555f deploy: align container java defaults 2026-07-01 12:01:47 +08:00
lingniu
5ede13e6d4 deploy: persist ingest archive paths 2026-07-01 11:59:11 +08:00
lingniu
5ec3be97aa deploy: align jt808 launchctl port 2026-07-01 11:56:10 +08:00
lingniu
12423efd64 docs: align active ingest runbooks 2026-07-01 11:53:21 +08:00
lingniu
5beb81f8cd docs: clarify ingest and consumer persistence roles 2026-07-01 11:45:59 +08:00
lingniu
77d5aa4ef7 refactor: narrow active app component scans 2026-07-01 11:41:02 +08:00
lingniu
c6347c50a4 fix: keep telemetry fields out of default history api 2026-07-01 11:32:59 +08:00
lingniu
d4c54b5d0f refactor: centralize default history protocols 2026-07-01 11:28:16 +08:00
lingniu
ebbf2ca8fb docs: clarify kafka-only production channel 2026-07-01 11:25:00 +08:00
lingniu
61ad4a2439 chore: clarify current history path naming 2026-07-01 11:21:11 +08:00
lingniu
986fc2fac3 refactor: remove event file store contracts 2026-07-01 11:18:02 +08:00
lingniu
7ce64e9963 refactor: remove legacy event file store 2026-07-01 11:04:11 +08:00
lingniu
e80fa6f38c refactor: rename kafka sink module 2026-07-01 10:56:33 +08:00
lingniu
849e8f8a8b refactor: expose kafka sink configuration 2026-07-01 10:50:50 +08:00
lingniu
303b2eb47e refactor: enforce kafka mq sink type 2026-07-01 10:38:17 +08:00
lingniu
e2b6ab63d6 refactor: remove legacy session store enum 2026-07-01 10:35:12 +08:00
lingniu
87830fee09 refactor: make session store redis by default 2026-07-01 10:31:26 +08:00
lingniu
53a9b3b3b7 docs: clarify jt808 mileage metric storage 2026-07-01 10:30:53 +08:00
lingniu
163f313066 refactor: simplify ingest app boundaries 2026-07-01 10:23:12 +08:00
lingniu
7b0b69295f refactor: enforce mysql identity store 2026-07-01 10:18:00 +08:00
lingniu
74e01acc7a perf: reuse mqtt identity resolution 2026-07-01 10:12:52 +08:00
lingniu
418aefc189 refactor: centralize telemetry metadata formatting 2026-07-01 10:05:36 +08:00
lingniu
da17acb09a fix: preserve mileage metadata on location events 2026-07-01 10:00:38 +08:00
lingniu
136b8a040c fix: expose jt808 mileage metadata 2026-07-01 09:55:44 +08:00
lingniu
0236884f81 fix: calculate jt808 daily mileage from metric endpoints 2026-07-01 09:52:24 +08:00
lingniu
bb051391c2 fix: align jt808 protocol default port 2026-07-01 09:48:55 +08:00
lingniu
ce93236a5a build: expose current jt808 docker port 2026-07-01 09:45:09 +08:00
lingniu
ba1ec9a012 build: scope optional module versions to profiles 2026-07-01 09:42:15 +08:00
lingniu
072fa00ea9 docs: document default portainer stack boundary 2026-07-01 09:38:25 +08:00
lingniu
74fbdd2b1a docs: align jt808 mileage metric design 2026-07-01 09:36:57 +08:00
lingniu
d9329f4364 docs: clarify optional command gateway modules 2026-07-01 09:31:30 +08:00
lingniu
bdc035185e docs: mark stale gb32960 designs superseded 2026-07-01 09:28:59 +08:00
lingniu
d491f543b5 docs: align latest state design with redis only 2026-07-01 09:26:27 +08:00
lingniu
5d082d50d5 docs: document redis session prerequisites 2026-07-01 09:24:05 +08:00
lingniu
6faf3533a5 docs: remove memory session guidance 2026-07-01 09:19:26 +08:00
lingniu
ce3f2ffb90 refactor: remove production in-memory session store 2026-07-01 09:15:28 +08:00
lingniu
42e8742b10 refactor: require identity resolver for jt1078 malformed rtp 2026-07-01 09:07:12 +08:00
lingniu
a321b6351d docs: align readme with redis session runtime 2026-07-01 09:00:10 +08:00
lingniu
7c51c83220 refactor: remove generic mqtt enable alias 2026-07-01 08:56:59 +08:00
lingniu
9db8b15ae3 refactor: remove mqtt legacy endpoint fallback 2026-07-01 08:50:23 +08:00
lingniu
2ab1eeb1f8 refactor: require mqtt identity resolver 2026-07-01 08:45:29 +08:00
lingniu
dcd5f2a1a2 refactor: reject legacy identity store modes 2026-07-01 08:38:50 +08:00
lingniu
625d09c856 refactor: remove implicit memory session store 2026-07-01 08:36:07 +08:00
lingniu
f80c47df0f refactor: reject legacy jt808 mileage state store 2026-07-01 08:32:51 +08:00
lingniu
a08a132a3a test: cover jt808 gps mileage extension 2026-07-01 08:29:21 +08:00
lingniu
bb6ffa86fd chore: stop exposing actuator env by default 2026-07-01 08:24:09 +08:00
lingniu
62ef3f5bc1 refactor: restrict default raw history protocols 2026-07-01 08:20:18 +08:00
lingniu
8c61ae7139 chore: add yuejin gb32960 platform config 2026-07-01 08:15:55 +08:00
lingniu
37902be034 chore: expose telemetry fields switch in history compose 2026-07-01 08:08:30 +08:00
lingniu
e745419f76 refactor: hide telemetry field api by default 2026-07-01 08:04:59 +08:00
lingniu
80d3afa80a refactor: simplify jt808 daily mileage strategy 2026-07-01 08:01:12 +08:00
lingniu
59e8315fe6 refactor: avoid file mapper in tdengine history path 2026-07-01 07:55:20 +08:00
lingniu
56a6c01f95 refactor: make event file store optional 2026-07-01 07:51:45 +08:00
lingniu
51b1dd4174 refactor: make raw archive store optional 2026-07-01 07:42:16 +08:00
lingniu
ba4bb51a07 refactor: make latest state optional 2026-07-01 07:37:50 +08:00
lingniu
0987bb4718 refactor: make jsatl12 optional 2026-07-01 07:33:06 +08:00
lingniu
4f83cd2792 refactor: share portainer ingress env 2026-07-01 07:23:20 +08:00
lingniu
e47e3eebb4 refactor: simplify history and mileage runtime 2026-07-01 07:18:06 +08:00
lingniu
cf4d369335 docs: clarify event file store compatibility role 2026-07-01 07:08:44 +08:00
lingniu
9bd61ac91b docs: align adr with active production scope 2026-07-01 07:06:09 +08:00
lingniu
18aa32a721 refactor: keep kafka clients behind sink mq 2026-07-01 07:02:44 +08:00
lingniu
40dea58278 refactor: keep taos driver behind history store 2026-07-01 06:55:49 +08:00
lingniu
e03d1d7629 fix: remove legacy commons logging from history runtime 2026-07-01 06:52:21 +08:00
lingniu
6ccf8fdc26 fix: stop kafka consumers without concurrent close 2026-07-01 06:48:33 +08:00
lingniu
442898cc0b test: guard apps against build-only modules 2026-07-01 06:44:42 +08:00
lingniu
ef1fd6e4f5 perf: avoid global lock in jt808 mileage processor 2026-07-01 06:40:48 +08:00
lingniu
7c3f25eaf6 chore: align pom metadata with runtime stack 2026-07-01 06:36:41 +08:00
lingniu
d8f8235544 docs: align readme with runtime stack 2026-07-01 06:33:43 +08:00
lingniu
be2daf986a docs: align module flow with split apps 2026-07-01 06:30:08 +08:00
300 changed files with 17990 additions and 8468 deletions

View File

@@ -54,13 +54,16 @@
V2016/V2025 双版本信息体 parser、平台登入鉴权、VIN 白名单、读空闲告警。
- **JT/T 808 / JT/T 1078 / JSATL12 接入**`protocol-jt808` / `protocol-jt1078` /
`protocol-jsatl12`)。
- **MQTT 入站**与**信达平台 push 接入**`inbound-mqtt` / `inbound-xinda-push`)。
- **MQTT 入站**`inbound-mqtt`)。
- **Disruptor 事件总线**`ingest-core`):高吞吐解耦解码与下游 sink。
- **Sink**:本地归档(`sink-archive`、Kafka`sink-mq`,含 protobuf envelope
- **Sink**:本地归档(`sink-archive`、Kafka`sink-kafka`,含 protobuf envelope
- **会话状态**`session-core`)。
- **可观测性**Micrometer/Actuator 装配(`observability`)。
- **终端控制命令网关**`command-gateway`)。
### Removed
- 旧文件型事件索引模块和 DuckDB 依赖;历史查询统一收敛到 TDengine 与 `archive://...` 原始报文引用。
### Fixed —— GB/T 32960 应答帧时间字段
- `Gb32960MessageDecoder``VEHICLE_LOGIN` / `VEHICLE_LOGOUT` /
`PLATFORM_LOGIN` / `PLATFORM_LOGOUT` 命令预解析 body 首 6B 采集时间,写入

View File

@@ -2,11 +2,11 @@
> 本文件记录 lingniu-vehicle-ingest v2 的关键架构决策,后续每次重大调整追加条目(不删除旧条目)。
## ADR-001 MQ 选型Kafka
## ADR-001 消息通道Kafka
- **Status**: Accepted
- **Context**: 需要高吞吐、严格单车有序、成熟生态
- **Decision**: 使用 Kafka分区 key = VIN
- **Consequences**: 下游消费者需使用消费组 + 分区内顺序消费;不使用 RocketMQ 的事务/顺序特性
- **Consequences**: 下游消费者需使用消费组 + 分区内顺序消费;生产链路只支持 Kafka。
## ADR-002 线上消息格式Protobuf
- **Status**: Accepted
@@ -20,27 +20,22 @@
- **Decision**: 拆出独立 `command-gateway` 模块,复用 `session-core`;本次重构同步交付
- **Consequences**: HTTP 对外接口(原 JT808Controller / JT1078Controller迁移到此模块
## ADR-004 信达 Push保留模块但彻底重写
- **Status**: Accepted
## ADR-004 信达 PushRemoved
- **Status**: Superseded by ADR-011
- **Context**: 旧实现硬编码凭证、无重连、0300/0401 空实现
- **Decision**:
1. 新模块 `inbound-xinda-push` 隔离第三方库 `gps-push-client`
2. 配置外置yml + env
3. 自动重连 + Resilience4j 熔断
4. 0200/0300/0401 **仅做协议解析**,转换为 `VehicleEvent` 后直接投 Kafka
5. **业务处理全部下沉到下游消费者**,本服务不再实现报警/透传业务逻辑
- **Consequences**: 旧代码 messageReceived 里的 TODO 不再回到本仓库;需要协调下游消费者团队认领
- **Decision**: 删除信达 Push 源码、Maven profile、私仓依赖和部署入口。
- **Consequences**: 信达 Push 后续已从源码和构建面删除,优化优先投向 GB32960、JT808、宇通 MQTT、历史和统计链路。
## ADR-005 JT1078 / JSATL12 进第一批
- **Status**: Accepted
- **Status**: Superseded by ADR-012
- **Decision**: 第一批迁移即覆盖 JT1078 信令 + JSATL12 报警附件上传
- **Consequences**: Phase 2 工期相应拉长JSATL12 需要对象存储后端(本地 / S3 / OSS
## ADR-006 部署形态GB32960 三应用拆分
- **Status**: Superseded
- **Status**: Superseded by ADR-011
- **Context**: 原 `bootstrap-all` 一体化启动已被 GB32960 拆分部署替代
- **Decision**: 生产默认部署 `gb32960-ingest-app``vehicle-history-app``vehicle-analytics-app` 三个应用;旧 `bootstrap-all` 模块删除
- **Consequences**: CI/CD 分别构建三个镜像;协议接入、历史查询、统计消费可以独立发布和回滚
- **Decision**: 当时生产默认部署 `gb32960-ingest-app``vehicle-history-app``vehicle-analytics-app`;旧 `bootstrap-all` 模块删除
- **Consequences**: 该三应用边界已由 ADR-011 的三协议接入 + 历史 + 统计生产面取代;协议接入、历史查询、统计消费仍保持独立发布和回滚
## ADR-007 Java 25 + 虚拟线程 + Disruptor
- **Status**: Accepted
@@ -51,15 +46,60 @@
- 禁用 `synchronized`,使用 `ReentrantLock` 避免虚拟线程 pinning
- **Consequences**: 需开启 `jdk.VirtualThreadPinned` JFR 监控
## ADR-008 本服务不写业务库
## ADR-008 协议接入应用不直接写业务库
- **Status**: Accepted
- **Decision**: 所有业务表vehicle_data_actual / history / travel / gps_data 等)不再由本服务写入
- **Consequences**: 需要新建消费服务 `vehicle-analytics-service` 承接;迁移期采用双跑对账
- **Decision**: GB32960、JT808、Yutong MQTT 等协议接入应用只负责收、解析、校验、规整和投递 Kafka不直接写业务表。
- **Consequences**: 业务落库由独立 Kafka 消费应用承接:`vehicle-history-app` 写入 TDengine 历史与 RAW JSON`vehicle-analytics-app` 写入 MySQL `vehicle_stat_metric`
## ADR-009 构建工具Maven
- **Status**: Accepted
- **Decision**: 沿用 Maven统一 BOM 管理版本Spotless 格式化ArchUnit 守护分层
## ADR-010 框架Spring Boot 3.4
## ADR-010 框架Spring Boot 3.5
- **Status**: Accepted
- **Decision**: Spring Boot 3.4.x支持 JDK 25AutoConfiguration 用于按需启停
- **Decision**: Spring Boot 3.5.x支持 JDK 25AutoConfiguration 用于按需启停
## ADR-011 默认生产面:三协议接入 + 历史 + 统计
- **Status**: Accepted
- **Context**: 生产接入范围已经从 GB32960 三应用拆分,扩展为三条活跃接入链路和两个消费应用;信达 Push 已废弃并删除,不应再出现在构建、部署或优化目标里。
- **Decision**:
1. 默认生产面包含 GB32960、JT808、Yutong MQTT、vehicle-history-app、vehicle-analytics-app。
2. Xinda Push 源码、Maven profile、Woodpecker 镜像发布和历史消费绑定全部移除。
3. `command-gateway` 和 JT1078 继续作为可选能力,通过 `optional-command-gateway` profile 显式启用。
- **Consequences**: 默认构建和部署保持精简;后续性能、可靠性、字段解析和存储优化都服务三条活跃协议链路。
## ADR-012 JSATL12 附件上传Optional only
- **Status**: Accepted
- **Context**: 当前默认生产面只部署 GB32960、JT808、Yutong MQTT、vehicle-history-app、vehicle-analytics-appJSATL12 附件上传没有独立生产 app也不在 Portainer 和 Woodpecker 活跃镜像列表中。
- **Decision**:
1. `protocol-jsatl12` 不进入默认 Maven reactor。
2. JSATL12 仅保留在 `optional-attachments` profile需要附件上传能力时显式构建。
3. 默认优化和验证优先覆盖活跃接入链路,附件上传能力保持源码可用但不增加默认构建面。
- **Consequences**: 默认构建更轻,生产部署边界更清晰;附件上传相关测试需要通过 `-Poptional-attachments` 显式运行。
## ADR-013 最新状态服务Optional only
- **Status**: Accepted
- **Context**: Redis 最新状态查询是独立消费能力,但当前默认 Portainer 部署和 Woodpecker 镜像发布只包含三条接入链路、history 和 analytics`vehicle-state-service` 没有独立 app也不应增加默认构建面。
- **Decision**:
1. `vehicle-state-service` 不进入默认 Maven reactor。
2. vehicle-state-service 仅保留在 `optional-latest-state` profile需要 Redis 最新状态查询能力时显式构建。
3. 默认优化和验证优先覆盖活跃接入、TDengine 历史和 MySQL 指标链路。
- **Consequences**: 默认构建和部署边界继续收窄;最新状态能力保持源码可用,但其测试需要通过 `-Poptional-latest-state` 显式运行。
## ADR-014 raw-archive-store 原型已删除
- **Status**: Accepted
- **Context**: 默认生产 raw bytes 写入由 `sink-archive` 负责,历史查询通过 TDengine `raw_frames``archive://...` 引用追溯;`raw-archive-store` 只是未部署的独立读写原型,继续保留会造成 raw archive 路径歧义。
- **Decision**:
1. 删除 `raw-archive-store` 模块和对应 optional profile。
2. 默认生产链路只保留 `sink-archive`、Kafka raw topic、TDengine raw_frames 这条 raw 路径。
3. 如需新的 archive store 能力,先以生产链路需求重新设计,不恢复旧原型。
- **Consequences**: 默认构建面和可选构建面继续收窄raw bytes 写入职责集中在 `sink-archive`
## ADR-015 文件型事件索引Removed
- **Status**: Accepted
- **Context**: 默认历史查询已收敛到 TDengine `raw_frames``vehicle_locations` 和按需解码;旧文件型索引会增加一套无生产部署的查询和依赖边界。
- **Decision**:
1. 删除旧文件型事件索引模块和 profile。
2. 父 POM 不再管理旧索引驱动依赖。
3. 历史查询和 RAW 回放统一以 TDengine + `archive://...` 引用为准。
- **Consequences**: 默认构建面和可选构建面都更小;需要历史查询时只维护 TDengine 一条路径。

View File

@@ -5,11 +5,15 @@ ARG APP_VERSION
ENV APP_NAME=${APP_NAME}
ENV APP_VERSION=${APP_VERSION}
ENV JAVA_OPTS=""
ENV JAVA_OPTS="--sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true"
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY modules/apps/${APP_NAME}/target/${APP_NAME}.jar /app/app.jar
EXPOSE 20100 20200 20300 20400 20500 32960 10808
EXPOSE 20100 20200 20310 20400 20500 32960 808
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app/app.jar"]
ENTRYPOINT ["sh", "-c", "exec java $JAVA_OPTS -jar /app/app.jar"]

View File

@@ -4,9 +4,9 @@
## 设计目标
- **协议接入统一抽象**GB/T 32960、JT/T 808、JT/T 1078、JSATL12、MQTT 全部变成可插拔 Inbound Adapter信达 Push 仅保留为废弃兼容模块
- **协议接入统一抽象**GB/T 32960、JT/T 808、Yutong MQTT 是默认生产接入JT/T 1078、JSATL12 作为显式 profile 的可选能力;信达 Push 已废弃并删除源码
- **原子能力化**:每个协议一个独立 Maven 模块 + 独立 AutoConfiguration + 独立配置开关
- **业务 / 实时彻底解耦**本服务只负责 **收 → 解析 → 校验 → 规整 → 投递 Kafka**,不碰业务
- **业务 / 实时彻底解耦**协议接入应用只负责收、解析、校验、规整和投递 Kafka历史与统计由独立 Kafka 消费应用落
- **高并发低延迟**Netty + Disruptor + Java 25 虚拟线程;目标单节点 ≥ 5 万 msg/sP99 < 50 ms
- **可观测、可回放、可灰度**:统一 traceId、Envelope、幂等键、DLQ、原始报文冷存
@@ -15,17 +15,17 @@
| 类别 | 选型 |
|---|---|
| 语言 | Java **25** |
| 框架 | Spring Boot 3.4.x |
| 网络 | Netty 4.1 |
| 并发 | Disruptor 4 + 虚拟线程 + `StructuredTaskScope` |
| 消息队列 | **Kafka**vin 分区,保证单车有序) |
| 框架 | Spring Boot 3.5.3 |
| 网络 | Netty 4.2.9.Final |
| 并发 | Disruptor 4 + 虚拟线程 |
| 事件流 | **Kafka**唯一生产消息通道,vin 分区,保证单车有序) |
| 线上消息格式 | **Protobuf**,调试走 JSON |
| MQTT 客户端 | HiveMQ MQTT Client |
| 会话缓存 | Redis 生产索引 + Caffeine 内存降级 |
| MQTT 客户端 | Eclipse Paho 1.2.5 |
| 会话索引 | Redis |
| 熔断/限流 | Resilience4j 2 |
| 可观测 | Micrometer + Prometheus + OpenTelemetry |
| 冷存 | S3 / OSS / 本地 |
| 构建 | Maven + Spotless + ArchUnit |
| 可观测 | Micrometer + Prometheus |
| 冷存 | 本地文件系统 |
| 构建 | Maven |
## 模块划分
@@ -36,32 +36,32 @@ lingniu-vehicle-ingest/
│ │ ├── ingest-api/ SPI + sealed 领域事件 + 注解
│ │ ├── ingest-codec-common/ 公共编解码工具BCD/CRC/BCC/bit utils
│ │ ├── ingest-core/ Pipeline / Dispatcher / Disruptor / Session 桥
│ │ ├── session-core/ 设备会话 + 鉴权 + Token + Redis/Memory SessionStore
│ │ ├── session-core/ 设备会话 + 鉴权 + Token + Redis SessionStore
│ │ ├── vehicle-identity/ 跨协议车辆身份解析 + MySQL 外部标识绑定
│ │ └── observability/ metrics / tracing / health
│ │ └── observability/ metrics / health
│ ├── protocols/
│ │ ├── protocol-gb32960/ GB/T 32960
│ │ ├── protocol-jt808/ JT/T 808统一身份映射 + 事件 metadata 内部 VIN + 注册/鉴权/心跳/注销/位置/批量位置/参数/属性/媒体/透传/未知上行和坏帧兜底/断链清理会话/下行分包)
│ │ ├── protocol-jt1078/ JT/T 1078808 信令按需桥接 + 常用下行信令编码 + TCP/UDP RTP 媒体流分段归档 + 事件 metadata 内部 VIN + 坏 RTP 统一 RawArchive/Passthrough + 归档失败兜底 + SIM 身份映射)
│ │ └── protocol-jsatl12/ 苏标主动安全报警附件(真实 DataPacket 分帧 + 附件流归档 + T1210 文件清单身份继承 + 9212 补传应答 + MediaMeta 引用事件 + 坏帧/坏信令/归档失败兜底
│ │ ├── protocol-jt1078/ JT/T 1078optional-command-gateway profile808 信令按需桥接 + 常用下行信令编码 + TCP/UDP RTP 媒体流分段归档 + 事件 metadata 内部 VIN + 坏 RTP 统一 RawArchive/Passthrough + 归档失败兜底 + SIM 身份映射)
│ │ └── protocol-jsatl12/ 苏标主动安全报警附件(optional-attachments profile
│ ├── inbound/
│ │ ── inbound-mqtt/ MQTT 接入endpoint 生命周期 + profile 注册扩展 + 统一身份映射 + PEM 双向 TLS + 未知 profile/解析失败/profile异常/连接订阅失败兜底 + 统一 UNKNOWN 身份 metadata
│ │ └── inbound-xinda-push/ 信达 Push 接入(废弃兼容模块,仅 -Plegacy-xinda 显式构建)
│ │ ── inbound-mqtt/ MQTT 接入endpoint 生命周期 + profile 注册扩展 + 统一身份映射 + PEM 双向 TLS + 未知 profile/解析失败/profile异常/连接订阅失败兜底 + 统一 UNKNOWN 身份 metadata
│ ├── sinks/
│ │ ├── sink-mq/ Kafka producer + Protobuf Envelope
│ │ ── sink-archive/ 原始报文冷存
│ │ └── event-file-store/ Parquet + DuckDB 文件型明细库
│ │ ├── sink-kafka/ Kafka producer + Protobuf Envelope
│ │ ── sink-archive/ 原始报文冷存
│ ├── services/
│ │ ├── event-history-service/ Kafka 全字段事件消费 + 历史查询/导出
│ │ ├── vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询
│ │ ├── event-history-service/ Kafka 全字段事件消费 + 历史查询
│ │ ├── vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询optional-latest-state profile
│ │ └── vehicle-stat-service/ Kafka 全字段事件消费 + 可配置日统计
│ ├── testing/
│ │ └── vehicle-identity-test-support/ 协议测试共享身份解析夹具
│ └── apps/
│ ├── command-gateway/ HTTP → 设备下行命令(808 位置/参数/属性/控制/区域删除/报警确认 + 1078 音视频控制
│ ├── command-gateway/ 可选 HTTP → 设备下行命令(optional-command-gateway profile
│ ├── gb32960-ingest-app/ GB32960 TCP 接入 + Kafka 投递
│ ├── jt808-ingest-app/ JT808 TCP 接入 + Kafka 投递
│ ├── yutong-mqtt-app/ 宇通 MQTT 接入 + Kafka 投递
│ ├── vehicle-history-app/ 历史查询 + 原始归档索引消费
│ └── vehicle-analytics-app/ 车辆状态/日统计消费
│ ├── vehicle-history-app/ TDengine 历史查询 + RAW JSON
│ └── vehicle-analytics-app/ JT808 每日里程指标消费
├── docs/ 架构文档、模块图、实施计划
└── reference/ 参考资料
```
@@ -71,12 +71,18 @@ lingniu-vehicle-ingest/
```bash
# 要求JDK 25, Maven 3.9+
mvn -v
mvn clean install -DskipTests
mvn -pl :gb32960-ingest-app -am spring-boot:run
mvn -pl :jt808-ingest-app -am spring-boot:run
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
```
信达 Push 已废弃,不参与默认 Maven reactor、CI、Portainer 部署和历史消费链路;如需兼容排查,可显式使用 `-Plegacy-xinda` 构建
生产服务只在 ECS/Portainer 上运行本机只用于源码检查、Maven 构建和单元测试。部署、健康检查和真实流量验证步骤见 `docs/operations/gb32960-service-split-runbook.md``docs/operations/vehicle-ingest-tdengine-verification.md`
信达 Push 已废弃并删除源码,不参与 Maven reactor、CI、Portainer 部署和历史消费链路。
JSATL12 附件上传不在默认 Maven reactor 中;需要时使用 `-Poptional-attachments` 构建。
Command Gateway/JT1078 下行与音视频信令不在默认 Maven reactor 中;需要时使用 `-Poptional-command-gateway` 构建。
Redis 最新状态查询不在默认 Maven reactor 中;需要时使用 `-Poptional-latest-state` 构建 `vehicle-state-service`
## 迁移说明
@@ -90,8 +96,9 @@ mvn -pl :jt808-ingest-app -am spring-boot:run
## 核心原则
1. **本服务不写业务库**:历史、行程、在线检测、里程统计全部由 Kafka 下游消费者实现
2. **协议即插拔**:每个 `protocol-*` / `inbound-*` 都可独立开关(`lingniu.ingest.<name>.enabled`
3. **顺序保证**同一 VIN 严格有序Disruptor hash + Kafka 分区 key
4. **幂等消费**Envelope 带 `eventId`,下游去重
5. **原始可回放**Dispatcher 会先发布 `RawArchive`,并把同一份 `rawArchiveKey/rawArchiveUri` 追加到业务事件 metadata下游明细库、Kafka Envelope 和导出都使用 `archive://...` 逻辑 URI 追溯原始 bytes实际文件由 `sink-archive`/`ArchiveStore` 管理;`event-file-store` 只额外保存 RawArchive 查询索引,不嵌入原始 bytes
1. **接入与业务落库分离**:协议接入应用只负责收、解析、校验、规整和投递 Kafka`vehicle-history-app` 写入 TDengine 历史与 RAW JSON`vehicle-analytics-app` 将 JT808 `daily_mileage_km` 写入 MySQL `vehicle_stat_metric`
2. **Kafka 唯一消息通道**:生产链路只支持 Kafka接入、历史、统计统一通过 Kafka Envelope 解耦
3. **协议即插拔**每个 `protocol-*` / `inbound-*` 都可独立开关(`lingniu.ingest.<name>.enabled`
4. **顺序保证**同一 VIN 严格有序Disruptor hash + Kafka 分区 key
5. **幂等消费**Envelope 带 `eventId`,下游去重
6. **原始可回放**:协议接入会把 `rawArchiveKey/rawArchiveUri` 追加到事件 metadataKafka Envelope、TDengine `raw_frames` 和导出都使用 `archive://...` 逻辑 URI 追溯原始 bytes实际文件由 `sink-archive` 管理

View File

@@ -1,95 +0,0 @@
# macOS launchctl 本机部署模板
本目录用于把本机 32960/JT808/history 三个服务的启动方式仓库化。模板面向当前生产验证链路:
- `gb32960-ingest-app`: TCP `32960`HTTP `20100`
- `jt808-ingest-app`: TCP `808`HTTP `20482`
- `vehicle-history-app`: HTTP `20200`
`vehicle-history-app` 的热历史查询只依赖 Kafka 与 TDengineRAW 帧的 `parsedJson``metadataJson``rawUri` 已进入 TDengine `raw_frames`。GB32960/JT808 接入服务可以继续保留 `SINK_ARCHIVE_PATH` 作为原始 `.bin` 冷备history 本机模板不再要求共享 archive root。
`vehicle-history-app` 的生产高吞吐模板是 TDengine-only
- 不暴露 DuckDB/EventFileStore 启动变量,避免文件型明细库进入实时消费热路径。
- `TDENGINE_TELEMETRY_FIELDS_ENABLED=false`:默认只写 `raw_frames` 和协议位置表,避免逐字段写放大。
- 通用 RAW 查询默认直接返回 TDengine `raw_frames` 中的 `parsedJson`/`parsedFields`;后续解析器新增字段后,优先通过 replay 或专用解析任务补写 TDengine而不是让 history 服务依赖本地 archive 挂载。
- 如果需要兼容旧 `/api/event-history/records` 低层记录接口或逐字段趋势宽表,可以显式打开对应开关;开启后要重新压测 Kafka lag 和 TDengine 写入吞吐。
## 生成 plist
在仓库根目录执行:
```bash
export PROJECT_ROOT="$(pwd)"
export JAVA_BIN="$(/usr/libexec/java_home 2>/dev/null)/bin/java"
export INGEST_ARCHIVE_PATH="$PROJECT_ROOT/data/archive-live"
export TDENGINE_JDBC_URL='jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts'
export TDENGINE_USERNAME='root'
export TDENGINE_PASSWORD='<tdengine-password>'
export VEHICLE_IDENTITY_MYSQL_JDBC_URL='jdbc:mysql://<mysql-host>:3306/vehicle_ingest'
export VEHICLE_IDENTITY_MYSQL_USERNAME='<mysql-user>'
export VEHICLE_IDENTITY_MYSQL_PASSWORD='<mysql-password>'
export VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL='60s'
mkdir -p "$HOME/Library/LaunchAgents" "$PROJECT_ROOT/data"
mkdir -p /tmp/lingniu-gb32960-live /tmp/lingniu-jt808-live /tmp/lingniu-history-live
for service in gb32960 jt808 vehicle-history; do
sed \
-e "s#__PROJECT_ROOT__#$PROJECT_ROOT#g" \
-e "s#__JAVA_BIN__#$JAVA_BIN#g" \
-e "s#__SHARED_ARCHIVE_PATH__#$INGEST_ARCHIVE_PATH#g" \
-e "s#__TDENGINE_JDBC_URL__#$TDENGINE_JDBC_URL#g" \
-e "s#__TDENGINE_USERNAME__#$TDENGINE_USERNAME#g" \
-e "s#__TDENGINE_PASSWORD__#$TDENGINE_PASSWORD#g" \
-e "s#__VEHICLE_IDENTITY_MYSQL_JDBC_URL__#$VEHICLE_IDENTITY_MYSQL_JDBC_URL#g" \
-e "s#__VEHICLE_IDENTITY_MYSQL_USERNAME__#$VEHICLE_IDENTITY_MYSQL_USERNAME#g" \
-e "s#__VEHICLE_IDENTITY_MYSQL_PASSWORD__#$VEHICLE_IDENTITY_MYSQL_PASSWORD#g" \
-e "s#__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__#$VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL#g" \
"deploy/local/launchctl/com.lingniu.${service}.plist.template" \
> "$HOME/Library/LaunchAgents/com.lingniu.${service}.plist"
done
```
如果生产配置由 Nacos 下发,可以把模板里的 `NACOS_CONFIG_ENABLED` 改成 `true`,并补充 `NACOS_SERVER_ADDR``NACOS_NAMESPACE``NACOS_GROUP``NACOS_USERNAME``NACOS_PASSWORD`。端口、Kafka topic、接入服务冷备 archive 路径、TDengine 连接建议仍保留为启动环境变量,便于 Portainer、launchctl 和临时压测保持一致。
history 消费者会按协议和 raw/event 自动拆分 consumer group。设置 `KAFKA_GROUP_HISTORY=vehicle-history-live` 后,实际 group 为:
- `vehicle-history-live-gb32960-event`
- `vehicle-history-live-gb32960-raw`
- `vehicle-history-live-jt808-event`
- `vehicle-history-live-jt808-raw`
本地模板与 ECS 生产口径一致,身份绑定只使用 MySQL。启动前必须提供 `VEHICLE_IDENTITY_MYSQL_*` 连接参数;服务会自动创建 `vehicle_identity_binding``vehicle_identity_binding_registration` 两张表:前者只保存 `plate``vin`,后者保存注册字段;服务按刷新周期用注册表里的车牌关联 VIN后续 raw/event 会带真实 VIN。
## 启停
```bash
launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.gb32960.plist"
launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.jt808.plist"
launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.vehicle-history.plist"
launchctl list | rg 'com.lingniu.(gb32960|jt808|vehicle-history)'
```
替换 jar 前先停止服务,避免 KeepAlive 在 jar 拷贝中途拉起半包:
```bash
launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.gb32960.plist" || true
launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.jt808.plist" || true
launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.vehicle-history.plist" || true
```
## 验证地址
```bash
curl -sS http://127.0.0.1:20100/actuator/health
curl -sS http://127.0.0.1:20482/actuator/health
curl -sS http://127.0.0.1:20200/actuator/health
```
Swagger
- GB32960 ingest: `http://127.0.0.1:20100/swagger-ui/index.html`
- JT808 ingest: `http://127.0.0.1:20482/swagger-ui/index.html`
- History query: `http://127.0.0.1:20200/swagger-ui/index.html`

View File

@@ -1,65 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.lingniu.gb32960</string>
<key>WorkingDirectory</key>
<string>__PROJECT_ROOT__</string>
<key>ProgramArguments</key>
<array>
<string>__JAVA_BIN__</string>
<string>-jar</string>
<string>__PROJECT_ROOT__/modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar</string>
</array>
<key>EnvironmentVariables</key>
<dict>
<key>HTTP_PORT</key>
<string>20100</string>
<key>GB32960_PORT</key>
<string>32960</string>
<key>GB32960_AUTH_ENABLED</key>
<string>false</string>
<key>GB32960_PLATFORM_IP_HYUNDAI</key>
<string>115.29.187.205</string>
<key>KAFKA_ENABLED</key>
<string>true</string>
<key>KAFKA_BROKERS</key>
<string>114.55.58.251:9092</string>
<key>KAFKA_NODE_ID</key>
<string>gb32960-live-32960</string>
<key>KAFKA_TOPIC_GB32960_EVENT</key>
<string>vehicle.event.gb32960.v1</string>
<key>KAFKA_TOPIC_GB32960_RAW</key>
<string>vehicle.raw.gb32960.v1</string>
<key>KAFKA_TOPIC_GB32960_DLQ</key>
<string>vehicle.dlq.gb32960.v1</string>
<key>SINK_ARCHIVE_PATH</key>
<string>__SHARED_ARCHIVE_PATH__</string>
<key>VEHICLE_IDENTITY_STORE</key>
<string>mysql</string>
<key>VEHICLE_IDENTITY_MYSQL_TABLE</key>
<string>vehicle_identity_binding</string>
<key>VEHICLE_IDENTITY_MYSQL_JDBC_URL</key>
<string>__VEHICLE_IDENTITY_MYSQL_JDBC_URL__</string>
<key>VEHICLE_IDENTITY_MYSQL_USERNAME</key>
<string>__VEHICLE_IDENTITY_MYSQL_USERNAME__</string>
<key>VEHICLE_IDENTITY_MYSQL_PASSWORD</key>
<string>__VEHICLE_IDENTITY_MYSQL_PASSWORD__</string>
<key>VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL</key>
<string>__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__</string>
<key>NACOS_CONFIG_ENABLED</key>
<string>false</string>
<key>MANAGEMENT_HEALTH_REDIS_ENABLED</key>
<string>false</string>
</dict>
<key>StandardOutPath</key>
<string>/tmp/lingniu-gb32960-live/launchctl-gb32960.log</string>
<key>StandardErrorPath</key>
<string>/tmp/lingniu-gb32960-live/launchctl-gb32960.err.log</string>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
</dict>
</plist>

View File

@@ -1,61 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.lingniu.jt808</string>
<key>WorkingDirectory</key>
<string>__PROJECT_ROOT__</string>
<key>ProgramArguments</key>
<array>
<string>__JAVA_BIN__</string>
<string>-jar</string>
<string>__PROJECT_ROOT__/modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar</string>
</array>
<key>EnvironmentVariables</key>
<dict>
<key>HTTP_PORT</key>
<string>20482</string>
<key>JT808_PORT</key>
<string>808</string>
<key>KAFKA_ENABLED</key>
<string>true</string>
<key>KAFKA_BROKERS</key>
<string>114.55.58.251:9092</string>
<key>KAFKA_NODE_ID</key>
<string>jt808-live-808</string>
<key>KAFKA_TOPIC_JT808_EVENT</key>
<string>vehicle.event.jt808.v1</string>
<key>KAFKA_TOPIC_JT808_RAW</key>
<string>vehicle.raw.jt808.v1</string>
<key>KAFKA_TOPIC_JT808_DLQ</key>
<string>vehicle.dlq.jt808.v1</string>
<key>SINK_ARCHIVE_PATH</key>
<string>__SHARED_ARCHIVE_PATH__</string>
<key>VEHICLE_IDENTITY_STORE</key>
<string>mysql</string>
<key>VEHICLE_IDENTITY_MYSQL_TABLE</key>
<string>vehicle_identity_binding</string>
<key>VEHICLE_IDENTITY_MYSQL_JDBC_URL</key>
<string>__VEHICLE_IDENTITY_MYSQL_JDBC_URL__</string>
<key>VEHICLE_IDENTITY_MYSQL_USERNAME</key>
<string>__VEHICLE_IDENTITY_MYSQL_USERNAME__</string>
<key>VEHICLE_IDENTITY_MYSQL_PASSWORD</key>
<string>__VEHICLE_IDENTITY_MYSQL_PASSWORD__</string>
<key>VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL</key>
<string>__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__</string>
<key>NACOS_CONFIG_ENABLED</key>
<string>false</string>
<key>MANAGEMENT_HEALTH_REDIS_ENABLED</key>
<string>false</string>
</dict>
<key>StandardOutPath</key>
<string>/tmp/lingniu-jt808-live/launchctl-jt808.log</string>
<key>StandardErrorPath</key>
<string>/tmp/lingniu-jt808-live/launchctl-jt808.err.log</string>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
</dict>
</plist>

View File

@@ -1,75 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.lingniu.vehicle-history</string>
<key>WorkingDirectory</key>
<string>__PROJECT_ROOT__</string>
<key>ProgramArguments</key>
<array>
<string>__JAVA_BIN__</string>
<string>-jar</string>
<string>__PROJECT_ROOT__/modules/apps/vehicle-history-app/target/vehicle-history-app.jar</string>
</array>
<key>EnvironmentVariables</key>
<dict>
<key>HTTP_PORT</key>
<string>20200</string>
<key>KAFKA_ENABLED</key>
<string>true</string>
<key>KAFKA_CONSUMER_ENABLED</key>
<string>true</string>
<key>KAFKA_CONSUMER_AUTO_OFFSET_RESET</key>
<string>latest</string>
<key>KAFKA_CONSUMER_CLIENT_ID_PREFIX</key>
<string>vehicle-history-live</string>
<key>KAFKA_GROUP_HISTORY</key>
<string>vehicle-history-live</string>
<key>KAFKA_CONSUMER_CONCURRENCY</key>
<string>3</string>
<key>KAFKA_BROKERS</key>
<string>114.55.58.251:9092</string>
<key>KAFKA_TOPIC_GB32960_EVENT</key>
<string>vehicle.event.gb32960.v1</string>
<key>KAFKA_TOPIC_GB32960_RAW</key>
<string>vehicle.raw.gb32960.v1</string>
<key>KAFKA_TOPIC_JT808_EVENT</key>
<string>vehicle.event.jt808.v1</string>
<key>KAFKA_TOPIC_JT808_RAW</key>
<string>vehicle.raw.jt808.v1</string>
<key>TDENGINE_HISTORY_ENABLED</key>
<string>true</string>
<key>TDENGINE_HISTORY_DATABASE</key>
<string>vehicle_ts</string>
<key>TDENGINE_JDBC_URL</key>
<string>__TDENGINE_JDBC_URL__</string>
<key>TDENGINE_DRIVER_CLASS_NAME</key>
<string>com.taosdata.jdbc.ws.WebSocketDriver</string>
<key>TDENGINE_USERNAME</key>
<string>__TDENGINE_USERNAME__</string>
<key>TDENGINE_PASSWORD</key>
<string>__TDENGINE_PASSWORD__</string>
<key>TDENGINE_MIN_IDLE</key>
<string>0</string>
<key>TDENGINE_MAX_POOL_SIZE</key>
<string>16</string>
<key>TDENGINE_INITIALIZE_SCHEMA</key>
<string>true</string>
<key>TDENGINE_TELEMETRY_FIELDS_ENABLED</key>
<string>false</string>
<key>NACOS_CONFIG_ENABLED</key>
<string>false</string>
<key>MANAGEMENT_HEALTH_REDIS_ENABLED</key>
<string>false</string>
</dict>
<key>StandardOutPath</key>
<string>/tmp/lingniu-history-live/vehicle-history.log</string>
<key>StandardErrorPath</key>
<string>/tmp/lingniu-history-live/vehicle-history.err.log</string>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
</dict>
</plist>

View File

@@ -0,0 +1,117 @@
version: "3.8"
# Go sidecar stack for the vehicle ingest redesign.
# It is intentionally separate from deploy/portainer/docker-compose.yml so the
# current Java production stack can keep running until Go evidence is captured.
x-common-env: &common-env
TZ: Asia/Shanghai
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.yutong-mqtt.v1}
KAFKA_TOPIC_UNIFIED: ${KAFKA_TOPIC_UNIFIED:-vehicle.event.unified.v1}
x-restart: &restart-policy
restart: unless-stopped
networks:
- vehicle-ingest
logging:
driver: json-file
options:
max-size: "100m"
max-file: "5"
services:
go-vehicle-gateway:
<<: *restart-policy
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
container_name: go-vehicle-gateway
command: ["/app/gateway"]
mem_limit: ${GO_GATEWAY_MEM_LIMIT:-512m}
environment:
<<: *common-env
GB32960_TCP_ADDR: ":32960"
JT808_TCP_ADDR: ":808"
TCP_READ_BUFFER_BYTES: ${TCP_READ_BUFFER_BYTES:-65536}
TCP_IDLE_TIMEOUT_SECONDS: ${TCP_IDLE_TIMEOUT_SECONDS:-180}
TCP_MAX_CONNECTIONS: ${TCP_MAX_CONNECTIONS:-20000}
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
YUTONG_MQTT_ENDPOINT: ${YUTONG_MQTT_ENDPOINT:-yutong}
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
YUTONG_MQTT_TOPICS: ${YUTONG_MQTT_TOPICS:-/ytforward/shln/+}
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-2}
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-go-yutong-mqtt}
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
IDENTITY_MYSQL_DSN: ${IDENTITY_MYSQL_DSN:-}
VEHICLE_IDENTITY_TABLE: ${VEHICLE_IDENTITY_TABLE:-vehicle_identity_binding}
ports:
- "${GO_GB32960_TCP_PORT:-32960}:32960"
- "${GO_JT808_TCP_PORT:-808}:808"
volumes:
- "${KAFKA_SPOOL_HOST_DIR:-/opt/lingniu-go/spool/gateway}:${KAFKA_SPOOL_DIR:-/data/spool/gateway}"
- "${YUTONG_MQTT_CERT_HOST_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:${YUTONG_MQTT_CERT_CONTAINER_DIR:-/opt/lingniuServices/certificate/yutong/vehicledatareception}:ro"
go-history-writer:
<<: *restart-policy
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
container_name: go-history-writer
command: ["/app/history-writer"]
mem_limit: ${GO_HISTORY_WRITER_MEM_LIMIT:-768m}
environment:
<<: *common-env
KAFKA_GROUP: ${KAFKA_GROUP_GO_HISTORY:-go-history-writer}
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_HISTORY:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1}
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
TDENGINE_DSN: ${TDENGINE_DSN:?set TDENGINE_DSN, example root:password@ws(172.17.111.57:6041)/lingniu_vehicle_ts}
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
TDENGINE_ENSURE_SCHEMA: ${TDENGINE_ENSURE_SCHEMA:-true}
go-stat-writer:
<<: *restart-policy
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
container_name: go-stat-writer
command: ["/app/stat-writer"]
mem_limit: ${GO_STAT_WRITER_MEM_LIMIT:-512m}
environment:
<<: *common-env
KAFKA_GROUP: ${KAFKA_GROUP_GO_STAT:-go-stat-writer}
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_STAT:-vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1}
MYSQL_DSN: ${MYSQL_DSN:?set MYSQL_DSN}
MYSQL_ENSURE_SCHEMA: ${MYSQL_ENSURE_SCHEMA:-true}
LOCAL_TZ: ${LOCAL_TZ:-Asia/Shanghai}
go-realtime-api:
<<: *restart-policy
image: ${LINGNIU_GO_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_GO_IMAGE_NAMESPACE:-oneos}/vehicle-gateway-go:${LINGNIU_GO_IMAGE_VERSION:?set LINGNIU_GO_IMAGE_VERSION}
container_name: go-realtime-api
command: ["/app/realtime-api"]
mem_limit: ${GO_REALTIME_API_MEM_LIMIT:-512m}
environment:
<<: *common-env
HTTP_ADDR: ":20210"
KAFKA_GROUP: ${KAFKA_GROUP_GO_REALTIME:-go-realtime-api}
KAFKA_TOPICS: ${KAFKA_TOPICS_GO_REALTIME:-vehicle.event.unified.v1}
REDIS_ADDR: ${REDIS_ADDR:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379}
REDIS_USERNAME: ${REDIS_USERNAME:-}
REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
REDIS_DB: ${REDIS_DB:-50}
ONLINE_TTL_SECONDS: ${ONLINE_TTL_SECONDS:-600}
MYSQL_DSN: ${MYSQL_DSN:-}
TDENGINE_DRIVER: ${TDENGINE_DRIVER:-taosWS}
TDENGINE_DSN: ${TDENGINE_DSN:-}
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-lingniu_vehicle_ts}
ports:
- "${GO_REALTIME_HTTP_PORT:-20210}:20210"
networks:
vehicle-ingest:
name: vehicle-ingest

View File

@@ -1,10 +1,13 @@
version: "3.8"
# Default production stack: GB32960, JT808, Yutong MQTT, history, analytics.
# Legacy and optional services stay outside this compose file.
# Set LINGNIU_IMAGE_VERSION in Portainer Stack env, for example:
# main-0.1.0-SNAPSHOT-22468e7
# Override LINGNIU_IMAGE_REGISTRY / LINGNIU_IMAGE_NAMESPACE only when publishing to another registry.
x-common-env: &common-env
TZ: Asia/Shanghai
JAVA_OPTS: ${JAVA_OPTS:--XX:+UseZGC -XX:MaxRAMPercentage=75}
JAVA_OPTS: ${JAVA_OPTS:---sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true -XX:+UseZGC -XX:MaxRAMPercentage=75}
NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true}
NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848}
NACOS_NAMESPACE: ${NACOS_NAMESPACE:-}
@@ -14,48 +17,51 @@ x-common-env: &common-env
NACOS_USERNAME: ${NACOS_USERNAME:-}
NACOS_PASSWORD: ${NACOS_PASSWORD:-}
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
KAFKA_ENABLED: ${KAFKA_ENABLED:-true}
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true}
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:-900000}
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
x-identity-mysql-env: &identity-mysql-env
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
x-redis-session-env: &redis-session-env
REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DATABASE: ${REDIS_DATABASE:-50}
REDIS_USERNAME: ${REDIS_USERNAME:-}
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
x-readiness-healthcheck: &readiness-healthcheck
interval: 30s
timeout: 5s
retries: 3
start_period: 60s
services:
gb32960-ingest-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: gb32960-ingest-app
restart: unless-stopped
mem_limit: ${GB32960_MEM_LIMIT:-768m}
environment:
<<: *common-env
<<:
- *common-env
- *identity-mysql-env
- *redis-session-env
HTTP_PORT: 20100
GB32960_PORT: 32960
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID:-gb32960-ingest-portainer}
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
GB32960_AUTH_ENABLED: ${GB32960_AUTH_ENABLED:-false}
GB32960_PLATFORM_USER_HYUNDAI: ${GB32960_PLATFORM_USER_HYUNDAI:-Hyundai}
GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}
GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}
GB32960_PLATFORM_USER_YUEJIN: ${GB32960_PLATFORM_USER_YUEJIN:-YueJin}
GB32960_PLATFORM_PWD_YUEJIN: ${GB32960_PLATFORM_PWD_YUEJIN:-}
GB32960_PLATFORM_IP_YUEJIN: ${GB32960_PLATFORM_IP_YUEJIN:-}
GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481pd.redis.rds.aliyuncs.com}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DATABASE: ${REDIS_DATABASE:-50}
REDIS_USERNAME: ${REDIS_USERNAME:-}
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
SINK_ARCHIVE_PATH: ${GB32960_ARCHIVE_PATH:-/data/archive}
ports:
- "${GB32960_HTTP_PORT:-20100}:20100"
- "${GB32960_TCP_PORT:-32960}:32960"
@@ -63,27 +69,27 @@ services:
- gb32960-ingest-data:/data
networks:
- vehicle-ingest
healthcheck:
<<: *readiness-healthcheck
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20100/actuator/health/readiness >/dev/null"]
jt808-ingest-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: jt808-ingest-app
restart: unless-stopped
mem_limit: ${JT808_MEM_LIMIT:-768m}
environment:
<<: *common-env
<<:
- *common-env
- *identity-mysql-env
- *redis-session-env
HTTP_PORT: 20400
JT808_PORT: 808
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481pd.redis.rds.aliyuncs.com}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DATABASE: ${REDIS_DATABASE:-50}
REDIS_USERNAME: ${REDIS_USERNAME:-}
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
SINK_ARCHIVE_PATH: ${JT808_ARCHIVE_PATH:-/data/archive}
ports:
- "${JT808_HTTP_PORT:-20400}:20400"
- "${JT808_TCP_PORT:-808}:808"
@@ -91,23 +97,26 @@ services:
- jt808-ingest-data:/data
networks:
- vehicle-ingest
healthcheck:
<<: *readiness-healthcheck
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20400/actuator/health/readiness >/dev/null"]
yutong-mqtt-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: yutong-mqtt-app
restart: unless-stopped
mem_limit: ${YUTONG_MQTT_MEM_LIMIT:-512m}
environment:
<<: *common-env
<<:
- *common-env
- *identity-mysql-env
HTTP_PORT: 20500
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
YUTONG_MQTT_AUTO_STARTUP: ${YUTONG_MQTT_AUTO_STARTUP:-true}
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_MQTT:-yutong-mqtt-portainer}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
YUTONG_MQTT_ENDPOINT_NAME: ${YUTONG_MQTT_ENDPOINT_NAME:-yutong}
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
YUTONG_MQTT_TOPIC: "${YUTONG_MQTT_TOPIC:-#}"
@@ -116,32 +125,43 @@ services:
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED: ${YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED:-true}
SINK_ARCHIVE_PATH: ${YUTONG_MQTT_ARCHIVE_PATH:-/data/archive}
ports:
- "${YUTONG_MQTT_HTTP_PORT:-20500}:20500"
volumes:
- yutong-mqtt-data:/data
networks:
- vehicle-ingest
healthcheck:
<<: *readiness-healthcheck
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20500/actuator/health/readiness >/dev/null"]
vehicle-history-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: vehicle-history-app
restart: unless-stopped
mem_limit: ${VEHICLE_HISTORY_MEM_LIMIT:-1536m}
depends_on:
- gb32960-ingest-app
- jt808-ingest-app
- yutong-mqtt-app
environment:
<<: *common-env
HTTP_PORT: 20200
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_HISTORY:-true}
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history}
KAFKA_CONSUMER_MAX_POLL_RECORDS: ${KAFKA_CONSUMER_MAX_POLL_RECORDS_HISTORY:-2000}
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_HISTORY:-1800000}
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_HISTORY_DLQ: ${KAFKA_TOPIC_HISTORY_DLQ:-vehicle.dlq.history.v1}
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
KAFKA_GROUP_HISTORY_GB32960_EVENT: ${KAFKA_GROUP_HISTORY_GB32960_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-gb32960-event}
KAFKA_GROUP_HISTORY_JT808_EVENT: ${KAFKA_GROUP_HISTORY_JT808_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-jt808-event}
KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-yutong-mqtt-event}
KAFKA_GROUP_HISTORY_GB32960_RAW: ${KAFKA_GROUP_HISTORY_GB32960_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-gb32960-raw}
KAFKA_GROUP_HISTORY_JT808_RAW: ${KAFKA_GROUP_HISTORY_JT808_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-jt808-raw}
KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-yutong-mqtt-raw}
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts}
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}
@@ -154,19 +174,23 @@ services:
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
networks:
- vehicle-ingest
healthcheck:
<<: *readiness-healthcheck
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20200/actuator/health/readiness >/dev/null"]
vehicle-analytics-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: ${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}/${LINGNIU_IMAGE_NAMESPACE:-oneos}/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: vehicle-analytics-app
restart: unless-stopped
mem_limit: ${VEHICLE_ANALYTICS_MEM_LIMIT:-1024m}
depends_on:
- gb32960-ingest-app
- jt808-ingest-app
environment:
<<: *common-env
HTTP_PORT: 20300
HTTP_PORT: 20310
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_ANALYTICS:-true}
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics}
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_ANALYTICS:-900000}
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
KAFKA_GROUP_STAT: ${KAFKA_GROUP_STAT:-vehicle-stat}
VEHICLE_STAT_ENABLED: ${VEHICLE_STAT_ENABLED:-true}
VEHICLE_STAT_ZONE_ID: ${VEHICLE_STAT_ZONE_ID:-Asia/Shanghai}
@@ -175,9 +199,12 @@ services:
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
ports:
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300"
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20310}:20310"
networks:
- vehicle-ingest
healthcheck:
<<: *readiness-healthcheck
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:20310/actuator/health/readiness >/dev/null"]
networks:
vehicle-ingest:
@@ -186,3 +213,4 @@ networks:
volumes:
gb32960-ingest-data:
jt808-ingest-data:
yutong-mqtt-data:

66
deploy/systemd/README.md Normal file
View File

@@ -0,0 +1,66 @@
# Go Vehicle Gateway Native Deployment
本目录用于 ECS 原生部署。当前 goal 后续不再使用 Docker/Portainer 作为 Go 接入链路的部署方式。
## Runtime Layout
```text
/opt/lingniu-go-native/
current -> /opt/lingniu-go-native/releases/<git-short-sha>
releases/<git-short-sha>/
gateway
history-writer
stat-writer
realtime-api
env/
gateway.env
history-writer.env
stat-writer.env
realtime-api.env
spool/gateway/
```
## Services
| systemd unit | Binary | Purpose |
|---|---|---|
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP `32960`、JT808 TCP `808`、宇通 MQTT 接入,写 Kafka RAW/unified |
| `lingniu-go-history-writer.service` | `history-writer` | 消费 RAW topic写 TDengine `raw_frames` 和核心时序表 |
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 RAW topic写 MySQL `vehicle_daily_metric` |
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费 unified topic写 Redis并提供 realtime/raw/stat 查询 API |
## Build
在开发机或 CI 上构建 Linux amd64 二进制:
```bash
cd go/vehicle-gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/gateway ./cmd/gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/history-writer ./cmd/history-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/stat-writer ./cmd/stat-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /tmp/lingniu-go-native/realtime-api ./cmd/realtime-api
```
## Cutover Notes
1. 先上传二进制到新 release 目录并更新 `current` symlink。
2. 从现有生产 env 生成四个 systemd 专用 env 文件;不要把密钥写入仓库。
3. 停止旧 Docker Go 容器或 Java 容器,释放 `808``32960``20210`
4. 执行:
```bash
systemctl daemon-reload
systemctl enable lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
```
## Verification
```bash
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=JT808&limit=1'
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=GB32960&limit=1'
curl -sS 'http://127.0.0.1:20210/api/history/raw-frames?protocol=YUTONG_MQTT&limit=1'
```

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Vehicle Gateway
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/gateway.env
ExecStart=/opt/lingniu-go-native/current/gateway
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go History Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/history-writer.env
ExecStart=/opt/lingniu-go-native/current/history-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Realtime API
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/realtime-api.env
ExecStart=/opt/lingniu-go-native/current/realtime-api
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,18 @@
[Unit]
Description=Lingniu Go Stat Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/lingniu-go-native/current
EnvironmentFile=/opt/lingniu-go-native/env/stat-writer.env
ExecStart=/opt/lingniu-go-native/current/stat-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -257,14 +257,15 @@
<section>
<h2>一、系统定位</h2>
<p>
当前项目不是业务库写入服务。它的核心职责是把 GB/T 32960、JT/T 808、JT/T 1078、JSATL12、
MQTT 等不同来源的车辆数据统一转换为 <code>VehicleEvent</code> 和全字段 <code>TelemetrySnapshot</code>
当前项目不是业务库写入服务。它的核心职责是把 GB/T 32960、JT/T 808、Yutong MQTT
等默认生产来源的车辆数据统一转换为 <code>VehicleEvent</code> 和全字段 <code>TelemetrySnapshot</code>
JT/T 1078、JSATL12 作为显式 profile 的可选能力保留,
再通过 Kafka、原始报文归档、TDengine 历史索引、Redis 热状态和独立统计模块交给下游业务。
</p>
<div class="summary-grid">
<div class="summary-card">
<strong>接入方式</strong>
<span>Netty TCP、MQTT、后续可扩展更多 Inbound AdapterXinda Push 不在默认生产面</span>
<span>Netty TCP、MQTT、后续可扩展更多 Inbound Adapter信达 Push 源码已删除</span>
</div>
<div class="summary-card">
<strong>统一模型</strong>
@@ -308,7 +309,7 @@ flowchart TB
codec["ingest-codec-common<br/>BCD / CRC / BCC / bit 工具"]
session["session-core<br/>设备会话、命令分发接口、会话存储"]
identity["vehicle-identity<br/>跨协议身份解析、MySQL 绑定表"]
obs["observability<br/>metrics / tracing / health"]
obs["observability<br/>metrics / health"]
api["ingest-api<br/>ProtocolId / RawFrame / VehicleEvent / 注解 SPI"]
registry["HandlerRegistry<br/>按协议、命令、infoType 路由"]
dispatcher["Dispatcher<br/>RawFrame 到 Handler 到 EventBus"]
@@ -316,28 +317,34 @@ flowchart TB
end
subgraph sink["输出与明细存储 modules/sinks"]
mq["sink-mq<br/>VehicleEvent 到 Protobuf Envelope 到 Kafka"]
archive["sink-archive<br/>RawArchive 到本地/S3/OSS 冷存"]
kafka["sink-kafka<br/>VehicleEvent 到 Protobuf Envelope 到 Kafka"]
archive["sink-archive<br/>RawArchive 到本地文件系统冷存"]
tdengineStore["tdengine-history-store<br/>TDengine raw_frames + locations"]
fileStore["event-file-store<br/>Parquet/DuckDB 兼容可选"]
end
subgraph services["消费服务 modules/services"]
history["event-history-service<br/>Kafka event/raw 到 TDengine 历史查询"]
state["vehicle-state-service<br/>Kafka 全字段事件到 Redis 热状态"]
state["vehicle-state-service<br/>可选 Redis 热状态<br/>optional-latest-state"]
stat["vehicle-stat-service<br/>Kafka 全字段事件到指标表"]
end
subgraph apps["应用入口 modules/apps"]
gateway["command-gateway<br/>HTTP 到设备命令入口"]
bootall["bootstrap-all<br/>聚合所有 AutoConfiguration 一体启动"]
gbApp["gb32960-ingest-app<br/>GB32960 TCP 接入"]
jtApp["jt808-ingest-app<br/>JT808 TCP 接入"]
yutongApp["yutong-mqtt-app<br/>宇通 MQTT 接入"]
historyApp["vehicle-history-app<br/>历史查询与 TDengine 写入"]
analyticsApp["vehicle-analytics-app<br/>808 日里程指标消费"]
gateway["command-gateway<br/>可选 HTTP 设备命令入口"]
end
vehicle --> gb
vehicle --> jt808
vehicle --> gbApp
vehicle --> jtApp
vehicle --> jt1078
vehicle --> jsatl12
mqttSource --> mqtt
mqttSource --> yutongApp
gbApp --> gb
jtApp --> jt808
yutongApp --> mqtt
mqtt --> mqttProfile
mqttProfile --> api
commandClient --> gateway
@@ -355,13 +362,14 @@ flowchart TB
api --> registry
registry --> dispatcher
dispatcher --> bus
bus --> mq
bus --> kafka
bus --> archive
mq --> history
mq --> state
mq --> stat
kafka --> historyApp
kafka --> analyticsApp
historyApp --> history
analyticsApp --> stat
kafka -.可选热状态.-> state
history --> tdengineStore
history -.兼容可选.-> fileStore
gateway --> session
session --> jt808
@@ -371,15 +379,7 @@ flowchart TB
obs -.监控.-> dispatcher
obs -.监控.-> bus
obs -.监控.-> mq
bootall -.装配.-> gb
bootall -.装配.-> jt808
bootall -.装配.-> mqtt
bootall -.装配.-> mq
bootall -.装配.-> archive
bootall -.装配.-> history
bootall -.装配.-> state
bootall -.装配.-> stat
obs -.监控.-> kafka
classDef entry fill:#eff6ff,stroke:#2563eb,color:#1e3a8a;
classDef core fill:#ecfdf3,stroke:#0f766e,color:#134e48;
@@ -387,11 +387,11 @@ flowchart TB
classDef command fill:#f5f3ff,stroke:#6d28d9,color:#4c1d95;
classDef support fill:#f8fafc,stroke:#64748b,color:#334155;
class gb,jt808,jt1078,jsatl12,mqtt,mqttProfile entry;
class gb,jt808,jt1078,jsatl12,mqtt,mqttProfile,gbApp,jtApp,yutongApp,historyApp,analyticsApp entry;
class api,registry,dispatcher,bus,identity core;
class mq,archive,tdengineStore,fileStore sink;
class kafka,archive,tdengineStore sink;
class gateway,session command;
class codec,obs,bootall support;
class codec,obs support;
</pre>
</div>
<div class="legend">
@@ -414,7 +414,7 @@ sequenceDiagram
participant Dispatcher as Dispatcher
participant Handler as Protocol Handler<br/>Mapper
participant EventBus as DisruptorEventBus
participant MQ as sink-mq<br/>Kafka Envelope
participant Kafka as sink-kafka<br/>Kafka Envelope
participant Archive as sink-archive<br/>ArchiveStore
participant History as vehicle-history-app
participant TDengine as tdengine-history-store<br/>raw_frames + locations
@@ -430,12 +430,12 @@ sequenceDiagram
Handler->>Handler: 协议字段映射为内部字段
Dispatcher->>Handler: 绑定 rawArchiveKey / rawArchiveUri
Handler->>EventBus: VehicleEvent(Realtime / Location / Alarm / Login ...)
EventBus->>MQ: 投递规整后的业务事件
MQ->>MQ: EnvelopeMapper 转 Protobuf
MQ->>History: Kafka event/raw topickey=vin单车有序
EventBus->>Kafka: 投递规整后的业务事件
Kafka->>Kafka: EnvelopeMapper 转 Protobuf
Kafka->>History: event/raw topickey=vin单车有序
History->>TDengine: 写 raw_frames、locations 和 raw parsed JSON
TDengine->>Consumer: 分页历史查询、原始帧追溯、位置查询
MQ->>Consumer: Redis 热状态、指标统计、告警工单等下游消费
Kafka->>Consumer: Redis 热状态、指标统计、告警工单等下游消费
</pre>
</div>
<div class="callout">
@@ -444,7 +444,7 @@ sequenceDiagram
</div>
<div class="callout">
<strong>原始追溯:</strong>
Dispatcher 为所有带 rawBytes 的 RawFrame 生成统一的 <code>rawArchiveKey</code>,业务事件 metadata 使用 <code>rawArchiveUri=archive://...</code> 指向同一份原始 bytes。归档文件的真实本地/S3/OSS 地址只属于 ArchiveStore查询、导出和 Kafka Envelope 使用逻辑 URI 解耦存储实现。
Dispatcher 为所有带 rawBytes 的 RawFrame 生成统一的 <code>rawArchiveKey</code>,业务事件 metadata 使用 <code>rawArchiveUri=archive://...</code> 指向同一份原始 bytes。归档文件的真实本地路径只属于 ArchiveStore查询、导出和 Kafka Envelope 使用逻辑 URI 解耦存储实现。
</div>
</section>
@@ -521,7 +521,7 @@ flowchart LR
</tr>
<tr>
<td>modules/core/session-core</td>
<td>设备会话、命令下发抽象、SessionStore、CommandDispatcher 默认实现。SessionStore 支持 <code>memory</code><code>redis</code> 两种后端;生产默认使用 Redis 保存 <code>sessionId</code>、VIN 和 phone 三索引及 TTL内存实现作为本地开发/无 Redis 降级,协议模块只依赖 SPI不感知存储后端</td>
<td>设备会话、命令下发抽象、SessionStore、CommandDispatcher 默认实现。SessionStore 仅保留 Redis 后端;协议进程本地只保存真实 Netty ChannelRedis 保存 <code>sessionId</code>、VIN 和 phone 三索引及 TTL协议模块只依赖 SPI不感知存储细节</td>
<td>设备连接、命令请求。</td>
<td>会话状态、命令分发结果、Redis 会话索引。</td>
<td><span class="tag">在线状态</span><span class="tag">下行控制</span><span class="tag green">多实例会话索引</span></td>
@@ -535,7 +535,7 @@ flowchart LR
</tr>
<tr>
<td>modules/core/observability</td>
<td>监控、指标、追踪、健康检查等横切能力。</td>
<td>监控、指标、健康检查等横切能力。</td>
<td>核心管线和 Sink 运行状态。</td>
<td>Micrometer 指标、健康信息。</td>
<td><span class="tag green">可用性</span><span class="tag">延迟监控</span></td>
@@ -563,7 +563,7 @@ flowchart LR
</tr>
<tr>
<td>modules/protocols/protocol-jsatl12</td>
<td>苏标主动安全报警附件接入。依赖 ArchiveStore 和 JT808 decoder生产默认端口按旧接收服务对齐为 <code>7612</code>;附件 DataPacket 按旧服务真实布局 <code>01cd + 50B文件名 + offset + length + data</code> 分帧解析,不再把文件名前 4 字节误判为帧长;附件数据写 ArchiveStore 后通过 Dispatcher 产出 MediaMeta 引用事件,事件 metadata 保留 fileName、fileOffset、declaredChunkSizeBytes、archiveKey 和 archiveRef便于附件明细查询、导出和补传诊断内置附件状态机处理 T1210 文件清单、DataPacket 分块区间和 T1212 上传完成消息,数据分块会按 T1210 文件清单中的文件名继承手机号并通过共享身份解析器得到内部 VIN入口 RawFrame、ArchivedChunk 和 MediaMeta 均保留 phone、vin、identityResolved、identitySource只有找不到文件归属时才标记 UNKNOWN按文件名合并无身份数据分块并返回 0x9212 完成/补传应答,补传应答包含缺失 offset/length 区间;附件归档失败会产出 JSATL12 Passthrough 错误事件并保留原始分块JT_MESSAGE 信令复用 JT808 decoder 后进入 Dispatcher桥接 RawFrame 使用共享身份解析后的内部 VIN并保留 phone、identityResolved、identitySource坏 JT 信令入口 RawFrame 同样标记 UNKNOWN 身份;超长/畸形附件帧、未闭合超长 JT 信令和坏 JT 信令都以 JSATL12 Passthrough 兜底,附件帧异常同时标记 frameError 与统一 parseError/parseErrorMessage保留原始字节、peer、长度和错误元数据帧长和 worker 线程支持部署配置。</td>
<td>苏标主动安全报警附件接入(仅 <code>optional-attachments</code> profile 显式构建,不属于默认生产面)。依赖 ArchiveStore 和 JT808 decoder端口按旧接收服务对齐为 <code>7612</code>;附件 DataPacket 按旧服务真实布局 <code>01cd + 50B文件名 + offset + length + data</code> 分帧解析,不再把文件名前 4 字节误判为帧长;附件数据写 ArchiveStore 后通过 Dispatcher 产出 MediaMeta 引用事件,事件 metadata 保留 fileName、fileOffset、declaredChunkSizeBytes、archiveKey 和 archiveRef便于附件明细查询、导出和补传诊断内置附件状态机处理 T1210 文件清单、DataPacket 分块区间和 T1212 上传完成消息,数据分块会按 T1210 文件清单中的文件名继承手机号并通过共享身份解析器得到内部 VIN入口 RawFrame、ArchivedChunk 和 MediaMeta 均保留 phone、vin、identityResolved、identitySource只有找不到文件归属时才标记 UNKNOWN按文件名合并无身份数据分块并返回 0x9212 完成/补传应答,补传应答包含缺失 offset/length 区间;附件归档失败会产出 JSATL12 Passthrough 错误事件并保留原始分块JT_MESSAGE 信令复用 JT808 decoder 后进入 Dispatcher桥接 RawFrame 使用共享身份解析后的内部 VIN并保留 phone、identityResolved、identitySource坏 JT 信令入口 RawFrame 同样标记 UNKNOWN 身份;超长/畸形附件帧、未闭合超长 JT 信令和坏 JT 信令都以 JSATL12 Passthrough 兜底,附件帧异常同时标记 frameError 与统一 parseError/parseErrorMessage保留原始字节、peer、长度和错误元数据帧长和 worker 线程支持部署配置。</td>
<td>主动安全附件流、JT808 信令帧。</td>
<td>归档文件、MediaMeta、JT808 统一事件、坏帧/坏信令/归档失败 Passthrough。</td>
<td><span class="tag amber">附件归档</span><span class="tag">报警证据</span><span class="tag">信令复用</span><span class="tag">错误隔离</span></td>
@@ -576,7 +576,7 @@ flowchart LR
<td><span class="tag">车企平台接入</span><span class="tag green">profile 扩展</span><span class="tag green">身份映射</span><span class="tag green">双向 TLS</span><span class="tag green">错误隔离</span></td>
</tr>
<tr>
<td>modules/sinks/sink-mq</td>
<td>modules/sinks/sink-kafka</td>
<td>Kafka Sink。把 VehicleEvent 转成 Protobuf Envelope按 VIN 分区投递;业务事件携带全字段 TelemetrySnapshot 和 rawArchiveUri显式 RawArchive Envelope 会填充 archive:// 逻辑 URI 与 size默认 Kafka Sink 仍不发送原始 bytes 本体;同时提供 KafkaEnvelopeConsumerFactory、KafkaEnvelopeConsumerRunner、KafkaEnvelopeConsumerWorker 和 KafkaEnvelopeDeadLetterSink统一服务侧 Kafka 消费启动、独立 group 绑定、死信发布和 topic 到 EnvelopeConsumerProcessor 的分发。</td>
<td>VehicleEvent。</td>
<td>Kafka Protobuf 消息、消费侧 DLQ 记录。</td>
@@ -584,23 +584,16 @@ flowchart LR
</tr>
<tr>
<td>modules/sinks/sink-archive</td>
<td>原始报文冷存,当前本地文件系统实现,后续可替换 S3/OSS;写入键与业务事件 metadata 中的 <code>rawArchiveKey</code> 保持一致。</td>
<td>原始报文冷存,当前本地文件系统实现;写入键与业务事件 metadata 中的 <code>rawArchiveKey</code> 保持一致。</td>
<td>RawArchive 事件、附件流。</td>
<td>可回放原始文件、<code>archive://...</code> 逻辑引用。</td>
<td><span class="tag amber">问题排查</span><span class="tag amber">合规留痕</span></td>
</tr>
<tr>
<td>modules/sinks/event-file-store</td>
<td>兼容文件型明细库,使用 Parquet 存储解析后的事件明细和 RawArchive 可查询索引,使用 DuckDB 按日期范围正序/倒序查询;默认生产部署不启用,仅用于旧低层记录接口或本地调查场景。</td>
<td>VehicleEvent 或 EventFileRecord。</td>
<td>protocol/date 分区的 Parquet 文件和兼容查询结果。</td>
<td><span class="tag">兼容查询</span><span class="tag amber">默认关闭</span></td>
</tr>
<tr>
<td>modules/sinks/tdengine-history-store</td>
<td>生产历史库边界,负责 TDengine schema、raw_frames/location/telemetry field 行映射、批量写入和分页查询语句raw_frames 保留完整 payloadJson.parsed位置表保持轻量字段并通过 rawUri 关联原始帧。</td>
<td>生产历史库边界,负责 TDengine schema、raw_frames/location 行映射、批量写入和分页查询语句raw_frames 保留完整 payloadJson.parsed位置表保持轻量字段并通过 rawUri 关联原始帧。</td>
<td>Kafka Protobuf Envelope、Raw Envelope。</td>
<td>TDengine <code>raw_frames</code>、位置表、可选 telemetry field 表。</td>
<td>TDengine <code>raw_frames</code> 和位置表。</td>
<td><span class="tag green">生产历史</span><span class="tag green">分页查询</span><span class="tag amber">raw 追溯</span></td>
</tr>
<tr>
@@ -612,7 +605,7 @@ flowchart LR
</tr>
<tr>
<td>modules/services/vehicle-state-service</td>
<td>消费 Kafka 全字段事件,更新车辆最新状态、位置、安全和最后事件到 Redis生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf 和缺快照 Envelope并把失败记录交给死信出口。</td>
<td>消费 Kafka 全字段事件,更新车辆最新状态、位置、安全和最后事件到 Redis仅通过 <code>optional-latest-state</code> profile 显式构建,不属于默认生产 reactor生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf 和缺快照 Envelope并把失败记录交给死信出口。</td>
<td>Kafka Protobuf Envelope。</td>
<td><code>vehicle:state:{vin}</code><code>vehicle:location:{vin}</code><code>vehicle:safety:{vin}</code></td>
<td><span class="tag green">毫秒级热查询</span><span class="tag red">氢泄露安全</span></td>
@@ -626,17 +619,45 @@ flowchart LR
</tr>
<tr>
<td>modules/apps/command-gateway</td>
<td>HTTP 到设备下行命令入口。支持会话查询、808 位置查询、参数查询/设置、终端控制、平台通用应答、终端属性查询 0x8107、区域删除 0x8601、人工报警确认 0x8203以及 1078 音视频属性查询、实时预览/控制、历史回放/控制、资源列表查询、文件上传/控制;通过 CommandDispatcher 复用 protocol-jt808 的在线通道、流水号和同步应答等待能力。</td>
<td>可选 HTTP 到设备下行命令入口。支持会话查询、808 位置查询、参数查询/设置、终端控制、平台通用应答、终端属性查询 0x8107、区域删除 0x8601、人工报警确认 0x8203以及 1078 音视频属性查询、实时预览/控制、历史回放/控制、资源列表查询、文件上传/控制;通过 CommandDispatcher 复用 protocol-jt808 的在线通道、流水号和同步应答等待能力。</td>
<td>业务系统命令请求。</td>
<td>CommandDispatcher 调用、设备应答摘要。</td>
<td><span class="tag">远程控制</span><span class="tag">参数设置</span><span class="tag">属性查询</span><span class="tag">区域删除</span><span class="tag">报警确认</span><span class="tag">视频控制</span></td>
</tr>
<tr>
<td>modules/apps/bootstrap-all</td>
<td>一体化启动模块,聚合各模块 AutoConfiguration 和运行配置</td>
<td>application.yml、Spring Boot 启动</td>
<td>完整接入服务实例</td>
<td><span class="tag green">部署</span><span class="tag green">开关配置</span></td>
<td>modules/apps/gb32960-ingest-app</td>
<td>生产 GB32960 TCP 接入应用,只负责接收、鉴权、解析、冷存引用和 Kafka 投递</td>
<td>GB/T 32960 TCP 32960</td>
<td><code>vehicle.raw.gb32960.v1</code><code>vehicle.event.gb32960.v1</code></td>
<td><span class="tag green">生产接入</span><span class="tag">32960</span></td>
</tr>
<tr>
<td>modules/apps/jt808-ingest-app</td>
<td>生产 JT808 TCP 接入应用,只负责 808 注册/鉴权/位置等上行解析、冷存引用和 Kafka 投递。</td>
<td>JT/T 808 TCP 808。</td>
<td><code>vehicle.raw.jt808.v1</code><code>vehicle.event.jt808.v1</code></td>
<td><span class="tag green">生产接入</span><span class="tag">808</span></td>
</tr>
<tr>
<td>modules/apps/yutong-mqtt-app</td>
<td>生产宇通 MQTT 接入应用,按 endpoint profile 解析 MQTT 报文并投递 Kafka。</td>
<td>宇通 MQTT topic。</td>
<td><code>vehicle.raw.mqtt-yutong.v1</code><code>vehicle.event.mqtt-yutong.v1</code></td>
<td><span class="tag green">生产接入</span><span class="tag">MQTT</span></td>
</tr>
<tr>
<td>modules/apps/vehicle-history-app</td>
<td>生产历史应用,消费 32960、808、宇通 MQTT 的 raw/event Kafka Envelope写 TDengine 并提供历史查询。</td>
<td>Kafka Protobuf Envelope。</td>
<td>TDengine <code>raw_frames</code>、位置表和历史查询 API。</td>
<td><span class="tag green">历史查询</span><span class="tag amber">raw JSON</span></td>
</tr>
<tr>
<td>modules/apps/vehicle-analytics-app</td>
<td>生产指标应用,当前只消费 JT808 事件并用 GPS 总里程首末差值写每日里程指标。</td>
<td><code>vehicle.event.jt808.v1</code></td>
<td>MySQL <code>vehicle_stat_metric</code></td>
<td><span class="tag green">每日里程</span><span class="tag">指标表</span></td>
</tr>
</tbody>
</table>
@@ -691,7 +712,7 @@ flowchart LR
<tr>
<td>明细展示和导出</td>
<td><code>vehicle-history-app</code> + <code>tdengine-history-store</code></td>
<td>生产查询通过 TDengine raw_frames、位置表和 rawUri 关联完成;Parquet/DuckDB 仅作为兼容文件型明细库,不进入默认热路径</td>
<td>生产查询通过 TDengine raw_frames、位置表和 rawUri 关联完成;不再维护文件型事件索引旁路</td>
</tr>
</tbody>
</table>

View File

@@ -0,0 +1,227 @@
# 当前 ECS Go 原生部署说明
更新时间2026-07-02 01:30 CST
本文档记录当前 `lingniu-vehicle-ingest` 的生产运行面。当前 goal 的接入链路已经切到 Go 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。密钥只维护在 ECS 环境文件或受控凭据中,不写入 Git。
## 部署范围
| systemd 服务 | 二进制 | 说明 | 对外端口 |
| --- | --- | --- | --- |
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP、JT808 TCP、宇通 MQTT 接入,解析后写 Kafka RAW 和统一事件 | TCP `32960`、TCP `808` |
| `lingniu-go-history-writer.service` | `history-writer` | 消费 Kafka RAW写 TDengine RAW、位置点、里程点核心表 | 无 HTTP |
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 Kafka RAW按总里程差值法写 MySQL 每日指标 | 无 HTTP |
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费统一事件写 Redis并提供实时、历史、统计查询 API | HTTP `20210` |
信达 Push 已废弃,不参与当前 Go 生产链路。旧 Java/Docker 服务不应占用 `808``32960``20210`
## 应用 ECS
| 项 | 值 |
| --- | --- |
| 公网 IP | `115.29.187.205` |
| 登录用户 | `root` |
| 部署目录 | `/opt/lingniu-go-native` |
| 当前 release | `/opt/lingniu-go-native/current` |
| 环境文件目录 | `/opt/lingniu-go-native/env` |
| systemd unit 目录 | `/etc/systemd/system` |
| 对外服务 | `32960``808``20210` |
运行目录结构:
```text
/opt/lingniu-go-native/
current -> /opt/lingniu-go-native/releases/<git-short-sha>
releases/<git-short-sha>/
gateway
history-writer
stat-writer
realtime-api
env/
gateway.env
history-writer.env
stat-writer.env
realtime-api.env
spool/gateway/
```
## 中间件
| 组件 | 内网地址 | 公网地址 | 用途 |
| --- | --- | --- | --- |
| Kafka | `172.17.111.56:9092` | `114.55.58.251:9092` | RAW 和统一事件消息总线 |
| TDengine | `172.17.111.57:6041` | `115.29.185.82:6041` | RAW、位置、里程点时序热存储 |
| MySQL RDS | `rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306` | `rm-bp179zbv481rnw3e2no.mysql.rds.aliyuncs.com:3306` | 身份映射、JT808 注册、每日指标 |
| Redis RDS | `r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379` | 无 | 准实时车辆状态缓存 |
生产应用之间访问中间件优先使用内网地址。RDS 白名单需要允许应用 ECS 私网访问。
## Kafka Topic
| Topic | 生产者 | 消费者 | 内容 |
| --- | --- | --- | --- |
| `vehicle.raw.go.gb32960.v1` | `gateway` | `history-writer``stat-writer` | GB32960 完整 RAW 记录 |
| `vehicle.raw.go.jt808.v1` | `gateway` | `history-writer``stat-writer` | JT808 完整 RAW 记录 |
| `vehicle.raw.go.yutong-mqtt.v1` | `gateway` | `history-writer` | 宇通 MQTT 完整 RAW 记录 |
| `vehicle.event.go.unified.v1` | `gateway` | `realtime-api` | 三类协议统一事件,用于 Redis 实时状态 |
`gateway` 配置了 Kafka retry生产 env 中启用 `KAFKA_SPOOL_DIR` 时,本地 spool 可在 Kafka 短暂不可用后回放。
Kafka 消费组:
| Consumer Group | Topic | 当前验证 |
| --- | --- | --- |
| `go-history-writer` | 三个 `vehicle.raw.go.*` topic | 2026-07-02 验证 lag 为 `0` |
| `go-stat-writer` | `vehicle.raw.go.gb32960.v1``vehicle.raw.go.jt808.v1` | 2026-07-02 验证 lag 为 `0` |
| `go-realtime-api` | `vehicle.event.go.unified.v1` | 2026-07-02 验证 lag 为 `0` |
可重复 smoke
```bash
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
```
该脚本通过 SSH 登录 Kafka ECS检查 Go 生产 topic 是否存在,并校验 `go-history-writer``go-stat-writer``go-realtime-api` 的消费组 lag。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
## TDengine 数据层
默认数据库:`lingniu_vehicle_ts`
| 表 | 类型 | 说明 |
| --- | --- | --- |
| `raw_frames` | stable | 完整 RAW 帧,包含 `raw_hex``parsed_json``fields_json``source_endpoint` 和协议/车辆 tags |
| `vehicle_locations` | stable | 最小化位置历史,保留 `frame_id` 回链 RAW |
| `vehicle_mileage_points` | stable | 最小化总里程点,供历史查询和统计复核 |
设计原则:
- 完整结构化 payload 只落在 `raw_frames.parsed_json` / `fields_json`
- `vehicle_locations``vehicle_mileage_points` 只保留核心查询字段,不重复保存完整 JSON。
- `telemetry_fields` 不由当前服务维护,字段配置解析由独立子服务处理。
- API 传入东八区时间时,服务会转换为 TDengine 当前存储使用的 UTC 字面量再查询。
## MySQL 数据层
默认业务库:`lingniu_vehicle_data`
| 表 | 维护方 | 说明 |
| --- | --- | --- |
| `vehicle_identity_binding` | 人工/外部主数据 | VIN 映射主表,用 `phone``device_id``plate` 降级定位 VIN |
| `jt808_registration` | `gateway` 自动写入 | 仅 JT808 注册和鉴权记录,以 `phone` 作为主键,记录设备、车牌、厂家、鉴权、来源端点 |
| `vehicle_daily_metric` | `stat-writer` 自动写入 | 每日指标表,保存 `daily_mileage_km``daily_total_mileage_km` |
每日里程算法:
```text
daily_mileage_km = 当日最大 total_mileage_km - 当日最小 total_mileage_km
daily_total_mileage_km = 当日最大 total_mileage_km
```
统计按 `Asia/Shanghai` 自然日归档,当前支持 GB32960、JT808、宇通 MQTT 中能解析出 `total_mileage_km` 的数据。JT808 的总里程来自位置附加信息 `0x01`,单位按协议转换为 km宇通 MQTT 的 `TOTAL_MILEAGE` 原始单位为米,入库前转换为 km。
## Redis 实时层
`realtime-api` 消费 `vehicle.event.go.unified.v1`,按 VIN/vehicle key 维护准实时快照。主要能力:
| API | 说明 |
| --- | --- |
| `/api/realtime/vehicles/{vin}` | 查询车辆合并实时快照 |
| `/api/realtime/vehicles/{vin}/online` | 查询车辆是否在线 |
| `/api/realtime/vehicles/{vin}/protocols/{protocol}` | 查询指定协议的实时快照 |
Redis 只作为实时缓存;历史和统计以 TDengine/MySQL 为准。
## API 入口
公网基础地址:
```text
http://115.29.187.205:20210
```
常用查询:
```bash
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=YUTONG_MQTT&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/locations?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/history/mileage-points?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
curl -sS 'http://115.29.187.205:20210/api/stats/daily-metrics?dateFrom=2026-07-02&dateTo=2026-07-02&limit=20'
```
分页参数统一使用 `limit``offset`
部署后推荐直接运行 Go 原生生产 smoke
```bash
python3 tools/go_prod_acceptance.py --date 2026-07-02
```
该聚合脚本会串行执行 systemd/端口检查、Kafka topic/consumer lag 检查、HTTP 数据链路检查,任一子检查失败时整体退出码为非 0。需要 SSH 可登录应用 ECS 和 Kafka ECS。
也可以单独运行子检查:
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
```
`go_systemd_prod_smoke.py` 通过 SSH 检查四个 Go systemd 服务是否 `active``enabled`,并确认 `808``32960``gateway` 监听、`20210``realtime-api` 监听,同时要求 `/opt/lingniu-go-native/current` 指向有效 release、四个生产二进制均可执行、`/opt/lingniu-go-native/spool/gateway` 没有待回放文件,避免旧 Java/Docker 进程占用生产端口、部署目录错乱或 Kafka spool 静默堆积。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
`go_native_prod_smoke.py` 通过 `20210` HTTP API 验证 GB32960、JT808、宇通 MQTT 的 RAW 查询及结构化 `parsed_json`,三类协议的位置和里程点查询及 `frame_id` RAW 回链GB32960/JT808 的 `daily_mileage_km``daily_total_mileage_km` 统计公式,以及三类协议的 Redis realtime snapshot、online、protocol 查询。实时 snapshot/protocol 查询会校验 `updated_at_ms` 新鲜度;默认检查今天东八区数据时,还会要求三类 RAW 最新样本不超过 15 分钟,避免旧数据误判为接收正常;任一检查不达标时退出码为非 0。
## 部署命令
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收:
```bash
python3 tools/go_native_deploy.py --date 2026-07-02
```
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout``--spool-poll-interval` 调整。
脚本执行的发布动作等价于:
```bash
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/gateway
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/history-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/stat-writer
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/realtime-api
scp release.tar.gz root@115.29.187.205:/tmp/
ln -sfn /opt/lingniu-go-native/releases/<git-short-sha> /opt/lingniu-go-native/current
systemctl daemon-reload
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
```
只更新查询 API 时,可以仅替换 `/opt/lingniu-go-native/current/realtime-api` 并重启:
```bash
systemctl restart lingniu-go-realtime-api.service
```
## 运行检查
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-history-writer --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-stat-writer --since '5 minutes ago' --no-pager
journalctl -u lingniu-go-realtime-api --since '5 minutes ago' --no-pager
```
2026-07-02 01:30 CST 已验证:
- 四个 Go systemd 服务均为 `active``enabled`
- `/opt/lingniu-go-native/current` 指向有效 release四个生产二进制均存在且可执行。
- `gateway` 正在监听 `32960``808`
- `realtime-api` 正在监听 `20210`
- gateway Kafka spool 当前无待回放文件。
- GB32960、JT808、YUTONG_MQTT 的东八区 RAW 查询均可命中生产数据,且最新 RAW 样本包含结构化 `parsed_json`
- GB32960、JT808、YUTONG_MQTT 的最新 RAW 样本均在 15 分钟内。
- GB32960、JT808、YUTONG_MQTT 的 `vehicle_locations``vehicle_mileage_points` 查询均可命中生产数据,且样本包含 `frame_id` 回链 RAW。
- `vehicle_daily_metric` 可查到 `daily_mileage_km``daily_total_mileage_km`,且公式满足 `daily_mileage_km = latest_total_mileage_km - first_total_mileage_km``daily_total_mileage_km = latest_total_mileage_km`
- Redis realtime 可查到 GB32960、JT808、YUTONG_MQTT 的在线状态和协议快照,且 snapshot/protocol 快照最近刷新。

View File

@@ -1,28 +1,33 @@
# GB32960 Split Service Runbook
This runbook covers the split GB32960 ingest, history, and analytics runtimes. Kafka is the durability boundary between the protocol service and downstream business services.
This runbook covers the default production runtimes: GB32960 ingest, JT808 ingest, Yutong MQTT ingest, history, and analytics. Kafka is the durability boundary between protocol ingest services and downstream business services.
## Service Roles
| Service | Module | Role | Local ports |
| Service | Module | Role | Production ports |
| --- | --- | --- | --- |
| GB32960 ingest | `:gb32960-ingest-app` | Accept GB32960 TCP connections, decode/authenticate frames, produce raw and normalized records to Kafka, and send protocol ACKs after required Kafka production succeeds. | TCP `32960`, HTTP `20100` |
| JT808 ingest | `:jt808-ingest-app` | Accept JT/T 808 TCP connections on the production receive port, parse raw frames, archive frame bytes, and produce raw/event envelopes to Kafka. | TCP `808`, HTTP `20482` |
| Vehicle history | `:vehicle-history-app` | Consume GB32960/JT808 raw/event Kafka records, store compact history rows plus TDengine `raw_frames` `parsedJson`/`metadataJson`/`rawUri`, and expose TDengine-backed history APIs. | HTTP `20200` |
| Vehicle analytics | `:vehicle-analytics-app` | Consume JT808 event Kafka records, calculate daily mileage from reported GPS total mileage, and write metrics to `vehicle_stat_metric`. | HTTP `20300` |
| JT808 ingest | `:jt808-ingest-app` | Accept JT/T 808 TCP connections on the production receive port, parse raw frames, archive frame bytes, and produce raw/event envelopes to Kafka. | TCP `808`, HTTP `20400` |
| Yutong MQTT ingest | `:yutong-mqtt-app` | Connect to Yutong MQTT, parse subscribed messages, archive payloads, and produce raw/event envelopes to Kafka. | HTTP `20500` |
| Vehicle history | `:vehicle-history-app` | Consume GB32960/JT808/Yutong MQTT raw/event Kafka records, store compact history rows plus TDengine `raw_frames` `parsedJson`/`metadataJson`/`rawUri`, and expose TDengine-backed history APIs. | HTTP `20200` |
| Vehicle analytics | `:vehicle-analytics-app` | Consume JT808 event Kafka records, calculate daily mileage from reported GPS total mileage, and write metrics to `vehicle_stat_metric`. | HTTP `20310` |
## Kafka Contract
| Topic | Producer | Consumers | Purpose |
| --- | --- | --- | --- |
| `vehicle.raw.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Raw frame records. History writes the parsed JSON, metadata JSON, and raw URI to TDengine `raw_frames`. |
| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` | Normalized vehicle events keyed by VIN when available. |
| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Normalized vehicle events keyed by VIN when available. |
| `vehicle.dlq.gb32960.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed production or consumer processing. |
| `vehicle.raw.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app` | JT808 raw frame records for TDengine `raw_frames` and raw-frame troubleshooting APIs. |
| `vehicle.event.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app` | JT808 parsed location/session/alarm events keyed by phone or mapped VIN. |
| `vehicle.event.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` | JT808 parsed location/session/alarm events keyed by phone or mapped VIN; analytics uses location total mileage for daily mileage metrics. |
| `vehicle.dlq.jt808.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed JT808 production or consumer processing. |
| `vehicle.raw.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT raw payload records for TDengine `raw_frames` and troubleshooting APIs. |
| `vehicle.event.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT parsed telemetry events keyed by mapped VIN or device identity. |
| `vehicle.dlq.mqtt-yutong.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed Yutong MQTT production or consumer processing. |
| `vehicle.dlq.history.v1` | History app consumer error paths | Operators/replay tooling | Dead-letter records for failed history storage processing. |
Default local consumer groups:
Default consumer groups:
- History: `vehicle-history`
- Analytics statistics: `vehicle-stat`
@@ -35,13 +40,13 @@ History and analytics failures do not block GB32960 ACKs. They are downstream Ka
Authorization failures are rejected before the Kafka durability boundary and can be ACKed/rejected immediately according to protocol handling.
## Local Prerequisites
## Build Prerequisites
- Java 25 and Maven available on PATH.
- Local Kafka reachable at `127.0.0.1:9092`, plus Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer`.
- Production GB32960/JT808 session state uses Redis by default. For local
one-process smoke tests without Redis, set `SESSION_STORE=memory` explicitly.
- No repository-local Kafka bootstrap script or compose file was found during Task 8 verification on 2026-06-23.
- Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer` available on the ECS/operator host.
- Session state requires Redis. GB32960/JT808 ingest services keep live Netty
channels in-process, but session indexes and TTL metadata use Redis only.
- Redis is configured through Portainer environment variables such as `REDIS_HOST` and `REDIS_DATABASE=50`; there is no memory session-store fallback.
If your shell needs an explicit JDK:
@@ -55,99 +60,82 @@ export PATH="$JAVA_HOME/bin:/opt/homebrew/bin:$PATH"
From the repository root:
```bash
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
```
Expected result: `BUILD SUCCESS` and these runnable jars:
- `modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar`
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
- `modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar`
- `modules/apps/vehicle-history-app/target/vehicle-history-app.jar`
- `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar`
## Create Topics
Run these only after Kafka is reachable at `127.0.0.1:9092`:
Run these only on the ECS/operator host after Kafka is reachable. The default bootstrap address is the ECS private Kafka address used by the production stack:
```bash
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.jt808.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.jt808.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.jt808.v1 --partitions 3 --replication-factor 1
KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:-172.17.111.56:9092}"
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.jt808.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.jt808.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.jt808.v1 --partitions 3 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.mqtt-yutong.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.mqtt-yutong.v1 --partitions 12 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.mqtt-yutong.v1 --partitions 3 --replication-factor 1
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.history.v1 --partitions 3 --replication-factor 1
```
Optional sanity check:
```bash
kafka-topics --bootstrap-server 127.0.0.1:9092 --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808)\.v1'
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808|mqtt-yutong|history)\.v1'
```
## Start Split Services Locally
## Production Runtime
Run each command in a separate terminal from the repository root.
Production services run on ECS through Portainer/Docker. Do not start GB32960, JT808, Yutong MQTT, history, or analytics services on local developer machines; local machines are for code build, unit tests, and source inspection only.
GB32960 ingest:
Use `deploy/portainer/docker-compose.yml` as the production runtime manifest. The default stack is:
```bash
KAFKA_BROKERS=127.0.0.1:9092 \
GB32960_PORT=32960 \
HTTP_PORT=20100 \
SESSION_STORE=memory \
java --sun-misc-unsafe-memory-access=allow \
-jar modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar
```
- `gb32960-ingest-app`
- `jt808-ingest-app`
- `yutong-mqtt-app`
- `vehicle-history-app`
- `vehicle-analytics-app`
JT808 ingest:
`vehicle-history-app` intentionally has no raw archive volume. The history hot path is Kafka to TDengine: raw envelopes are stored in TDengine `raw_frames` with `parsedJson`, `metadataJson`, and `rawUri`. Ingest services may still write original `.bin` files as cold backup, but history API queries must not depend on a shared local archive mount.
```bash
KAFKA_BROKERS=127.0.0.1:9092 \
JT808_PORT=808 \
HTTP_PORT=20400 \
SESSION_STORE=memory \
java --sun-misc-unsafe-memory-access=allow \
-jar modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar
```
Keep `TDENGINE_HISTORY_ENABLED=true` in the `vehicle-history-app` environment so history reads and writes use TDengine in production.
Vehicle history:
```bash
KAFKA_BROKERS=127.0.0.1:9092 \
HTTP_PORT=20200 \
TDENGINE_HISTORY_ENABLED=true \
TDENGINE_HISTORY_DATABASE=vehicle_ts \
TDENGINE_JDBC_URL='jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts' \
TDENGINE_USERNAME=root \
TDENGINE_PASSWORD='<tdengine-password>' \
TDENGINE_TELEMETRY_FIELDS_ENABLED=false \
java --sun-misc-unsafe-memory-access=allow \
-jar modules/apps/vehicle-history-app/target/vehicle-history-app.jar
```
`SINK_ARCHIVE_PATH` is intentionally omitted from the history JVM. The history hot path is Kafka to TDengine: raw envelopes are stored in TDengine `raw_frames` with `parsedJson`, `metadataJson`, and `rawUri`. Ingest services may still write original `.bin` files as cold backup, but history API queries must not depend on a shared local archive mount.
Vehicle analytics:
```bash
KAFKA_BROKERS=127.0.0.1:9092 \
HTTP_PORT=20300 \
java --sun-misc-unsafe-memory-access=allow \
-jar modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar
```
`vehicle-analytics-app` is intentionally a metric runtime only. Latest-state APIs belong to `vehicle-state-service` and should be deployed as a separate consumer if that surface is needed.
`vehicle-analytics-app` is intentionally a metric runtime only. Latest-state APIs belong to `vehicle-state-service`, which stays outside the default reactor and should be built with `-Poptional-latest-state` and deployed as a separate consumer if that surface is needed.
## Health Verification
Run these from the ECS host or inside the matching container:
```bash
curl -sS http://127.0.0.1:20100/actuator/health
curl -sS http://127.0.0.1:20100/actuator/health/liveness
curl -sS http://127.0.0.1:20100/actuator/health/readiness
curl -sS http://127.0.0.1:20400/actuator/health
curl -sS http://127.0.0.1:20400/actuator/health/liveness
curl -sS http://127.0.0.1:20400/actuator/health/readiness
curl -sS http://127.0.0.1:20500/actuator/health
curl -sS http://127.0.0.1:20500/actuator/health/liveness
curl -sS http://127.0.0.1:20500/actuator/health/readiness
curl -sS http://127.0.0.1:20200/actuator/health
curl -sS http://127.0.0.1:20300/actuator/health
curl -sS http://127.0.0.1:20200/actuator/health/liveness
curl -sS http://127.0.0.1:20200/actuator/health/readiness
curl -sS http://127.0.0.1:20310/actuator/health
curl -sS http://127.0.0.1:20310/actuator/health/liveness
curl -sS http://127.0.0.1:20310/actuator/health/readiness
```
Expected: each endpoint returns `{"status":"UP"}` or an equivalent Spring Boot health JSON with top-level status `UP`.
Expected: each aggregate, liveness, and readiness endpoint returns `{"status":"UP"}` or an equivalent Spring Boot health JSON with top-level status `UP`.
## End-to-End Verification
@@ -173,8 +161,10 @@ Expected: the ACK file is non-empty and the decoded bytes begin with the GB32960
Verify only what was actually run in your environment:
```bash
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000
```
Use TDengine CLI or a JDBC client to verify history writes:
@@ -185,6 +175,8 @@ SELECT COUNT(*) FROM raw_frames WHERE protocol = 'GB32960';
SELECT COUNT(*) FROM vehicle_locations WHERE protocol = 'GB32960';
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'JT808';
SELECT COUNT(*) FROM jt808_locations WHERE protocol = 'JT808';
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'MQTT_YUTONG';
SELECT COUNT(*) FROM vehicle_locations WHERE protocol = 'MQTT_YUTONG';
```
Useful HTTP query checks after events are consumed. For `realtime_001.hex`, the fixture VIN is `LTEST000000000001`; replace `<date-from>`, `<date-to>`, and `<stat-date>` with values that cover the consumed record's event time:
@@ -193,7 +185,7 @@ Useful HTTP query checks after events are consumed. For `realtime_001.hex`, the
curl -sS 'http://127.0.0.1:20200/api/event-history/locations?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
curl -sS 'http://127.0.0.1:20200/api/event-history/raw-frames?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/dictionary'
curl -sS 'http://127.0.0.1:20300/api/vehicle-stat/LTEST000000000001/daily?date=<stat-date>'
curl -sS 'http://127.0.0.1:20310/api/vehicle-stat/LTEST000000000001/daily?date=<stat-date>'
```
Expected E2E result when Kafka and all services are running:
@@ -202,24 +194,23 @@ Expected E2E result when Kafka and all services are running:
- `vehicle.raw.gb32960.v1` receives a raw frame envelope.
- `vehicle.event.gb32960.v1` receives one or more normalized records.
- TDengine `raw_frames` receives GB32960/JT808 RAW rows with parsed JSON and metadata.
- TDengine location tables receive compact GB32960/JT808 location rows.
- TDengine `raw_frames` receives Yutong MQTT RAW rows with parsed JSON and metadata when MQTT payloads are consumed.
- TDengine location tables receive compact GB32960/JT808/Yutong MQTT location rows when applicable telemetry is present.
- JT808 analytics writes daily mileage rows to MySQL `vehicle_stat_metric` after consuming applicable `vehicle.event.jt808.v1` records.
Do not expect `vehicle-history-app` to create raw `.bin` archive files from Kafka raw records in the current implementation. `RawArchiveEventSink` can write archive files only when it receives `VehicleEvent.RawArchive.rawBytes()` inside the same JVM; the Kafka envelope carries only `RawArchiveRef` metadata.
Do not mark any of these as verified unless the matching command was run and the output was inspected.
## Local launchctl Deployment
## Runtime Location Policy
Use the templates under `deploy/local/launchctl/` for the local production-verification machine. They codify the current three-service deployment and keep the runtime knobs in one place:
Production services run on ECS through Portainer/Docker.
- GB32960 ingest: `com.lingniu.gb32960`, TCP `32960`, HTTP `20100`
- JT808 ingest: `com.lingniu.jt808`, TCP `808`, HTTP `20482`
- History: `com.lingniu.vehicle-history`, HTTP `20200`
- Do not create local service runners for GB32960, JT808, Yutong MQTT, history, or analytics.
- Do not run the service jars directly on developer machines for production verification.
- Keep local work limited to source edits, builds, and automated tests.
The launchctl history template does not inject `SINK_ARCHIVE_PATH`. Use TDengine `raw_frames` as the authoritative hot-query store; keep any ingest-side raw archive path as an optional cold backup for operator inspection.
For Portainer deployments, mirror this same shape: no history archive volume, history consumes raw/event Kafka topics and writes TDengine history rows.
For Portainer deployments, keep history without an archive volume: history consumes raw/event Kafka topics and writes TDengine history rows.
## Rollback Guidance
@@ -231,37 +222,37 @@ If the split deployment is unhealthy:
4. If Kafka itself is unavailable, stop `gb32960-ingest-app` or route GB32960 traffic to a previously verified release that preserves the Kafka durability boundary.
5. After rollback, compare Kafka consumer group lag and DLQ contents before resuming the split services.
Rollback commands depend on the deployment environment. For local testing, stop the three Java processes and restart the previous verified app jars or images.
Rollback commands depend on the deployment environment. For the default production stack, roll back the Portainer stack image version and let Docker restart the affected containers.
## Task 8 Verification Notes
Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`:
- Package build ran with `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true` and ended with `BUILD SUCCESS`.
- Package build should use the current active-app command shown in the Build section.
- Repository-local Kafka setup inspection found no Kafka script, no Docker Compose file, and no compose YAML within the searched repository paths.
- `nc -z -w 2 127.0.0.1 9092` exited `1`, so no local Kafka broker was reachable at `127.0.0.1:9092`.
- `kafka-topics`, `kafka-topics.sh`, `docker`, and `docker-compose` were not found on PATH.
- Kafka topic creation, service startup, health checks, Kafka record checks, archive checks, TDengine checks, stat output checks, and ACK observation were not run because the local Kafka prerequisite was absent.
## Latest Local Verification
## Latest Build Verification
Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`:
- Targeted module tests: `mvn -pl :sink-mq,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test` ended with `BUILD SUCCESS`; Maven reported 55 protocol/sink/core tests, 30 event-history tests, 14 vehicle-state tests, and 19 vehicle-stat tests with 0 failures/errors/skips.
- Split app tests: `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app test` ended with `BUILD SUCCESS`; Maven reported 6 app composition/default tests with 0 failures/errors/skips.
- Package verification: `mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true` ended with `BUILD SUCCESS` and repackaged all three app jars.
- Split app startup: not run in this verification pass because local Kafka was still absent.
- Kafka raw records: not verified; no local Kafka broker or Kafka CLI tools were available.
- Kafka event records: not verified; no local Kafka broker or Kafka CLI tools were available.
- Targeted module tests: `mvn -pl :sink-kafka,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test` ended with `BUILD SUCCESS`; Maven reported 55 protocol/sink/core tests, 30 event-history tests, 14 vehicle-state tests, and 19 vehicle-stat tests with 0 failures/errors/skips.
- Split app tests should cover the current active apps listed in the Service Roles section.
- Package verification should use the active-app package command from the Build section.
- Split app startup: not run in this verification pass because an ECS/Portainer runtime was outside that build-only check.
- Kafka raw records: not verified in that build-only check.
- Kafka event records: not verified in that build-only check.
- TDengine history rows: not verified; downstream services were not started without Kafka.
- Analytics output: not verified; downstream services were not started without Kafka.
- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above in a Kafka-backed local or staging environment.
- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above on ECS or staging.
## Remote Kafka Live Verification
Observed on 2026-06-23 with Kafka `114.55.58.251:9092`:
- Replaced the local `bootstrap-all` listener on TCP `32960` with `gb32960-ingest-app` on TCP `32960` and HTTP `20100`.
- `gb32960-ingest-app` handled TCP `32960` and HTTP `20100`.
- Ingest app health check `curl -sS http://127.0.0.1:20100/actuator/health` returned `{"status":"UP"}`.
- Kafka AdminClient confirmed topics `vehicle.raw.gb32960.v1` and `vehicle.event.gb32960.v1` existed, and created `vehicle.dlq.gb32960.v1`.
- The app accepted the live Hyundai platform connection from `8.134.95.166`, authenticated platform login, and received real GB32960 realtime/report/login/logout frames.
@@ -271,15 +262,12 @@ Observed on 2026-06-23 with Kafka `114.55.58.251:9092`:
This verifies the live path `GB32960 TCP 32960 -> gb32960-ingest-app -> Kafka raw/event topics` against the remote Kafka broker. Downstream `vehicle-history-app` and `vehicle-analytics-app` were not started in this pass.
## 2026-06-29 Local Production-Path Verification
## 2026-06-29 Superseded Local Verification Record
Observed on 2026-06-29 in worktree `.worktrees/vehicle-ingest-redesign-phase1`:
Observed on 2026-06-29 in worktree `.worktrees/vehicle-ingest-redesign-phase1`. This record is retained only as historical evidence of parser/storage behavior; do not repeat it as a deployment procedure because production services now run on ECS through Portainer/Docker.
- `jt808-ingest-app` was running through launchctl on TCP `808` and HTTP `20482`; `curl -sS http://127.0.0.1:20482/actuator/health` returned top-level status `UP`.
- `gb32960-ingest-app` was running through launchctl on TCP `32960` and HTTP `20100`; `curl -sS http://127.0.0.1:20100/actuator/health` returned top-level status `UP`.
- `vehicle-history-app` was running through launchctl on HTTP `20200`; `curl -sS http://127.0.0.1:20200/actuator/health` returned top-level status `UP`.
- External JT808 traffic on TCP `808` was parsed and written through Kafka into TDengine. Direct TDengine checks showed more than `6000` JT808 `raw_frames`, more than `1900` JT808 `vehicle_locations`, and more than `15000` JT808 `telemetry_fields`.
- External JT808 traffic on TCP `808` was parsed and written through Kafka into TDengine. Direct TDengine checks showed more than `6000` JT808 `raw_frames` and more than `1900` JT808 `vehicle_locations`; current production history does not maintain a `telemetry_fields` table.
- A GB32960 synthetic realtime frame sent to TCP `32960` received a binary ACK beginning with `2323`. The same VIN `LTEST202606290001` was queryable from TDengine-backed history APIs with speed, mileage, longitude, and latitude fields.
- History raw and event Kafka consumption used separate bindings so raw-frame writes remained current while location/field event ingestion continued.
This verifies the local path `GB32960/JT808 TCP -> protocol app -> Kafka raw/event topics -> vehicle-history-app -> TDengine raw_frames/location tables -> Swagger/API query`.
The verified behavior was `GB32960/JT808 TCP -> protocol app -> Kafka raw/event topics -> vehicle-history-app -> TDengine raw_frames/location tables -> Swagger/API query`; the deployment method from that historical run is superseded.

View File

@@ -0,0 +1,888 @@
# Go Vehicle Gateway 旁路验证记录
验证时间2026-07-01 22:00-22:10 CST
## 部署版本
- Git commit`60ca42e`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-60ca42e-20260701220622`
- 部署方式ECS `115.29.187.205` 上 Docker Compose 旁路部署
- Compose 目录:`/opt/lingniu-go`
## 旁路端口和 topic
为了不影响现有 Java 生产服务,本次没有占用生产端口和生产 topic。
| 项 | 值 |
|---|---|
| GB32960 旁路 TCP | `115.29.187.205:23296 -> container :32960` |
| JT808 旁路 TCP | `115.29.187.205:18080 -> container :808` |
| Realtime API | `115.29.187.205:20210` |
| GB32960 RAW topic | `vehicle.raw.go.gb32960.v1` |
| JT808 RAW topic | `vehicle.raw.go.jt808.v1` |
| Yutong MQTT RAW topic | `vehicle.raw.go.yutong-mqtt.v1` |
| Unified topic | `vehicle.event.go.unified.v1` |
## 容器状态
ECS 上四个 Go 容器均已启动:
```text
go-vehicle-gateway Up
go-history-writer Up
go-stat-writer Up
go-realtime-api Up
```
## Kafka 验证
`18080` 注入 JT808 样例帧后,`vehicle.raw.go.jt808.v1`
`vehicle.event.go.unified.v1` 均消费到统一 JSON envelope。
样例解析结果包含:
- `protocol=JT808`
- `message_id=0x0200`
- `phone=013307795425`
- `longitude=121.069881`
- `latitude=30.590151`
- `speed_kmh=23`
- `total_mileage_km=10241.2`
- 表 27 附加项 `0x01` 原始值 `0001900C`
## TDengine 验证
目标库:`lingniu_vehicle_ts`
```text
raw_frames count = 3
vehicle_locations count = 2
vehicle_mileage_points count = 2
```
最新 RAW 标识:
```text
GB32960 | LNBSCB3D4R1234567 | LNBSCB3D4R1234567 |
JT808 | JT808:013307795425 | | 013307795425
JT808 | JT808:013307795425 | | 013307795425
```
说明JT808 样例 phone 当前没有在 `vehicle_identity_binding` 中解析到 VIN因此
TDengine tag 使用 `vehicle_key=JT808:013307795425`
## Redis 验证
`23296` 注入带 VIN 的 GB32960 合成帧后Realtime API 可查询:
```text
GET /api/realtime/vehicles/LNBSCB3D4R1234567
```
返回字段包含:
- `vin=LNBSCB3D4R1234567`
- `protocols=["GB32960"]`
- `online=true`
- `speed_kmh=30`
- `total_mileage_km=10000`
- `soc_percent=85`
- `longitude=121`
- `latitude=30.56`
## MySQL 统计验证
`23296` 注入带 VIN 的 GB32960 合成帧后,`vehicle_daily_metric` 写入:
```text
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_mileage_km | 0.000 | 10000.000 | 10000.000 | 1
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_total_mileage_km | 10000.000 | 10000.000 | 10000.000 | 1
```
说明:本次 GB32960 合成帧的设备时间为测试值,因此统计日期为 `2020-07-01`
## 已修复问题
- TDengine WebSocket 普通 `INSERT` 使用参数占位符时,字符串未按预期加引号,导致语法错误。已在 `60ca42e` 中改为显式 SQL literal 转义。
- MySQL Go DSN 中 `charset=utf8mb4%2Cutf8` 会被 driver 传给 MySQL导致语法错误ECS 旁路 env 已改为 `charset=utf8mb4`
- TDengine ECS 当前可用 root 密码与早先记录不一致,本次旁路 env 已按实际可认证值修正。
## 未完成
- 未切换真实生产端口 `32960/808`
- 未将真实外部 32960/808 流量转发到 Go 旁路端口。
- 未开启宇通 MQTT 正式订阅。
- JT808 样例未解析到 VIN需维护 `vehicle_identity_binding` 后再验证 808 每日统计。
## 2026-07-01 22:22 真实归档帧复验
部署版本已更新:
- Git commit`5f4f4fd`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5f4f4fd-20260701222128`
本轮补充验证内容:
- JT808 支持 2019 版 versioned header真实帧 `0x0200` 可解析出 `phone=14894135060``longitude=117.178187``latitude=30.570155``speed_kmh=33.4``total_mileage_km=7868.9`
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 可解析 `0x01` 整车、`0x02` 驱动电机、`0x03` 燃料电池、`0x04` 发动机、`0x05` 位置、`0x06` 极值、`0x07` 报警、`0x08` 电压、`0x09` 温度;后续 `0x30` 广东燃料电池扩展暂以 unknown raw 保留。
- GB32960 真实帧字段包含 `fuel_cell_hydrogen_consumption_kg_per_100km=1.8``longitude=120.800326``latitude=31.634907``total_mileage_km=53490.9`
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回上述经纬度和氢耗字段。
- TDengine `vehicle_locations``vehicle_mileage_points` 已写入真实 GB32960 位置和里程点:
```text
2020-07-01 08:09:22 | GB32960 | LB9A32A21R0LS1707 | lon=120.800326 | lat=31.634907 | mileage=53490.9
```
后续仍需补齐:
- JT808 真实 phone `14894135060` 仍未解析到 VIN需维护身份绑定后再验证 VIN 维度实时和统计。
## 2026-07-01 22:28 广东燃料电池扩展复验
部署版本已更新:
- Git commit`a15de91`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-a15de91-20260701222743`
本轮补充验证内容:
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 的 vendor 尾部已从 `unknown_unit` 补为结构化块:
`0x30 gd_fc_stack``0x31 gd_fc_auxiliary``0x32 gd_fc_dcdc``0x33 gd_fc_air_conditioner`
`0x34 gd_fc_vehicle_info``0x80 gd_fc_demo_extension``0x83 gd_fc_vendor_tlv`
- Kafka `vehicle.raw.go.gb32960.v1` 最新消息包含上述 block并输出关键扁平字段
`gd_fc_stack_hydrogen_inlet_pressure_kpa=97``gd_fc_stack_cell_count=108`
`gd_fc_vehicle_hydrogen_mass_kg=4.3``gd_fc_demo_stack_temp_c=65`
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回 vendor 扁平字段:
```text
gd_fc_dcdc_controller_temp_c=47
gd_fc_demo_stack_temp_c=65
gd_fc_stack_cell_count=108
gd_fc_stack_frame_cell_count=108
gd_fc_stack_hydrogen_inlet_pressure_kpa=97
gd_fc_stack_water_outlet_temp_c=65
gd_fc_vehicle_hydrogen_mass_kg=4.3
```
- TDengine `raw_frames` 中 GB32960 行数增加到 `4`,最新 RAW 标识:
```text
2026-07-01 14:28:24.869 | 172.20.0.1:47082 | GB32960 | LB9A32A21R0LS1707 | OK
```
后续仍需补齐:
- JT808 真实 phone `14894135060` 到 VIN 的绑定数据维护与统计复验。
- Go 旁路尚未切换到生产 `32960/808` 端口;仍运行在 `23296/18080`
## 2026-07-01 22:43 JT808 前导零 phone 身份解析复验
部署版本已更新:
- Git commit`6b55694`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-6b55694-20260701224300`
本轮修复内容:
- JT808 2011 老版包头的终端手机号按 BCD[6] 解析会保留前导 `0`,例如包头为 `013079963379`
- `vehicle_identity_binding.phone` 中实际维护的是去前导零后的手机号 `13079963379`
- 身份解析候选键已调整为先查原始 phone再查去前导零 phone之后再按 `device_id``plate` 降级。
真实帧复验样本:
- 归档帧:`/opt/lingniu/vehicle-ingest/gb32960/archive/2026/07/01/JT808/unknown/1782903942845000.bin`
- 消息:`0x0200` 位置信息汇报
- 包头 phone`013079963379`
- 绑定表 phone`13079963379`
- 解析 VIN`LKLG7C4E3NA774736`
Kafka `vehicle.raw.go.jt808.v1` 最新消息确认:
```text
vin=LKLG7C4E3NA774736
phone=013079963379
parsed.header.phone_trim_zero=13079963379
parsed.identity.resolved=true
parsed.identity.source=phone
parsed.identity.value=13079963379
```
Realtime API 确认:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736
longitude=119.557997
latitude=29.049092
speed_kmh=0
total_mileage_km=0
device_time=2026-07-01T19:05:40+08:00
```
TDengine 确认:
```text
raw_frames:
2026-07-01 14:49:10.913 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | message_id=512
vehicle_locations:
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | lon=119.557997 | lat=29.049092 | speed=0 | mileage=0
vehicle_mileage_points:
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | mileage=0
```
MySQL `vehicle_daily_metric` 确认:
```text
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
```
说明:该真实位置帧携带的表 27 附加项 `0x01` 总里程值为 `0`,因此本次只验证身份解析、链路落地和差值统计写入;要验证非零日里程,需要回放或等待同一 VIN 的非零总里程连续位置点。
## 2026-07-01 22:55 Kafka 发布重试部署复验
部署版本已更新:
- Git commit`7e7066d`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7e7066d-20260701225503`
本轮修复内容:
- gateway 的 Kafka 发布增加同步重试封装。
- 默认配置:
- `KAFKA_PUBLISH_ATTEMPTS=3`
- `KAFKA_PUBLISH_BACKOFF_MS=100`
- `KAFKA_PUBLISH_TIMEOUT_MS=3000`
- RAW 仍先于 unified event 写入RAW 写失败不会继续写 unified。
- 单元测试覆盖首次失败后成功、重试耗尽返回错误、上下文取消时停止重试。
部署后容器状态:
```text
go-vehicle-gateway | go-7e7066d-20260701225503 | Up
go-history-writer | go-7e7066d-20260701225503 | Up
go-stat-writer | go-7e7066d-20260701225503 | Up
go-realtime-api | go-7e7066d-20260701225503 | Up
```
真实 808 帧回放复验:
```text
source_endpoint=172.20.0.1:51966
protocol=JT808
message_id=0x0200
phone=013079963379
vin=LKLG7C4E3NA774736
parsed.identity.value=13079963379
```
Kafka `vehicle.raw.go.jt808.v1` 确认新消息写入partition 6 offset 从 `1` 增加到 `2`。最新消息包含:
```text
received_at_ms=1782917751903
source_endpoint=172.20.0.1:51966
vin=LKLG7C4E3NA774736
fields.longitude=119.557997
fields.latitude=29.049092
fields.total_mileage_km=0
```
TDengine `raw_frames` 确认新 RAW 落地:
```text
2026-07-01 14:55:51.903 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | source=172.20.0.1:51966
```
Realtime API 确认 Redis 已刷新:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736
received_at_ms=1782917751903
source_endpoint=172.20.0.1:51966
longitude=119.557997
latitude=29.049092
total_mileage_km=0
```
说明:当前重试层只能覆盖 Kafka 短暂抖动;如果 Kafka 长时间不可用,仍需后续增加本地磁盘 spool/WAL 和恢复补发。
## 2026-07-01 23:02 切换 Go 到生产 32960/808 端口
本轮操作:
- 停止 ECS 上原有 Java Docker 容器:
- `gb32960-ingest-app`
- `jt808-ingest-app`
- `yutong-mqtt-app`
- `vehicle-history-app`
- `vehicle-analytics-app`
- `vehicle-state-app`
- `telemetry-field-parser-app`
- 保留 Go sidecar 和 Portainer agent。
- 修改 `/opt/lingniu-go/.env`
- `GO_GB32960_TCP_PORT=32960`
- `GO_JT808_TCP_PORT=808`
- Go gateway 使用镜像:
- `crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-243de5c-20260701230028`
- 开启 Kafka 本地 spool
- host`/opt/lingniu-go/spool/gateway`
- container`/data/spool/gateway`
端口验证:
```text
0.0.0.0:32960 -> go-vehicle-gateway:32960
0.0.0.0:808 -> go-vehicle-gateway:808
18080/23296 -> 已释放
```
启动日志确认:
```text
kafka durable spool enabled dir=/data/spool/gateway replay_interval_ms=1000
tcp listener started protocol=GB32960 addr=[::]:32960
tcp listener started protocol=JT808 addr=[::]:808
```
真实连接确认:
```text
GB32960 external remote: 8.134.95.166:50084
JT808 external remote: 115.231.168.135:13801
```
生产 `808` 回放复验:
```text
source_endpoint=172.20.0.1:54680
vin=LKLG7C4E3NA774736
phone=013079963379
longitude=119.557997
latitude=29.049092
total_mileage_km=0
```
TDengine 最新 RAW 复验显示生产 808 已有真实外部数据持续落地:
```text
2026-07-01 15:03:02.960 | JT808 | LKLG7C4E1NA774802 | 013079963291 | 222.66.200.68:28394
2026-07-01 15:03:02.114 | JT808 | | 14894135583 | 122.152.221.156:33871
2026-07-01 15:03:01.352 | JT808 | | 013307795519 | 115.231.168.135:13801
2026-07-01 15:03:01.330 | JT808 | LKLG7C4E3NA774736 | 013079963379 | 222.66.200.68:28393
```
Kafka spool 目录确认:
```text
/opt/lingniu-go/spool/gateway file count = 0
```
说明:生产 808 中仍有部分 phone 未解析出 VIN后续需要继续维护 `vehicle_identity_binding` 的 phone/device_id/plate 到 VIN 映射,或者增强 808 注册/鉴权帧中的 identity 回写。
## 2026-07-01 23:09 808 里程统计防 0 污染复验
部署版本已更新:
- Git commit`d347525`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-d347525-20260701230845`
本轮修复内容:
- `stat-writer` 不再把 `total_mileage_km <= 0` 的采样写入 MySQL 每日指标。
- MySQL upsert 对历史 `first_total_mileage_km=0` / `latest_total_mileage_km=0` 做纠偏;后续首次有效总里程会替换掉历史 0。
- `realtime-api` 合并实时快照时,非正 `total_mileage_km` 不再覆盖已有正值;速度、位置、状态等其他字段仍按事件时间更新。
测试验证:
```text
go test ./...
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
```
生产回放验证:
- 第一条 808 帧:`device_time=2026-07-01T23:09:35+08:00``total_mileage_km=12345.6`
- 第二条 808 帧:`device_time=2026-07-01T23:09:36+08:00``total_mileage_km=0`
- 两条帧使用同一 VIN`LKLG7C4E3NA774736`
Redis merged 快照确认:第二条 0 里程帧没有覆盖已有有效总里程。
```text
event_time_ms=1782918576000
source_endpoint=172.20.0.1:40946
speed_kmh=0
longitude=119.557997
latitude=29.049092
total_mileage_km=12345.6
field_times_ms.total_mileage_km=1782918575000
```
MySQL `vehicle_daily_metric` 确认0 里程采样未进入统计sample_count 只因有效总里程采样增加一次。
```text
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=13 | first=12345.600 | latest=12345.600
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 12345.600 | sample_count=13 | first=12345.600 | latest=12345.600
```
说明:生产 TDengine 中已经存在大量 808 非零 `total_mileage_km` 采样,但许多 phone 尚未映射到 VIN因此 MySQL 按 VIN 的统计只会覆盖已解析 VIN 的车辆。下一步应继续补齐 `vehicle_identity_binding`,让更多 808 车辆进入统计。
## 2026-07-01 23:23 808 注册身份自动维护复验
部署版本已更新:
- Git commit`7b44f97`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7b44f97-20260701232200`
本轮修复内容:
- 808 `0x0100` 注册帧解析 `province``city``manufacturer``device_type``device_id``plate_color``plate`
- 808 `0x0102` 鉴权帧解析 `auth_token`
- Go gateway 在 identity resolve 后写入 `jt808_registration`,普通上行也会刷新 `latest_seen_at`
- 注册帧通过车牌解析到 VIN 后,会尝试把 phone/device/plate 反写到 `vehicle_identity_binding`
- 注册帧车牌支持 GBK 解码,避免 `Incorrect string value` 写入错误。
验证命令:
```text
go test ./...
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
```
ECS 运行状态:
```text
go-vehicle-gateway | go-7b44f97-20260701232200 | 0.0.0.0:808->808, 0.0.0.0:32960->32960
go-history-writer | go-7b44f97-20260701232200 | Up
go-stat-writer | go-7b44f97-20260701232200 | Up
go-realtime-api | go-7b44f97-20260701232200 | 0.0.0.0:20210->20210
```
生产日志确认:
- 新版本启动后真实 808、32960 连接持续进入。
- GBK 车牌注册帧不再出现 `Incorrect string value`
MySQL `jt808_registration` 确认:
```text
registration_rows=214
distinct_phone=214
vin_rows=74
plate_rows=74
device_rows=72
013079963379 | device_id=9963379 | plate=沪A06788F | vin=LKLG7C4E3NA774736 | latest_registered_at=2026-07-01 23:23:01
013079963321 | device_id=9963321 | plate=沪A59613F | vin=LKLG7C4EXNA774796 | latest_registered_at=2026-07-01 23:23:04
```
说明:`jt808_registration.phone` 保存 808 包头原始 phone`vehicle_identity_binding.phone` 中已有部分去前导零 phone。Go resolver 会同时查询原始 phone 和去前导零 phone。
## 2026-07-01 23:28 生产链路与指标清理复验
本轮复验目标:
- 确认 Go gateway 的 Kafka durable spool 是重启窗口残留,而不是持续发布失败。
- 确认 stat-writer / history-writer / realtime-api 的 Kafka consumer group 无 lag。
- 确认真实 808 车辆 registration、TDengine RAW/里程点、Redis 实时查询均在持续更新。
- 清理之前生产合成回放产生的 JT808 当日指标测试数据。
Kafka topic / consumer group
```text
vehicle.raw.go.jt808.v1 有持续 offset
vehicle.raw.go.gb32960.v1 有持续 offset
vehicle.event.go.unified.v1 有持续 offset
go-stat-writer LAG=0
go-history-writer LAG=0
go-realtime-api LAG=0
```
TDengine
```text
raw_frames:
JT808 2922
GB32960 27
vehicle_mileage_points:
JT808 2050
GB32960 2
```
Redis 实时查询样例:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736
vin=LKLG7C4E3NA774736
source_endpoint=222.66.200.68:29646
plate=沪A06788F
longitude=119.556866
latitude=29.047911
total_mileage_km=12345.6
```
说明:该 Redis merged 快照保留了此前有效 `total_mileage_km=12345.6`,后续真实 808 位置帧上报的 `total_mileage_km=0` 不会覆盖该字段。TDengine 中该 VIN 后续真实里程点的 `total_mileage_km` 均为 0因此 stat-writer 按“只统计正总里程”的规则不再写 JT808 每日指标。
MySQL 指标清理:
```text
删除前:
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_mileage_km | first=0 | latest=0 | sample_count=7
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_total_mileage_km | first=0 | latest=0 | sample_count=7
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
删除后等待真实流量窗口:
JT808 2026-07-01 metric rows = 0
```
清理原因:
- `LKLG7C4E3NA774736` 的 12345.6 来自此前生产连通性合成回放,不是真实平台上报。
- `LKLG7C4E2NA774775` 的 0 指标来自旧逻辑污染,当前 stat-writer 已跳过非正总里程。
Kafka spool
```text
23:24 spool_count=904
23:25 spool_count=862
23:28 spool_count=722
new_spool_1min=0
```
说明spool 正在回放下降,且最近 1 分钟没有新增文件;生产 gateway 日志未见 `Incorrect string value`、Kafka publish error 或 replay error。
## 2026-07-01 23:34 每日指标查询 API 复验
部署版本已更新:
- Git commit`56f4811`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-56f4811-20260701233311`
新增接口:
```text
GET /api/stats/daily-metrics
```
查询参数:
```text
vin 可选
protocol 可选GB32960 / JT808 / YUTONG_MQTT
metricKey 可选daily_mileage_km / daily_total_mileage_km
dateFrom 可选YYYY-MM-DD
dateTo 可选YYYY-MM-DD
limit 可选,默认 50最大 1000
offset 可选,默认 0
```
生产可访问样例:
```text
http://115.29.187.205:20210/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01&limit=10
```
返回确认:
```json
{
"items": [
{
"vin": "LB9A32A21R0LS1707",
"stat_date": "2020-07-01",
"protocol": "GB32960",
"metric_key": "daily_mileage_km",
"metric_value": 0,
"metric_unit": "km",
"first_total_mileage_km": 53490.9,
"latest_total_mileage_km": 53490.9,
"sample_count": 3,
"calculation_method": "TOTAL_MILEAGE_DIFF",
"created_at": "2026-07-01 22:14:13",
"updated_at": "2026-07-01 22:28:25"
},
{
"vin": "LB9A32A21R0LS1707",
"stat_date": "2020-07-01",
"protocol": "GB32960",
"metric_key": "daily_total_mileage_km",
"metric_value": 53490.9,
"metric_unit": "km",
"first_total_mileage_km": 53490.9,
"latest_total_mileage_km": 53490.9,
"sample_count": 3,
"calculation_method": "TOTAL_MILEAGE_DIFF",
"created_at": "2026-07-01 22:14:13",
"updated_at": "2026-07-01 22:28:25"
}
],
"limit": 10,
"offset": 0,
"total": 2
}
```
JT808 真实生产查询:
```text
http://115.29.187.205:20210/api/stats/daily-metrics?protocol=JT808&dateFrom=2026-07-01&dateTo=2026-07-01&limit=10
```
返回 `total=0`。原因是此前合成回放产生的测试指标已清理,当前真实 808 已绑定 VIN 的车辆仍上报 `total_mileage_km=0`stat-writer 按规则不写非正总里程指标。
实时接口回归:
```text
http://115.29.187.205:20210/api/realtime/vehicles/LKLG7C4E3NA774736/online
```
返回 `online=true`,说明新增 stats API 没有影响 Redis 实时查询。
发布后 spool 观察:
```text
发布后立即spool_count=1101new_spool_1min=102
等待 60 秒spool_count=1033new_spool_1min=0
```
说明:本次发布重启窗口产生的 spool 正在回放下降,且最近 1 分钟没有新增gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。
## 2026-07-01 23:50 生产端口接管与 RAW 查询 API 复验
部署版本:
- Git commit`5937132`
- 镜像:`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5937132-20260701234220`
生产端口状态:
```text
go-vehicle-gateway 0.0.0.0:808->808/tcp
go-vehicle-gateway 0.0.0.0:32960->32960/tcp
```
旧 Java 容器状态:
```text
gb32960-ingest-app restart=no status=exited
jt808-ingest-app restart=no status=exited
yutong-mqtt-app restart=no status=exited
vehicle-history-app restart=no status=exited
vehicle-analytics-app restart=no status=exited
vehicle-state-app restart=no status=exited
telemetry-field-parser-app restart=no status=exited
```
生产连接确认:
- JT808 端口 `808` 已收到外部连接:`222.66.200.68``115.231.168.135`
- GB32960 端口 `32960` 已收到外部连接:`8.134.95.166``117.160.0.65`
- ECS 当前没有 `8089` 监听。
RAW 查询 API
```text
GET /api/history/raw-frames
```
生产可访问样例:
```text
http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
```
接口复验结果:
```text
JT808_RAW:
total=1
protocol=JT808
vin=LKLG7C4E3NA774736
phone=013079963379
message_id=512
message_id_hex=0x0200
parse_status=OK
raw_hex_len=94
parsed_json_len=651
fields_json_len=186
GB32960_RAW:
total=1
protocol=GB32960
vin=LB9A32A21R0LS1707
message_id=2
message_id_hex=0x0002
parse_status=OK
raw_hex_len=1548
parsed_json_len=3698
fields_json_len=563
```
同轮回归验证:
```text
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
online=true
protocols=["JT808"]
GET /api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2026-07-01&limit=1
total=1
protocol=GB32960
metric_key=daily_mileage_km
calculation_method=TOTAL_MILEAGE_DIFF
```
说明:
- RAW 查询已返回 `raw_hex``parsed_json``fields_json`,可以用于查看完整解析结果。
- 日统计和实时查询在生产端口接管后仍可用。
## 2026-07-02 00:00 原生 systemd 部署与宇通 MQTT 复验
根据本次 goal 的新要求Go 接入链路后续不再使用 Docker 部署。本轮已中断 Docker 镜像构建路径,改为 ECS 原生 Linux 二进制 + systemd。
部署版本:
- Git commit`594ab2d`
- 部署目录:`/opt/lingniu-go-native`
- 当前 release`/opt/lingniu-go-native/releases/594ab2d`
- 当前指针:`/opt/lingniu-go-native/current -> /opt/lingniu-go-native/releases/594ab2d`
systemd 服务:
```text
lingniu-go-gateway.service active
lingniu-go-history-writer.service active
lingniu-go-stat-writer.service active
lingniu-go-realtime-api.service active
```
端口确认:
```text
[::]:32960 gateway
[::]:808 gateway
[::]:20210 realtime-api
```
Go Docker 容器已停止:
```text
go-vehicle-gateway Exited
go-history-writer Exited
go-stat-writer Exited
go-realtime-api Exited
```
宇通 MQTT
```text
yutong mqtt client started
mqtt subscribed broker=ssl://cpxlm.axxc.cn:38883 topic=/ytforward/shln/+ qos=1
```
说明MQTT broker 存在短连接 `EOF` 后自动重连现象,但订阅后已收到真实数据并落 RAW。
RAW 查询复验:
```text
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=3
total=3
protocol=YUTONG_MQTT
vin=LMRKH9AC1R1004131
vehicle_key=LMRKH9AC1R1004131
parse_status=OK
parsed_json_len=1104
fields_json_len=116
```
同轮回归:
```text
GET /api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
total=1
parse_status=OK
GET /api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
total=1
parse_status=OK
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
online=true
protocols=["JT808"]
```
本轮落地文件:
- `deploy/systemd/README.md`
- `deploy/systemd/lingniu-go-gateway.service`
- `deploy/systemd/lingniu-go-history-writer.service`
- `deploy/systemd/lingniu-go-stat-writer.service`
- `deploy/systemd/lingniu-go-realtime-api.service`
## 2026-07-02 00:04 宇通 MQTT EOF 诊断
现象:
Go 原生部署后,`lingniu-go-gateway` 日志中出现多次:
```text
mqtt connection lost error=EOF
mqtt connection lost error=write: broken pipe
mqtt subscribed topic=/ytforward/shln/+ qos=1
```
10 分钟窗口统计:
```text
mqtt subscribed = 33
mqtt connection lost = 32
```
对照旧 Java `yutong-mqtt-app` 历史日志,停止前也存在同类行为:
```text
mqtt endpoint [yutong] received topic=/ytforward/shln/1 bytes=574
mqtt endpoint [yutong] connection lost
Caused by: java.io.EOFException
mqtt endpoint [yutong] re-subscribed topic=/ytforward/shln/+ qos=1
mqtt endpoint [yutong] received topic=/ytforward/shln/3 bytes=586
```
判断:
- EOF/短连接不是 Go TLS 支持或 systemd 原生部署新引入的问题。
- 旧 Java 和新 Go 都表现为订阅、收到数据、broker 断开、自动重连。
- 当前链路可持续收到宇通 MQTT 数据并写入 TDengine RAW。
复验:
```text
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=1
total=1
protocol=YUTONG_MQTT
vin=LMRKH9ACXR1004094
parse_status=OK
parsed_json_len=1080
fields_json_len=115
```
运维建议:
- 单条 EOF 不作为不可用判断。
-`YUTONG_MQTT` RAW 入库增长、最近成功 `mqtt subscribed` 时间、服务 `systemctl is-active` 作为可用性判断。
- 如果 EOF 频率继续升高且 RAW 不再增长,再联系宇通侧确认 broker 长连接策略、同一 clientId 并发限制或网络出口策略。

View File

@@ -0,0 +1,158 @@
# Go Vehicle Gateway 验证手册
本文档用于验证 Go 重构运行面:
- `go-vehicle-gateway`GB32960 TCP、JT808 TCP、宇通 MQTT 接入。
- `go-history-writer`Kafka RAW 写 TDengine。
- `go-stat-writer`Kafka RAW 写 MySQL 每日指标。
- `go-realtime-api`Kafka unified event 写 Redis并提供实时查询 API。
## 本地构建验证
```bash
cd go/vehicle-gateway
go test ./...
go build ./cmd/gateway
go build ./cmd/history-writer
go build ./cmd/stat-writer
go build ./cmd/realtime-api
```
## 生产原生部署验证
```bash
python3 tools/go_prod_acceptance.py --date 2026-07-02
```
该命令会聚合执行:
- `go_systemd_prod_smoke.py`:应用 ECS systemd 服务 active/enabled、端口归属和 gateway spool
- `go_kafka_prod_smoke.py`Kafka topic 和消费组 lag
- `go_native_prod_smoke.py`HTTP API、RAW `parsed_json`、历史核心表 `frame_id` 回链、每日指标公式、Redis 实时新鲜度、TDengine/MySQL/Redis 数据链路
如需单独检查 systemd
```bash
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
```
脚本通过 SSH 检查:
- `lingniu-go-gateway.service`
- `lingniu-go-history-writer.service`
- `lingniu-go-stat-writer.service`
- `lingniu-go-realtime-api.service`
- 上述服务均为 `active``enabled`
- `808``32960``gateway` 监听
- `20210``realtime-api` 监听
- `/opt/lingniu-go-native/spool/gateway` 无待回放文件
## 生产环境变量
当前 goal 的生产面使用 ECS 原生 systemd 部署,不再以 Docker/Portainer 作为生产运行方式。以下环境变量维护在 `/opt/lingniu-go-native/env/*.env`
```bash
KAFKA_BROKERS=172.17.111.56:9092
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
IDENTITY_MYSQL_DSN=${MYSQL_DSN}
VEHICLE_IDENTITY_TABLE=vehicle_identity_binding
REDIS_ADDR=r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379
REDIS_PASSWORD=<password>
REDIS_DB=50
```
宇通 MQTT 开启时再配置:
```bash
YUTONG_MQTT_ENABLED=true
YUTONG_MQTT_URI=<mqtt-uri>
YUTONG_MQTT_TOPICS=/ytforward/shln/+
YUTONG_MQTT_CLIENT_ID=lingniu-go-yutong-mqtt
YUTONG_MQTT_USERNAME=<username>
YUTONG_MQTT_PASSWORD=<password>
```
## ECS 进程检查
```bash
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
ss -lntp | egrep ':(808|32960|20210)\b'
```
## Kafka 验证
优先使用 smoke 脚本做可重复检查:
```bash
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
```
脚本会检查 `vehicle.raw.go.gb32960.v1``vehicle.raw.go.jt808.v1``vehicle.raw.go.yutong-mqtt.v1``vehicle.event.go.unified.v1` 是否存在,并汇总 `go-history-writer``go-stat-writer``go-realtime-api` 的消费 lag。运行环境需要 SSH 免密或已建立可用的 SSH 认证方式。
手工核对命令:
```bash
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.go.jt808.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.go.gb32960.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.raw.go.yutong-mqtt.v1 --max-messages 1 --timeout-ms 10000
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
--topic vehicle.event.go.unified.v1 --max-messages 1 --timeout-ms 10000
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-history-writer
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-stat-writer
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
--describe --group go-realtime-api
```
## TDengine 验证
```sql
USE lingniu_vehicle_ts;
SHOW STABLES;
SELECT COUNT(*) FROM raw_frames;
SELECT COUNT(*) FROM vehicle_locations;
SELECT COUNT(*) FROM vehicle_mileage_points;
```
## MySQL 验证
```sql
SELECT protocol, metric_key, COUNT(*)
FROM vehicle_daily_metric
GROUP BY protocol, metric_key;
SELECT *
FROM vehicle_daily_metric
WHERE metric_key IN ('daily_mileage_km', 'daily_total_mileage_km')
ORDER BY updated_at DESC
LIMIT 20;
```
## Redis 实时查询
```bash
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>'
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/online'
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/protocols/JT808'
```
## 验收证据
正式切换前至少记录:
- 一个真实 GB32960 VIN 的 RAW、location、mileage point、Redis snapshot。
- 一个真实 JT808 VIN/phone 的 RAW、location、mileage point、daily metric、Redis snapshot。
- 一个真实宇通 MQTT VIN 或 device_id 的 RAW、Redis snapshot。
- `go-history-writer``go-stat-writer` 重启后 Kafka offset 能继续推进。

View File

@@ -4,18 +4,20 @@
The analytics app can calculate daily mileage from JT808 telemetry only. It consumes `vehicle.event.jt808.v1` and saves the current daily result into the common `vehicle-stat` metric repository.
Only supported message backbone: Kafka.
## Storage
There is no separate JT808 daily-mileage table and no Redis mileage state. The derived value is stored as one `daily_mileage_km` metric row in the common JDBC/MySQL table `vehicle_stat_metric`; the first and latest GPS total mileage values are kept on that same row as calculation source columns.
Runtime state: none outside `vehicle_stat_metric`. There is no separate JT808 daily-mileage table. The derived value is stored as one `daily_mileage_km` metric row in the common JDBC/MySQL table `vehicle_stat_metric`; the local-day minimum and maximum GPS total mileage values are kept on that same row as calculation source columns.
The JT808 daily-mileage value is calculated from the GPS total mileage reported in location additional information:
```text
daily_mileage_km = last_total_mileage_km - first_total_mileage_km
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
calculation_method = JT808_TOTAL_MILEAGE_DIFF
```
The first valid JT808 location point for a vehicle and local day stores the day start total on the metric row and writes `daily_mileage_km=0.0`. Later ordered points update the latest total on the same row and rewrite the derived `daily_mileage_km`. If a later total mileage is lower than the stored latest total, the previous latest total remains.
The first valid JT808 location point for a vehicle and local day stores both calculation endpoints on the metric row and writes `daily_mileage_km=0.0`. Later ordered or replayed points update the lower endpoint when the reported GPS total mileage is smaller, update the upper endpoint when it is larger, and rewrite the derived `daily_mileage_km` on the same row.
## Runtime Settings
@@ -29,4 +31,4 @@ MYSQL_USERNAME=<user>
MYSQL_PASSWORD=<password>
```
Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `daily_mileage_km` metric row, so no separate mileage state store is required.
Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `daily_mileage_km` metric row.

View File

@@ -1,11 +1,11 @@
# Vehicle Ingest TDengine 链路验收手册
这份手册用于验收当前 32960/JT808 接入重构链路:
这份手册用于验收当前 GB32960/JT808/Yutong MQTT 接入重构链路:
1. 协议 App 接收 TCP 报文。
2. 协议 App 归档原始帧元数据,并把事件信封发布到 Kafka。
3. `vehicle-history-app` 消费 Kafka。
4. history 写入 raw、location 到 TDengine按需开启 telemetry field 逐字段宽表
4. history 写入 raw、location 到 TDengine不写 telemetry_fields
5. Swagger/API 通过 TDengine 做历史分页查询。
每一步都必须执行命令并检查输出后,才能标记为已验证。
@@ -24,102 +24,72 @@ export TDENGINE_USERNAME=root
export TDENGINE_PASSWORD='<tdengine-password>'
export TDENGINE_MIN_IDLE=0
export TDENGINE_MAX_POOL_SIZE=32
export EVENT_FILE_STORE_ENABLED=false
export TDENGINE_TELEMETRY_FIELDS_ENABLED=false
```
`TDENGINE_MIN_IDLE=0` 用于减少冷启动时 TDengine 连接重试日志。TDengine 地址稳定后,再按生产并发调大连接池。
高吞吐生产验收默认关闭 `EVENT_FILE_STORE_ENABLED``TDENGINE_TELEMETRY_FIELDS_ENABLED`验证 TDengine `raw_frames`、位置表和分页查询闭环。RAW 帧的 `parsedJson``metadataJson``rawUri` 直接进入 TDengineGB32960 snapshot/fields 接口优先基于 `raw_frames` 中的结构化 JSON 返回结果;接入服务写出的原始 `.bin` 只作为冷备复核材料。需要兼容旧 `/api/event-history/records` 低层记录接口或逐字段趋势宽表时,再显式开启对应开关,并单独压测写入放大
高吞吐生产验收验证 TDengine `raw_frames`、位置表和分页查询闭环。RAW 帧的 `parsedJson``metadataJson``rawUri` 直接进入 TDengineGB32960 snapshot/fields 接口基于 `raw_frames` 与当前解析器即时返回结构化结果;接入服务写出的原始 `.bin` 只作为冷备复核材料。逐字段趋势宽表由独立字段解析服务消费 Kafka RAW/事件后维护,不由 history-app 写入
## 构建
```bash
mvn -pl modules/apps/jt808-ingest-app,modules/apps/gb32960-ingest-app,modules/apps/vehicle-history-app -am package -DskipTests
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
```
预期产物:
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
- `modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar`
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
- `modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar`
- `modules/apps/vehicle-history-app/target/vehicle-history-app.jar`
- `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar`
## 启动 JT808 接入服务,监听 808 端口
## ECS 服务状态验证
生产服务只在 ECS 上运行,通过 Portainer/Docker 管理。不要在本机用 `java -jar`、plist 或其它守护进程方式启动 GB32960、JT808、Yutong MQTT、history、analytics。
```bash
HTTP_PORT=20482 \
JT808_PORT=808 \
KAFKA_CONSUMER_ENABLED=false \
java -jar modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar
```
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' \
| egrep 'gb32960|jt808|yutong|vehicle-history|vehicle-analytics|NAMES'
验证:
ss -lntp | egrep ':(808|32960|20100|20200|20310|20400|20500)\b'
```bash
curl -sS http://127.0.0.1:20482/actuator/health
lsof -nP -iTCP:808 -sTCP:LISTEN
```
预期:健康检查为 `UP`,并且 Java 进程监听 TCP `808`
## 启动 GB32960 接入服务
```bash
HTTP_PORT=20100 \
GB32960_PORT=32960 \
KAFKA_CONSUMER_ENABLED=false \
java -jar modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar
```
验证:
```bash
curl -sS http://127.0.0.1:20400/actuator/health
curl -sS http://127.0.0.1:20400/actuator/health/liveness
curl -sS http://127.0.0.1:20400/actuator/health/readiness
curl -sS http://127.0.0.1:20100/actuator/health
lsof -nP -iTCP:32960 -sTCP:LISTEN
```
## 启动 TDengine 版历史服务
```bash
HTTP_PORT=20200 \
KAFKA_CONSUMER_ENABLED=true \
EVENT_FILE_STORE_ENABLED=false \
EVENT_FILE_STORE_PATH=./target/tdengine-verification/event-store \
TDENGINE_TELEMETRY_FIELDS_ENABLED=false \
java -jar modules/apps/vehicle-history-app/target/vehicle-history-app.jar
```
验证:
```bash
curl -sS http://127.0.0.1:20100/actuator/health/liveness
curl -sS http://127.0.0.1:20100/actuator/health/readiness
curl -sS http://127.0.0.1:20500/actuator/health
curl -sS http://127.0.0.1:20500/actuator/health/liveness
curl -sS http://127.0.0.1:20500/actuator/health/readiness
curl -sS http://127.0.0.1:20200/actuator/health
curl -sS http://127.0.0.1:20200/actuator/health/liveness
curl -sS http://127.0.0.1:20200/actuator/health/readiness
curl -sS http://127.0.0.1:20200/v3/api-docs \
| grep -E '/api/event-history/jt808/locations|/api/event-history/gb32960/snapshots|/api/event-history/telemetry/fields'
| grep -E '/api/event-history/locations|/api/event-history/raw-frames'
curl -sS http://127.0.0.1:20310/actuator/health
curl -sS http://127.0.0.1:20310/actuator/health/liveness
curl -sS http://127.0.0.1:20310/actuator/health/readiness
```
预期:健康检查为 `UP`OpenAPI 中包含 JT808 TDengine 分页查询接口和 GB32960 snapshot 接口。`/api/event-history/telemetry/fields` 会存在,但只有在 `TDENGINE_TELEMETRY_FIELDS_ENABLED=true` 后才会持续写入新字段数据
预期:五个容器均为运行状态;健康检查为 `UP`JT808/GB32960 TCP 端口在 ECS 上监听OpenAPI 中包含通用位置分页查询和 RAW 帧查询接口。history-app 不暴露 `/api/event-history/telemetry/fields`也不持续写入逐字段宽表。JT808 位置帧中有 GPS 总里程时,`vehicle-analytics-app` 按差值法写入 MySQL `vehicle_stat_metric``daily_mileage_km` 指标
## 本机 launchctl 部署
## 运行位置约束
当前本机生产验证建议使用仓库模板生成 plist
生产身份绑定固定使用 MySQL不再提供 file/memory/sqlite 运行时切换入口。生产服务固定运行在 ECS/Portainer不提供本机守护进程模板。
```bash
deploy/local/launchctl/
```
关键约束:
- `jt808-ingest-app` 监听 TCP `808`HTTP `20482`
- `jt808-ingest-app` 监听 TCP `808`HTTP `20400`
- `gb32960-ingest-app` 监听 TCP `32960`HTTP `20100`
- `yutong-mqtt-app` 监听 HTTP `20500`
- `vehicle-history-app` 监听 HTTP `20200`
- history 热查询只依赖 Kafka 和 TDengine `raw_frames`;本地模板不会给 history 注入 `SINK_ARCHIVE_PATH`
- GB32960/JT808 接入服务可以继续使用 `SINK_ARCHIVE_PATH` 保存原始 `.bin` 冷备,但这不是 history API 的实时查询前置条件
- 替换 jar 前先 `launchctl bootout`,再复制 jar 和 `launchctl bootstrap`,避免 KeepAlive 在 jar 拷贝中途重启
- `vehicle-analytics-app` 监听 HTTP `20310`
- history 热查询只依赖 Kafka 和 TDengine `raw_frames`history 不需要共享 archive volume
- GB32960/JT808/Yutong MQTT 接入服务可以继续使用 `SINK_ARCHIVE_PATH` 保存原始冷备,但这不是 history API 的实时查询前置条件
JT808 注册身份绑定:
- 生产默认使用 `VEHICLE_IDENTITY_STORE=mysql`,不再使用 file/memory 作为运行时身份绑定仓储
- 生产需要把 0x0100 注册信息维护到 MySQL 时,设置 `VEHICLE_IDENTITY_STORE=mysql`
- 生产需要把 0x0100 注册信息维护到 MySQL 时,只需要配置 `VEHICLE_IDENTITY_MYSQL_*` 连接参数
- 需要提供 `VEHICLE_IDENTITY_MYSQL_JDBC_URL``VEHICLE_IDENTITY_MYSQL_USERNAME``VEHICLE_IDENTITY_MYSQL_PASSWORD`
- 服务启动时自动创建 `vehicle_identity_binding``vehicle_identity_binding_registration` 两张表。`vehicle_identity_binding` 只维护 `plate``vin` 两列;`vehicle_identity_binding_registration``protocol + phone` 保存 JT808 0x0100 注册字段。
- 注册帧无真实 VIN 时也会写入 registration 表,`vin` 默认为 `unknown`。外部系统识别车辆后只需要维护车牌和 VIN例如`INSERT INTO vehicle_identity_binding (plate, vin) VALUES ('沪A61559F', 'LNVIN000000000001') ON DUPLICATE KEY UPDATE vin = VALUES(vin);`
@@ -129,16 +99,30 @@ JT808 注册身份绑定:
健康检查:
```bash
curl -sS http://127.0.0.1:20482/actuator/health
curl -sS http://127.0.0.1:20400/actuator/health
curl -sS http://127.0.0.1:20400/actuator/health/liveness
curl -sS http://127.0.0.1:20400/actuator/health/readiness
curl -sS http://127.0.0.1:20100/actuator/health
curl -sS http://127.0.0.1:20100/actuator/health/liveness
curl -sS http://127.0.0.1:20100/actuator/health/readiness
curl -sS http://127.0.0.1:20500/actuator/health
curl -sS http://127.0.0.1:20500/actuator/health/liveness
curl -sS http://127.0.0.1:20500/actuator/health/readiness
curl -sS http://127.0.0.1:20200/actuator/health
curl -sS http://127.0.0.1:20200/actuator/health/liveness
curl -sS http://127.0.0.1:20200/actuator/health/readiness
curl -sS http://127.0.0.1:20310/actuator/health
curl -sS http://127.0.0.1:20310/actuator/health/liveness
curl -sS http://127.0.0.1:20310/actuator/health/readiness
```
Swagger
API / Swagger
- JT808 ingest: `http://127.0.0.1:20482/swagger-ui/index.html`
- GB32960 ingest: `http://127.0.0.1:20100/swagger-ui/index.html`
- JT808 ingest health: `http://127.0.0.1:20400/actuator/health`
- GB32960 ingest health: `http://127.0.0.1:20100/actuator/health`
- Yutong MQTT ingest health: `http://127.0.0.1:20500/actuator/health`
- History query: `http://127.0.0.1:20200/swagger-ui/index.html`
- Analytics metrics: `http://127.0.0.1:20310/swagger-ui/index.html`
## Kafka Topic 验证
@@ -148,6 +132,8 @@ Swagger
- `vehicle.raw.gb32960.v1`
- `vehicle.event.jt808.v1`
- `vehicle.raw.jt808.v1`
- `vehicle.event.mqtt-yutong.v1`
- `vehicle.raw.mqtt-yutong.v1`
使用 Kafka CLI 或管理工具检查 topic 是否存在,并确认 history consumer group 在测试流量后没有持续堆积。默认 `KAFKA_GROUP_HISTORY=vehicle-history` 时,实际 group 会拆成:
@@ -155,10 +141,12 @@ Swagger
- `vehicle-history-gb32960-raw`
- `vehicle-history-jt808-event`
- `vehicle-history-jt808-raw`
- `vehicle-history-yutong-mqtt-event`
- `vehicle-history-yutong-mqtt-raw`
## JT808 实时转发验收
如果外部平台已经把 JT808 报文转发到本机 TCP `808`,至少观察 60 秒日志和 Kafka 数据。
如果外部平台已经把 JT808 报文转发到 ECS TCP `808`,至少观察 60 秒日志和 Kafka 数据。
必须拿到以下证据:
@@ -186,22 +174,6 @@ curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin=
预期snapshot 返回 `sourceFrames.rawArchiveUri` 和解析后的 `blocks`;字段投影返回所选字段。新增协议字段后,先通过 replay 或专用解析任务补写 `raw_frames.parsedJson`,再基于历史 RAW 结构化结果查询。
## Telemetry Field 查询验收
查询 GB32960 字段历史。该能力需要 `TDENGINE_TELEMETRY_FIELDS_ENABLED=true` 并确认压测后无持续 lag
```bash
curl -sS 'http://127.0.0.1:20200/api/event-history/telemetry/fields?protocol=GB32960&vin=<vin>&fieldKey=<field-key>&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
```
查询 JT808 字段历史。没有 VIN 映射时,优先使用 `phone`。该能力同样需要 `TDENGINE_TELEMETRY_FIELDS_ENABLED=true`
```bash
curl -sS 'http://127.0.0.1:20200/api/event-history/telemetry/fields?protocol=JT808&phone=<phone>&fieldKey=location.speed_kmh&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
```
预期:响应包含 `items`,可能包含 `nextCursor`。下一页使用 `nextCursor` 里的 `cursorTs``cursorId` 查询。
## TDengine 直接检查
使用 TDengine CLI 或 JDBC 客户端检查超级表和子表是否创建:
@@ -212,14 +184,12 @@ SHOW STABLES;
SELECT COUNT(*) FROM raw_frames;
SELECT COUNT(*) FROM vehicle_locations;
SELECT COUNT(*) FROM jt808_locations;
-- 仅在 TDENGINE_TELEMETRY_FIELDS_ENABLED=true 时检查:
SELECT COUNT(*) FROM telemetry_fields;
```
预期:
- `raw_frames``vehicle_locations``jt808_locations` 存在;有对应协议实时流量后,计数持续增长。
- `telemetry_fields` 表结构会创建;默认高吞吐模式不会持续增长,除非显式开启 `TDENGINE_TELEMETRY_FIELDS_ENABLED=true`
- 字段趋势宽表不属于 history-app 验收范围,由独立字段解析服务负责
- 有实时流量后,计数持续增加。
## 失败语义
@@ -228,15 +198,17 @@ SELECT COUNT(*) FROM telemetry_fields;
- TDengine 未启用时TDengine 专属接口不应该出现在 OpenAPI 中。
- 路径不存在时API 层应该返回 HTTP `404`,而不是泛化的存储失败。
## 2026-06-29 本机实时链路验证结果
## 2026-06-29 Superseded Local Live Verification Record
运行服务:
该记录仅保留为解析和存储行为的历史证据;生产运行位置已经收敛到 ECS/Portainer。
- `com.lingniu.jt808.808`: `jt808-ingest-app`TCP `808`HTTP `20482`
- `com.lingniu.gb32960.32960`: `gb32960-ingest-app`TCP `32960`HTTP `20100`
- `com.lingniu.vehicle-history.jt808`: `vehicle-history-app`HTTP `20200`
当时参与验证的服务:
健康检查已确认三个服务均返回 `{"status":"UP"}`
- `jt808-ingest-app`TCP `808`HTTP `20400`
- `gb32960-ingest-app`TCP `32960`HTTP `20100`
- `vehicle-history-app`HTTP `20200`
当次健康检查均返回 `{"status":"UP"}`
可重复运行 live 验收工具:
@@ -256,16 +228,16 @@ python3 tools/vehicle_ingest_live_verify.py \
JT808 真实转发链路已验证:
- 本机 TCP `808` 有外部平台 `222.66.200.68` 持续发送 JT808 报文。
- ECS TCP `808` 有外部平台 `222.66.200.68` 持续发送 JT808 报文。
- `vehicle.raw.jt808.v1``vehicle.event.jt808.v1` 被 history 消费。
- `raw_frames``protocol='JT808'` 的真实行数已超过 `36000`
- 外部平台真实 0x0100 注册帧已超过 `347` 条,真实 0x0200 位置帧已超过 `12400` 条。
- `jt808_locations` 中位置行数已超过 `8783`
- 在旧的字段宽表开启模式下,`telemetry_fields``protocol='JT808'` 的字段行数曾超过 `15000`;当前高吞吐默认不要求该表持续增长
- 当前高吞吐默认链路不维护 `telemetry_fields`JT808 明细验证以 `raw_frames``vehicle_locations` 和 API 分页结果为准
- 分页 API 已用终端号 `13079963301` 验证第一页和 nextCursor 第二页,样例位置包含 `longitude=118.913846``latitude=31.927309``statusFlag=3``archive://2026/06/29/JT808/...` 原始帧引用。
- 注册帧样例:终端号 `13079963320`、deviceId `9963320`、deviceType `SEG-9888G`、plate `沪A61559F`、maker `70112`、province `31`、city `113`、plateColor `2`。生产 VIN 反写依赖 MySQL identity store后续 raw/event 的 VIN 解析也从 MySQL 绑定读取。
## JT808 本机 smoke / 轻压测工具
## JT808 ECS smoke / 轻压测工具
仓库提供 `tools/jt808_e2e_smoke.py`,用于重复验证 TCP `808` 到 TDengine history 的闭环:
@@ -295,8 +267,6 @@ python3 tools/jt808_e2e_smoke.py \
USE vehicle_ts;
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'JT808';
SELECT COUNT(*) FROM jt808_locations WHERE protocol = 'JT808';
-- 仅在 TDENGINE_TELEMETRY_FIELDS_ENABLED=true 时复查:
SELECT COUNT(*) FROM telemetry_fields WHERE protocol = 'JT808';
SELECT event_time, vehicle_key, phone, message_id, raw_uri
FROM raw_frames
WHERE protocol = 'JT808'
@@ -325,7 +295,7 @@ GB32960 链路已用正式平台登录和合成实时帧验证:
- 当前正式平台尚未发送车辆实时上报 `0x02`;非测试 VIN 的 `message_id=2` 计数为 `0`,因此不能把平台登录误判为车辆数据在线。
- TCP `32960` 返回 GB32960 `2323` ACK。
- `raw_frames` 中测试 VIN `LTEST202606290001` 已有 `2` 条 raw 记录。
- 在旧的字段宽表开启模式下,`telemetry_fields` 中测试 VIN `LTEST202606290001` 曾有 `34` 条字段记录;当前默认通过 GB32960 snapshot/fields 即时解析 raw,不要求该表增长。
- 当前默认通过 GB32960 snapshot/fields 即时解析 `raw_frames` 中的 RAW JSON不维护字段宽表增长。
- latest raw 包含 `archive://2026/06/29/GB32960/LTEST202606290001/...`
- snapshots API 可查到 `speedKmh=62.4``totalMileageKm=223456.7``longitude=116.397128``latitude=39.916527`
@@ -335,9 +305,6 @@ GB32960 链路已用正式平台登录和合成实时帧验证:
USE vehicle_ts;
SELECT COUNT(*) FROM raw_frames
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001';
-- 仅在 TDENGINE_TELEMETRY_FIELDS_ENABLED=true 时复查:
SELECT COUNT(*) FROM telemetry_fields
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001';
SELECT event_time, vin, raw_uri, frame_id
FROM raw_frames
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001'
@@ -377,7 +344,7 @@ curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin=
## 当前注意事项
- 当前本机使用 TDengine WebSocket JDBC例如 `jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts`
- 当前生产链路使用 TDengine WebSocket JDBC例如 `jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts`
- `KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest` 适合生产接入新流量;如果要回放历史 Kafka 数据,需要切换 consumer group 或重置 offset。
- `vehicle-history-app` raw 和 event 使用不同 consumer bindingraw consumer 必须开启,否则 `raw_frames` 不会持续增长。
- JT808 设备没有 VIN 映射时,`vehicle_key` 使用 `jt808:<phone>`;启用 MySQL identity store 后registration 表中已反写 VIN 的 phone/deviceId/plate 会用于后续 raw/event 的 VIN 解析。

View File

@@ -543,7 +543,7 @@ EOF
Run: `mvn test -q`
Expected: BUILD SUCCESS。重点关注
- `protocol-gb32960` 模块全绿Golden、FullBlocks、Isolation、GuangdongFcEndToEnd、Decoder、Mapper、所有 parser 单测、profile
- 其它模块(`ingest-core``sink-mq` 等)不受影响(无改动应自动通过)
- 其它模块(`ingest-core``sink-kafka` 等)不受影响(无改动应自动通过)
若有红,停下来诊断。

View File

@@ -1,3 +1,10 @@
> **Superseded:** This 2026-06-23 DuckDB hot-store plan is historical context.
> Use `docs/target-architecture.md` and
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
> current production architecture: TDengine stores hot history, `sink-archive`
> owns raw bytes, and `event-file-store` / DuckDB are no longer part of the
> current build surface.
# GB32960 History DuckDB Hot Store Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.

View File

@@ -1,3 +1,10 @@
> **Superseded:** This 2026-06-23 service-split plan is historical context.
> Use `docs/target-architecture.md` and
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
> current production architecture: GB32960, JT808, and Yutong MQTT are the
> active ingest apps, history is backed by TDengine, and Xinda/event-file-store
> paths have been removed from the current build surface.
# GB32960 Service Split Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
@@ -13,12 +20,14 @@
## Current-State Notes
- Existing all-in-one entrypoint: `modules/apps/bootstrap-all/src/main/java/com/lingniu/ingest/bootstrap/IngestApplication.java`.
- Existing all-in-one module: `modules/apps/bootstrap-all/pom.xml` imports protocols, sinks, history, state, stat, command gateway, MQTT, Xinda, and JT modules.
- Existing all-in-one module at the time of this plan imported protocols,
sinks, history, state, stat, command gateway, MQTT, and JT modules. Xinda was
later removed from source and build configuration.
- Existing GB32960 handler: `modules/protocols/protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/inbound/Gb32960ChannelHandler.java`.
- Existing handler currently writes report ACK before `dispatcher.dispatch(rf)`.
- Existing `DisruptorEventBus` publishes to sinks asynchronously and does not wait for sink completion.
- Existing `KafkaEventSink.accepts()` rejects `VehicleEvent.RawArchive`.
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.mq.topics`.
- Existing versioned topic names are not yet present; current topic names live under `lingniu.ingest.sink.kafka.topics`.
- Existing history consumer logic: `modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java`.
- Existing analytics consumers: `VehicleStateEnvelopeIngestor` and `VehicleStatEnvelopeIngestor`.
- There is an unrelated working-tree change in `modules/apps/bootstrap-all/src/main/resources/application.yml` for GB32960 diagnostics capacity. Do not revert it. Do not include it in service-split commits unless the user explicitly asks.
@@ -48,11 +57,11 @@ Create app modules:
Modify shared build and Kafka contract files:
- `pom.xml`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
- `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java`
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java`
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java`
- `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java`
- `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java`
- `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java`
Modify ACK/Kafka durability boundary files:
@@ -157,7 +166,7 @@ Create `modules/apps/gb32960-ingest-app/pom.xml`:
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>sink-mq</artifactId>
<artifactId>sink-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -222,7 +231,7 @@ Create `modules/apps/vehicle-history-app/pom.xml`:
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>sink-mq</artifactId>
<artifactId>sink-kafka</artifactId>
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
@@ -303,7 +312,7 @@ Create `modules/apps/vehicle-analytics-app/pom.xml`:
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>sink-mq</artifactId>
<artifactId>sink-kafka</artifactId>
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
@@ -466,16 +475,19 @@ lingniu:
rate-limit:
per-vin-qps: 50
session:
store: ${SESSION_STORE:memory}
store: ${SESSION_STORE:redis}
ttl: ${SESSION_TTL:30m}
identity:
store: ${VEHICLE_IDENTITY_STORE:file}
file:
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
store: ${VEHICLE_IDENTITY_STORE:mysql}
mysql:
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd
linger-ms: 20
@@ -568,9 +580,8 @@ lingniu:
gb32960:
enabled: false
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
@@ -666,9 +677,8 @@ lingniu:
gb32960:
enabled: false
sink:
mq:
kafka:
enabled: ${KAFKA_ENABLED:true}
type: kafka
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
@@ -754,7 +764,7 @@ import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
@@ -770,7 +780,7 @@ class Gb32960IngestAppCompositionTest {
.withPropertyValues(
"lingniu.ingest.gb32960.enabled=true",
"lingniu.ingest.gb32960.port=0",
"lingniu.ingest.sink.mq.enabled=false",
"lingniu.ingest.sink.kafka.enabled=false",
"lingniu.ingest.sink.archive.enabled=false",
"lingniu.ingest.event-file-store.enabled=false",
"lingniu.ingest.event-history.enabled=false",
@@ -835,7 +845,7 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=false",
"lingniu.ingest.sink.mq.enabled=false")
"lingniu.ingest.sink.kafka.enabled=false")
.withConfiguration(AutoConfigurations.of(
com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration.class,
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration.class,
@@ -896,7 +906,7 @@ class VehicleAnalyticsAppCompositionTest {
"lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=true",
"lingniu.ingest.vehicle-stat.file-path=" + tempDir.resolve("vehicle-stat"),
"lingniu.ingest.sink.mq.enabled=false")
"lingniu.ingest.sink.kafka.enabled=false")
.withConfiguration(AutoConfigurations.of(
com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration.class,
com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration.class))
@@ -932,22 +942,22 @@ git commit -m "test: cover split service composition"
### Task 4: Version Kafka Topics for GB32960 Raw/Event/DLQ
**Files:**
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java`
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java`
- Modify: `modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java`
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java`
- Create or modify: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java`
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java`
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java`
- Modify: `modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java`
- Create or modify: `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java`
- Create or modify: `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java`
- [ ] **Step 1: Write topic routing tests**
Create `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java` if it does not exist. Add tests proving GB32960 raw and normalized events can route to versioned topics. Use the existing `VehicleEvent` constructors from `EventFileStoreSinkTest` and `EnvelopeMapperTelemetrySnapshotTest` as examples.
Create `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java` if it does not exist. Add tests proving GB32960 raw and normalized events can route to versioned topics. Use the existing `VehicleEvent` constructors from `EventFileStoreSinkTest` and `EnvelopeMapperTelemetrySnapshotTest` as examples.
Expected test names:
```java
@Test
void rawArchiveRoutesToVersionedGb32960RawTopic() {
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
topics.setRawArchive("vehicle.raw.gb32960.v1");
TopicRouter router = new TopicRouter(topics);
@@ -970,7 +980,7 @@ Also add a normalized realtime event test:
```java
@Test
void realtimeRoutesToVersionedGb32960EventTopic() {
SinkMqProperties.Topics topics = new SinkMqProperties.Topics();
KafkaSinkProperties.Topics topics = new KafkaSinkProperties.Topics();
topics.setRealtime("vehicle.event.gb32960.v1");
TopicRouter router = new TopicRouter(topics);
@@ -995,14 +1005,14 @@ Adjust constructor arguments to match the current sealed `VehicleEvent` definiti
Run:
```bash
mvn -pl :sink-mq -Dtest=TopicRouterTest test
mvn -pl :sink-kafka -Dtest=TopicRouterTest test
```
Expected before implementation: compile failure if constructors are wrong or assertion failure if defaults are old. Fix constructors first; keep routing assertions.
- [ ] **Step 3: Update `SinkMqProperties.Topics` defaults**
- [ ] **Step 3: Update `KafkaSinkProperties.Topics` defaults**
In `SinkMqProperties.Topics`, change defaults:
In `KafkaSinkProperties.Topics`, change defaults:
```java
private String realtime = "vehicle.event.gb32960.v1";
@@ -1027,12 +1037,12 @@ public boolean accepts(VehicleEvent event) {
Update the JavaDoc above it to say Kafka carries both normalized events and raw archive records in split-service mode.
- [ ] **Step 5: Run sink-mq tests**
- [ ] **Step 5: Run sink-kafka tests**
Run:
```bash
mvn -pl :sink-mq test
mvn -pl :sink-kafka test
```
Expected: PASS.
@@ -1040,11 +1050,11 @@ Expected: PASS.
- [ ] **Step 6: Commit**
```bash
git add modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java \
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/TopicRouter.java \
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java \
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/TopicRouterTest.java \
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEventSinkTest.java
git add modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaSinkProperties.java \
modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/TopicRouter.java \
modules/sinks/sink-kafka/src/main/java/com/lingniu/ingest/sink/kafka/KafkaEventSink.java \
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/TopicRouterTest.java \
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/KafkaEventSinkTest.java
git commit -m "feat: route gb32960 kafka topics"
```
@@ -1353,8 +1363,8 @@ void gb32960IngestDefaultsOnlyEnableProtocolAndKafkaProducer() {
assertThat(props.getProperty("spring.application.name")).isEqualTo("gb32960-ingest-app");
assertThat(props.getProperty("lingniu.ingest.gb32960.enabled")).isEqualTo("true");
assertThat(props.getProperty("lingniu.ingest.gb32960.port")).isEqualTo("${GB32960_PORT:32960}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.sink.kafka.enabled")).isEqualTo("${KAFKA_ENABLED:true}");
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
@@ -1378,7 +1388,7 @@ void historyDefaultsEnableStorageAndHistoryConsumerOnly() {
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("true");
assertThat(props.getProperty("lingniu.ingest.vehicle-state.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryEnvelopeConsumerProcessor.group-id"))
.isEqualTo("${KAFKA_GROUP_HISTORY:vehicle-history}");
}
```
@@ -1398,7 +1408,7 @@ void analyticsDefaultsEnableStatConsumerAndDisableProtocolAndHistoryStorage() {
assertThat(props.getProperty("lingniu.ingest.event-file-store.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.event-history.enabled")).isEqualTo("false");
assertThat(props.getProperty("lingniu.ingest.vehicle-stat.enabled")).isEqualTo("${VEHICLE_STAT_ENABLED:true}");
assertThat(props.getProperty("lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
assertThat(props.getProperty("lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id"))
.isEqualTo("${KAFKA_GROUP_STAT:vehicle-stat}");
}
```
@@ -1566,7 +1576,7 @@ git commit -m "docs: add gb32960 split service runbook"
Run:
```bash
mvn -pl :sink-mq,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test
mvn -pl :sink-kafka,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test
```
Expected: BUILD SUCCESS.

View File

@@ -1,3 +1,10 @@
> **Superseded:** This 2026-06-29 phase-1 implementation plan is historical
> context. Use `docs/target-architecture.md` and
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
> current production architecture: raw bytes are owned by `sink-archive`,
> history reads come from TDengine, and the former `raw-archive-store`
> prototype has been removed from the current build surface.
# Vehicle Ingest Redesign Phase 1 Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
@@ -16,7 +23,7 @@ This plan implements only the first slice of the redesign:
- `modules/core/ingest-facts`: protocol-neutral fact records and helpers.
- `modules/sinks/raw-archive-store`: raw archive write/read contract and local URI-safe implementation skeleton.
- `modules/sinks/sink-mq`: protobuf messages for raw frame facts and decoded facts.
- `modules/sinks/sink-kafka`: protobuf messages for raw frame facts and decoded facts.
- Parent Maven wiring and focused tests.
It intentionally does not yet modify:
@@ -53,9 +60,9 @@ Create:
Modify:
- `pom.xml`: add modules and dependency management entries.
- `modules/sinks/sink-mq/pom.xml`: add dependency on `ingest-facts`.
- `modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto`: add `RawFrameFactPayload`, `DecodedFactPayload`, and related enums/messages.
- `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/VehicleEnvelopeProtoCompatibilityTest.java`: verify new proto messages can round-trip.
- `modules/sinks/sink-kafka/pom.xml`: add dependency on `ingest-facts`.
- `modules/sinks/sink-kafka/src/main/proto/vehicle_envelope.proto`: add `RawFrameFactPayload`, `DecodedFactPayload`, and related enums/messages.
- `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/VehicleEnvelopeProtoCompatibilityTest.java`: verify new proto messages can round-trip.
---
@@ -1260,21 +1267,21 @@ git commit -m "feat: add raw archive store contract"
**Files:**
- Modify: `modules/sinks/sink-mq/pom.xml`
- Modify: `modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto`
- Create: `modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/VehicleEnvelopeProtoCompatibilityTest.java`
- Modify: `modules/sinks/sink-kafka/pom.xml`
- Modify: `modules/sinks/sink-kafka/src/main/proto/vehicle_envelope.proto`
- Create: `modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/VehicleEnvelopeProtoCompatibilityTest.java`
- [ ] **Step 1: Write failing proto compatibility test**
Create `VehicleEnvelopeProtoCompatibilityTest.java`:
```java
package com.lingniu.ingest.sink.mq;
package com.lingniu.ingest.sink.kafka;
import com.lingniu.ingest.sink.mq.proto.DecodedFactPayload;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.lingniu.ingest.sink.kafka.proto.DecodedFactPayload;
import com.lingniu.ingest.sink.kafka.proto.ParseStatusProto;
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@@ -1341,14 +1348,14 @@ class VehicleEnvelopeProtoCompatibilityTest {
Run:
```bash
mvn -pl modules/sinks/sink-mq -Dtest=VehicleEnvelopeProtoCompatibilityTest test
mvn -pl modules/sinks/sink-kafka -Dtest=VehicleEnvelopeProtoCompatibilityTest test
```
Expected: FAIL because proto generated classes do not exist.
- [ ] **Step 3: Add `ingest-facts` dependency to sink-mq**
- [ ] **Step 3: Add `ingest-facts` dependency to sink-kafka**
Add this dependency to `modules/sinks/sink-mq/pom.xml` after `ingest-api`:
Add this dependency to `modules/sinks/sink-kafka/pom.xml` after `ingest-api`:
```xml
<dependency>
@@ -1412,17 +1419,17 @@ message DecodedFactPayload {
Run:
```bash
mvn -pl modules/sinks/sink-mq -Dtest=VehicleEnvelopeProtoCompatibilityTest test
mvn -pl modules/sinks/sink-kafka -Dtest=VehicleEnvelopeProtoCompatibilityTest test
```
Expected: PASS.
- [ ] **Step 6: Run existing sink-mq tests**
- [ ] **Step 6: Run existing sink-kafka tests**
Run:
```bash
mvn -pl modules/sinks/sink-mq test
mvn -pl modules/sinks/sink-kafka test
```
Expected: PASS. Existing envelope tests must continue to pass because new proto fields are additive.
@@ -1430,9 +1437,9 @@ Expected: PASS. Existing envelope tests must continue to pass because new proto
- [ ] **Step 7: Commit**
```bash
git add modules/sinks/sink-mq/pom.xml \
modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto \
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/VehicleEnvelopeProtoCompatibilityTest.java
git add modules/sinks/sink-kafka/pom.xml \
modules/sinks/sink-kafka/src/main/proto/vehicle_envelope.proto \
modules/sinks/sink-kafka/src/test/java/com/lingniu/ingest/sink/kafka/VehicleEnvelopeProtoCompatibilityTest.java
git commit -m "feat: add fact payloads to kafka envelope"
```
@@ -1449,7 +1456,7 @@ git commit -m "feat: add fact payloads to kafka envelope"
Run:
```bash
mvn -pl modules/core/ingest-facts,modules/sinks/raw-archive-store,modules/sinks/sink-mq -am test
mvn -pl modules/core/ingest-facts,modules/sinks/raw-archive-store,modules/sinks/sink-kafka -am test
```
Expected: PASS.
@@ -1459,7 +1466,7 @@ Expected: PASS.
Run:
```bash
mvn -pl modules/sinks/sink-mq -am -DskipTests package
mvn -pl modules/sinks/sink-kafka -am -DskipTests package
```
Expected: PASS. Generated protobuf Java sources include `RawFrameFactPayload`, `DecodedFactPayload`, and `ParseStatusProto`.
@@ -1473,7 +1480,7 @@ rg -n "RawFrameFact|DecodedFact|RawArchiveWriter|vehicle.raw-frame.v1|vehicle.de
modules/apps modules/protocols modules/services modules/core modules/sinks
```
Expected: Matches only in the new modules, sink-mq proto/test, and planned references. No existing handler behavior should be changed in Phase 1.
Expected: Matches only in the new modules, sink-kafka proto/test, and planned references. No existing handler behavior should be changed in Phase 1.
- [ ] **Step 4: Commit final verification note if any docs changed**

View File

@@ -2,14 +2,15 @@
## Current Goal
Consume JT808 Kafka location events, keep rolling daily state, and write the derived daily mileage into the common vehicle-stat metric repository.
Consume JT808 Kafka location events and write the derived daily mileage into the common vehicle-stat metric repository.
## Current Design
- Only supported message backbone: Kafka.
- Source topic: `vehicle.event.jt808.v1`
- Runtime app: `vehicle-analytics-app`
- Runtime state: Redis `Jt808MileageStateStore`
- Metric output: `VehicleStatRepository.saveDailyStat(...)`
- Runtime state: none outside `vehicle_stat_metric`
- Metric output: `VehicleStatRepository.recordDailyMileageSample(...)`
- Production metric storage: JDBC/MySQL `vehicle_stat_metric`
- Date boundary: `Asia/Shanghai`
- Calculation method: `JT808_TOTAL_MILEAGE_DIFF`
@@ -17,10 +18,17 @@ Consume JT808 Kafka location events, keep rolling daily state, and write the der
JT808 daily mileage is calculated only from the GPS total mileage reported by JT808 location additional information:
```text
daily_mileage_km = last_total_mileage_km - first_total_mileage_km
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
```
The stream saves a metric only when at least two ordered location points for the same vehicle and local day contain valid `total_mileage_km`. Negative differences are treated as invalid and are not written.
The first valid local-day sample writes `daily_mileage_km=0.0`. Later ordered or replayed samples update the same metric row with:
```text
metric_value = max_total_mileage_km - min_total_mileage_km
metric_key = daily_mileage_km
```
The local-day minimum and maximum GPS total mileage values stay on that same metric row as calculation source columns so restarts recover from MySQL without Redis or another state table.
## Runtime Properties
@@ -28,16 +36,11 @@ The stream saves a metric only when at least two ordered location points for the
KAFKA_TOPIC_JT808_EVENT=vehicle.event.jt808.v1
VEHICLE_STAT_ENABLED=true
VEHICLE_STAT_JT808_MILEAGE_ENABLED=true
VEHICLE_STAT_REPOSITORY_TYPE=jdbc
MYSQL_JDBC_URL=<jdbc-url>
MYSQL_USERNAME=<user>
MYSQL_PASSWORD=<password>
REDIS_HOST=<host>
REDIS_PORT=6379
REDIS_DATABASE=50
REDIS_PASSWORD=<password>
```
## Notes
Do not create or write a protocol-specific JT808 daily-mileage table. Do not add GPS segment distance, speed integral, or jump-filter state back into this path unless the mileage definition changes again.
Do not create or write a protocol-specific JT808 daily-mileage table. Do not add another message backbone, distance accumulation, integral calculation, Redis state, or memory state back into this path unless the mileage definition changes again.

View File

@@ -0,0 +1,720 @@
# Go Vehicle Ingest Redesign Phase 1 Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Build the first production-capable Go runtime for GB32960, JT808, and Yutong MQTT ingestion with unified Kafka, TDengine, MySQL statistics, and Redis realtime state boundaries.
**Architecture:** Create a new single Go module at `go/vehicle-gateway` and migrate only the useful pieces from the existing `go/ingest-edge` and `go/vehicle-state` prototypes. The gateway produces unified envelopes to Kafka; independent consumers write TDengine history, MySQL daily metrics, and Redis realtime state.
**Tech Stack:** Go 1.26+, Kafka (`segmentio/kafka-go`), Redis (`redis/go-redis`), MySQL (`go-sql-driver/mysql`), TDengine official Go connector, MQTT (`eclipse/paho.mqtt.golang`), standard library TCP.
---
## File Structure
- Create: `go/vehicle-gateway/go.mod`
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
- Create: `go/vehicle-gateway/internal/protocol/jt808/*`
- Create: `go/vehicle-gateway/internal/protocol/gb32960/*`
- Create: `go/vehicle-gateway/internal/protocol/yutongmqtt/*`
- Create: `go/vehicle-gateway/internal/gateway/*`
- Create: `go/vehicle-gateway/internal/identity/*`
- Create: `go/vehicle-gateway/internal/eventbus/*`
- Create: `go/vehicle-gateway/internal/history/*`
- Create: `go/vehicle-gateway/internal/stats/*`
- Create: `go/vehicle-gateway/internal/realtime/*`
- Create: `go/vehicle-gateway/internal/observability/*`
- Modify: `README.md`
- Modify: `docs/target-architecture.md`
- Create: `deploy/portainer/docker-compose-go.yml`
The existing `go/ingest-edge` and `go/vehicle-state` directories are migration sources only. After phase 1 is verified, remove or mark them superseded.
---
### Task 1: Create Single Go Module
**Files:**
- Create: `go/vehicle-gateway/go.mod`
- Create: `go/vehicle-gateway/internal/observability/logger.go`
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
- [ ] **Step 1: Create module manifest**
Add `go/vehicle-gateway/go.mod`:
```go
module lingniu-vehicle-ingest/go/vehicle-gateway
go 1.26
require (
github.com/eclipse/paho.mqtt.golang v1.5.1
github.com/go-sql-driver/mysql v1.9.3
github.com/redis/go-redis/v9 v9.17.2
github.com/segmentio/kafka-go v0.4.49
github.com/taosdata/driver-go/v3 v3.8.1
)
```
- [ ] **Step 2: Add logger helper**
Add `go/vehicle-gateway/internal/observability/logger.go`:
```go
package observability
import (
"log/slog"
"os"
)
func NewLogger(service string) *slog.Logger {
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
return slog.New(handler).With("service", service)
}
```
- [ ] **Step 3: Add temporary gateway entrypoint**
Add `go/vehicle-gateway/cmd/gateway/main.go`:
```go
package main
import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
func main() {
logger := observability.NewLogger("vehicle-gateway")
logger.Info("vehicle gateway scaffold started")
}
```
- [ ] **Step 4: Verify scaffold builds**
Run:
```bash
cd go/vehicle-gateway
go mod tidy
go test ./...
go build ./cmd/gateway
```
Expected: all commands exit `0`.
---
### Task 2: Define Unified Envelope
**Files:**
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
- [ ] **Step 1: Write envelope tests**
Add `go/vehicle-gateway/internal/envelope/envelope_test.go`:
```go
package envelope
import "testing"
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
t.Fatalf("VehicleKey() = %q", got)
}
}
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
if got := e.VehicleKey(); got != "JT808:013307795425" {
t.Fatalf("VehicleKey() = %q", got)
}
}
func TestFrameEnvelopeEventIDStable(t *testing.T) {
e := FrameEnvelope{
Protocol: ProtocolJT808,
MessageID: "0x0200",
Phone: "013307795425",
Sequence: 1,
EventTimeMS: 1782745114000,
ReceivedAtMS: 1782745114999,
RawHex: "7e02000000ff7e",
}
a := e.StableEventID()
b := e.StableEventID()
if a == "" || a != b {
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
}
}
```
- [ ] **Step 2: Run tests and confirm failure**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/envelope
```
Expected: fail because `FrameEnvelope` is not defined.
- [ ] **Step 3: Implement envelope**
Add `go/vehicle-gateway/internal/envelope/envelope.go`:
```go
package envelope
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
)
type Protocol string
const (
ProtocolGB32960 Protocol = "GB32960"
ProtocolJT808 Protocol = "JT808"
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
)
type ParseStatus string
const (
ParseOK ParseStatus = "OK"
ParsePartial ParseStatus = "PARTIAL"
ParseBadFrame ParseStatus = "BAD_FRAME"
)
type FrameEnvelope struct {
EventID string `json:"event_id"`
TraceID string `json:"trace_id"`
Protocol Protocol `json:"protocol"`
MessageID string `json:"message_id"`
Sequence uint16 `json:"sequence"`
VIN string `json:"vin,omitempty"`
VehicleKeyHint string `json:"vehicle_key,omitempty"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
Plate string `json:"plate,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
Parsed map[string]any `json:"parsed,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
ParseStatus ParseStatus `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
}
func (e FrameEnvelope) VehicleKey() string {
if key := strings.TrimSpace(e.VIN); key != "" {
return key
}
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
return key
}
if key := strings.TrimSpace(e.Phone); key != "" {
return string(e.Protocol) + ":" + key
}
if key := strings.TrimSpace(e.DeviceID); key != "" {
return string(e.Protocol) + ":" + key
}
return string(e.Protocol) + ":unknown"
}
func (e FrameEnvelope) StableEventID() string {
if strings.TrimSpace(e.EventID) != "" {
return e.EventID
}
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
sum := sha256.Sum256([]byte(input))
return hex.EncodeToString(sum[:16])
}
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
if e.EventID == "" {
e.EventID = e.StableEventID()
}
if e.ParseStatus == "" {
e.ParseStatus = ParseOK
}
return json.Marshal(e)
}
```
- [ ] **Step 4: Verify tests pass**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/envelope
```
Expected: pass.
---
### Task 3: Implement JT808 Frame and 0200 Parser
**Files:**
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser.go`
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser_test.go`
- [ ] **Step 1: Add sample frame tests**
Use the production sample provided in the thread:
```text
7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E
```
Expected assertions:
- message id is `0x0200`
- phone keeps normalized BCD string `013307795425`
- sequence is `1`
- `fields.total_mileage_km` is parsed from additional item `0x01`
- latitude, longitude, speed, direction, alarm, status, and device time are present
- [ ] **Step 2: Implement parser**
Implementation rules:
- strip `0x7e` start/end delimiters
- unescape `0x7d 0x02 -> 0x7e`
- unescape `0x7d 0x01 -> 0x7d`
- verify XOR checksum
- parse 2011/2013 common header
- parse BCD phone from 6 bytes and keep both raw and normalized values in `parsed.header`
- parse 0200 fixed body
- parse additional item list as `id,length,value_hex`
- parse additional `0x01` as `total_mileage_km = uint32 / 10`
- [ ] **Step 3: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/protocol/jt808
```
Expected: pass.
---
### Task 4: Implement GB32960 Frame and Data Unit Parser
**Files:**
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser.go`
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser_test.go`
- [ ] **Step 1: Add parser tests**
Tests must cover:
- `##` frame boundary
- command id
- response flag
- 17-byte VIN
- encryption flag
- payload length
- BCC verification
- realtime data command `0x02`
- reissue data command `0x03`
- data unit `0x01` vehicle status fields
- data unit `0x05` position fields
- [ ] **Step 2: Implement parser**
Implementation rules:
- parse header without allocating large temporary buffers
- keep RAW as hex in envelope
- write all recognized data units to `Parsed`
- write core fields to `Fields`
- unsupported data unit stays in `Parsed["unknown_units"]`
- [ ] **Step 3: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/protocol/gb32960
```
Expected: pass.
---
### Task 5: Implement Kafka Sink
**Files:**
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink.go`
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink_test.go`
- [ ] **Step 1: Add sink interface**
Create a sink interface:
```go
package eventbus
import (
"context"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Sink interface {
PublishRaw(context.Context, envelope.FrameEnvelope) error
PublishUnified(context.Context, envelope.FrameEnvelope) error
Close() error
}
```
- [ ] **Step 2: Implement Kafka topic routing**
Rules:
- `GB32960 -> vehicle.raw.gb32960.v1`
- `JT808 -> vehicle.raw.jt808.v1`
- `YUTONG_MQTT -> vehicle.raw.yutong-mqtt.v1`
- unified topic is `vehicle.event.unified.v1`
- message key is `env.VehicleKey()`
- [ ] **Step 3: Verify routing with unit tests**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/eventbus
```
Expected: pass.
---
### Task 6: Implement TDengine History Writer
**Files:**
- Create: `go/vehicle-gateway/internal/history/schema.go`
- Create: `go/vehicle-gateway/internal/history/writer.go`
- Create: `go/vehicle-gateway/internal/history/writer_test.go`
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
- [ ] **Step 1: Add schema bootstrap SQL**
Implement schema strings for:
- database `lingniu_vehicle_ts`
- stable `raw_frames`
- stable `vehicle_locations`
- stable `vehicle_mileage_points`
- [ ] **Step 2: Implement writer**
Rules:
- `AppendRawFrame` always writes one raw row.
- `AppendLocation` writes only when longitude and latitude exist.
- `AppendMileagePoint` writes only when `total_mileage_km` exists.
- child table name is deterministic hash of `protocol + vehicle_key`.
- escape tag values.
- [ ] **Step 3: Use TDengine official driver**
Import WebSocket driver in command:
```go
import _ "github.com/taosdata/driver-go/v3/taosWS"
```
Default driver name:
```text
TDENGINE_DRIVER=taosWS
```
Short-term compatibility:
```text
TDENGINE_DRIVER=taosSql
```
Only use `taosSql` when ECS TDengine WebSocket is not available.
- [ ] **Step 4: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/history
go build ./cmd/history-writer
```
Expected: pass.
---
### Task 7: Implement MySQL Daily Metric Writer
**Files:**
- Create: `go/vehicle-gateway/internal/stats/schema.go`
- Create: `go/vehicle-gateway/internal/stats/daily_metric.go`
- Create: `go/vehicle-gateway/internal/stats/daily_metric_test.go`
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
- [ ] **Step 1: Add schema bootstrap**
Implement `vehicle_daily_metric` schema from the design spec.
- [ ] **Step 2: Add metric derivation tests**
Test cases:
- no `total_mileage_km` produces no metric
- one sample produces:
- `daily_mileage_km = 0`
- `daily_total_mileage_km = sample`
- later larger sample updates:
- `latest_total_mileage_km`
- `daily_mileage_km`
- `daily_total_mileage_km`
- out-of-order smaller sample updates:
- `first_total_mileage_km`
- `daily_mileage_km`
- [ ] **Step 3: Implement MySQL upsert**
Use one table and one idempotent upsert:
```sql
INSERT INTO vehicle_daily_metric
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
ON DUPLICATE KEY UPDATE
first_total_mileage_km = LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)),
latest_total_mileage_km = GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)),
metric_value = CASE
WHEN metric_key = 'daily_mileage_km'
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
- LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
WHEN metric_key = 'daily_total_mileage_km'
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
ELSE VALUES(metric_value)
END,
sample_count = sample_count + 1,
updated_at = CURRENT_TIMESTAMP
```
- [ ] **Step 4: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/stats
go build ./cmd/stat-writer
```
Expected: pass.
---
### Task 8: Implement Redis Realtime API
**Files:**
- Create: `go/vehicle-gateway/internal/realtime/repository.go`
- Create: `go/vehicle-gateway/internal/realtime/repository_test.go`
- Create: `go/vehicle-gateway/internal/realtime/http.go`
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
- [ ] **Step 1: Add repository contract**
Methods:
- `Update(ctx, envelope.FrameEnvelope) error`
- `GetMerged(ctx, vin string) (Snapshot, error)`
- `GetProtocol(ctx, vin string, protocol envelope.Protocol) (Snapshot, error)`
- `IsOnline(ctx, vin string) (OnlineStatus, error)`
- [ ] **Step 2: Implement merge logic**
Rules:
- update `vehicle:latest:{vin}:{protocol}`
- update `vehicle:latest:{vin}` with newest fields
- update `vehicle:online:{vin}`
- update sorted set `vehicle:last_seen`
- [ ] **Step 3: Implement HTTP API**
Routes:
- `GET /api/realtime/vehicles/{vin}`
- `GET /api/realtime/vehicles/{vin}/online`
- `GET /api/realtime/vehicles/{vin}/protocols/{protocol}`
- [ ] **Step 4: Verify**
Run:
```bash
cd go/vehicle-gateway
go test ./internal/realtime
go build ./cmd/realtime-api
```
Expected: pass.
---
### Task 9: Wire Gateway Runtime
**Files:**
- Create: `go/vehicle-gateway/internal/gateway/tcp_server.go`
- Create: `go/vehicle-gateway/internal/gateway/mqtt_client.go`
- Modify: `go/vehicle-gateway/cmd/gateway/main.go`
- [ ] **Step 1: Implement TCP server**
Rules:
- one goroutine per accepted connection
- bounded max connections
- read timeout and idle timeout
- protocol-specific frame extractor
- structured peer endpoint
- graceful shutdown on SIGTERM
- [ ] **Step 2: Implement MQTT client**
Rules:
- connect with official production config from environment
- subscribe configured topic list
- convert each message into envelope
- publish raw and unified events
- reconnect with backoff
- [ ] **Step 3: Verify local JSON mode**
Run without Kafka:
```bash
cd go/vehicle-gateway
GB32960_TCP_ADDR=:132960 JT808_TCP_ADDR=:18080 go run ./cmd/gateway
```
Expected: service starts and logs configured listeners.
---
### Task 10: Docker and ECS Deployment
**Files:**
- Create: `go/vehicle-gateway/Dockerfile`
- Create: `deploy/portainer/docker-compose-go.yml`
- Modify: `docs/operations/current-ecs-deployment.md`
- [ ] **Step 1: Add multi-stage Dockerfile**
Build all commands:
- `gateway`
- `history-writer`
- `stat-writer`
- `realtime-api`
- [ ] **Step 2: Add Portainer compose**
Services:
- `go-vehicle-gateway`
- `go-history-writer`
- `go-stat-writer`
- `go-realtime-api`
Each service must include:
- restart policy
- memory limit
- Kafka env
- MySQL/TDengine/Redis env as needed
- logging options
- [ ] **Step 3: Verify Linux build**
Run:
```bash
cd go/vehicle-gateway
GOOS=linux GOARCH=amd64 go build ./cmd/gateway
GOOS=linux GOARCH=amd64 go build ./cmd/history-writer
GOOS=linux GOARCH=amd64 go build ./cmd/stat-writer
GOOS=linux GOARCH=amd64 go build ./cmd/realtime-api
```
Expected: all commands exit `0`.
---
### Task 11: Production Verification
**Files:**
- Create: `docs/operations/go-vehicle-gateway-verification.md`
- [ ] **Step 1: Record test commands**
Document commands to verify:
- gateway process health
- Kafka topic consumption
- TDengine row counts
- MySQL daily metric rows
- Redis realtime lookup
- [ ] **Step 2: Validate real traffic**
Evidence required:
- one real 32960 VIN with RAW, location, mileage point, daily metric, Redis snapshot
- one real JT808 phone/VIN with RAW, location, mileage point, daily metric, Redis snapshot
- one real Yutong MQTT VIN with RAW and Redis snapshot
- [ ] **Step 3: Keep old Java services until evidence is captured**
Only disable Java equivalents after the evidence file contains successful command outputs and timestamps.
---
## Self-Review Checklist
- The plan creates one new Go module instead of extending scattered prototypes.
- The plan covers GB32960, JT808, and Yutong MQTT ingress.
- The plan covers Kafka, TDengine, MySQL, and Redis.
- The plan includes 32960 and 808 daily mileage and daily total mileage.
- The plan includes local and ECS verification.
- The plan does not restore Xinda Push.
- The plan keeps Java services untouched until Go evidence exists.

View File

@@ -1,3 +1,9 @@
> **Superseded:** This 2026-06-23 DuckDB-based design is historical context.
> Use `docs/target-architecture.md` and
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
> current production architecture: TDengine is the default hot history store,
> `event-file-store` and Xinda Push have been removed from the production codebase.
# GB32960 Production Readiness Design
## Goal
@@ -337,23 +343,19 @@ Validation:
### Daily Mileage
Preferred:
JT808 daily mileage uses the GPS total mileage reported in location additional
information. The local-day minimum and maximum GPS total mileage values rewrite
the same metric row, so ordered streaming and historical replay use one
calculation path.
```text
dailyMileage = today.lastTotalMileage - previousDay.lastTotalMileage
dailyMileage = today.maxTotalMileage - today.minTotalMileage
calculationMethod = JT808_TOTAL_MILEAGE_DIFF
```
Fallback:
```text
dailyMileage = today.lastTotalMileage - today.firstTotalMileage
```
Quality:
- `PREVIOUS_DAY_BASELINE` when previous day baseline exists;
- `INTRADAY_DELTA` when only same-day baseline exists;
- `INSUFFICIENT_DATA` when neither is reliable.
The result is stored in `vehicle_stat_metric` with
`metric_key=daily_mileage_km`. There is no JT808-specific daily mileage table,
no Redis mileage state, and no previous-day baseline calculation.
### Daily Power Consumption

View File

@@ -1,3 +1,10 @@
> **Superseded:** This 2026-06-23 split design is historical context.
> Use `docs/target-architecture.md` and
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
> current production architecture: TDengine is the default hot history store,
> `sink-archive` owns raw bytes, and `event-file-store` is optional
> compatibility only.
# GB32960 Service Split Design
Date: 2026-06-23
@@ -53,7 +60,7 @@ Initial module dependencies:
- `ingest-codec-common`
- `session-core`
- `protocol-gb32960`
- `sink-mq`
- `sink-kafka`
- `observability`
### `vehicle-history-app`
@@ -79,7 +86,7 @@ Non-responsibilities:
Initial module dependencies:
- `ingest-api`
- `sink-mq`
- `sink-kafka`
- `sink-archive`
- `event-file-store`
- `event-history-service`
@@ -109,7 +116,7 @@ Non-responsibilities:
Initial module dependencies:
- `ingest-api`
- `sink-mq`
- `sink-kafka`
- `vehicle-state-service`
- `vehicle-stat-service`
- `observability`
@@ -126,7 +133,7 @@ Use versioned topics:
- `vehicle.event.gb32960.v1`
- `vehicle.dlq.gb32960.v1`
The current topic names under `lingniu.ingest.sink.mq.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
The current topic names under `lingniu.ingest.sink.kafka.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
### Keys

View File

@@ -433,7 +433,7 @@ Realtime APIs:
Realtime state APIs:
- `GET /api/realtime/vehicles/{vehicleKey}`
- Reads Redis or in-memory latest state, not TDengine.
- Reads Redis latest state, not TDengine.
Export:
@@ -486,7 +486,7 @@ Read path:
- Most high-QPS queries require vehicle key and time range.
- Unbounded cross-vehicle queries are admin-only and capped.
- Realtime latest state is served from Redis/in-memory state.
- Realtime latest state is served from Redis latest-state cache.
- Raw detail replay reads one raw object and decodes on demand.
- Common dictionary and metadata responses are cached in memory.

View File

@@ -0,0 +1,498 @@
# Go 车辆接入重构设计规格
## 目标
使用 Go 重新设计车辆数据接入运行面,让系统回到第一性原则:
- 接入层只负责可靠收包、协议解析、应答、身份解析和投递。
- Kafka 是唯一在线消息通道。
- TDengine 保存高吞吐时序明细和完整 RAW 解析 JSON。
- MySQL 保存低频配置、身份映射和每日统计窄表。
- Redis 保存可重建的准实时状态。
- 32960、JT808、宇通 MQTT 使用同一套数据层接口,避免每个协议各自落库。
第一阶段只做生产链路的骨架和核心数据闭环接收、解析、Kafka、TDengine、MySQL 统计、Redis 实时查询、ECS 验证。历史查询 API 和业务侧复杂分析放在后续阶段。
## 参考原则
本设计只吸收开源项目的边界思想,不复制外部实现代码。
- GB32960 参考 `DarkInno/gb32960-go-sdk`连接管理、Handler、Forwarder、鉴权接口分离。
- JT808 参考 `cuteLittleDevil/go-jt808`:高并发 Go 原生 TCP、协议核心小而可扩展、事件/适配器机制。
- JT808 参考 `fakeyanss/jt808-server-go``FramePayload -> PacketData -> JT808Msg -> Reply` 的分层处理、Gateway 模式、保活和版本兼容。
- TDengine 参考官方数据模型:按数据采集点建立 supertable静态维度做 tags明细数据按子表写入Go 连接优先使用 WebSocket 驱动。
## 非目标
- 第一阶段不继续扩展 Java 服务。
- 第一阶段不做统一大查询 API。
- 第一阶段不恢复信达 Push。
- 第一阶段不把所有协议字段展开成一张超宽 TDengine 表。
- Redis 不作为历史事实源。
- MySQL 不保存高频原始明细。
## 目标目录
新的 Go 运行面放在 `go/vehicle-gateway`,不继续沿用实验性的多 module 分散结构。现有 `go/ingest-edge``go/vehicle-state` 可以作为迁移素材,最终归并或删除。
```text
go/vehicle-gateway/
cmd/
gateway/ # 协议接入进程32960 TCP、JT808 TCP、宇通 MQTT
history-writer/ # Kafka -> TDengine RAW/location/mileage points
stat-writer/ # Kafka -> MySQL 每日统计窄表
realtime-api/ # Redis 准实时查询 API
internal/
config/ # 环境变量、配置文件、校验
gateway/ # TCP/MQTT 生命周期、连接限制、优雅停机
protocol/
gb32960/ # 32960 编解码
jt808/ # 808 编解码
yutongmqtt/ # 宇通 MQTT payload 解析
identity/ # VIN、phone、device_id、plate 绑定
envelope/ # 统一事件模型
eventbus/ # Kafka producer/consumer
history/ # TDengine adapter
stats/ # 每日里程聚合
realtime/ # Redis adapter 与快照合并
observability/ # 日志、metrics、健康检查
```
## 协议接入设计
### GB32960
职责:
- 监听 TCP `32960`
- 支持车辆登入、实时信息上报、补发信息上报、车辆登出、平台登入/登出、心跳、校时。
- 正确返回协议应答。
- 完整解析 2016 数据单元:
- 整车数据
- 驱动电机数据
- 燃料电池数据
- 发动机数据
- 车辆位置数据
- 极值数据
- 报警数据
- 可充电储能装置电压数据
- 可充电储能装置温度数据
- RAW 中保存完整 parsed JSON。
- 关键字段标准化为统一 field
- `speed_kmh`
- `total_mileage_km`
- `soc_percent`
- `longitude`
- `latitude`
- `vehicle_status`
- `charge_status`
### JT808
职责:
- 监听 TCP `808`
- 支持 2011/2013 基础兼容,预留 2019 版本标记。
- 分层处理:
- Frame`0x7e` 包边界、转义、校验。
- Packet消息头、手机号 BCD、流水号、分包。
- Message注册、鉴权、心跳、注销、位置、批量位置、未知上行。
- Reply注册应答、平台通用应答。
- 终端手机号按协议头 BCD 解析,内部同时保留原始 BCD、规范化 phone、去前导零 phone。
- 注册帧和鉴权帧写入 808 registration 表。
- 如果设备直接上报位置帧,也要走 identity 解析,尝试用 phone、device_id、plate 找到 VIN。
- 0200 附加信息必须解析表 27
- `0x01` GPS 总里程DWORD单位 0.1 km。
- 其他已知附加项结构化放入 `parsed_json.additional`
- 标准化字段:
- `speed_kmh`
- `total_mileage_km`
- `longitude`
- `latitude`
- `altitude_m`
- `direction_deg`
- `alarm_flag`
- `status_flag`
### 宇通 MQTT
职责:
- 使用正式 MQTT 配置订阅生产 topic。
- 接收 payload 后解析为统一 envelope。
- 按 payload 中的车辆标识解析 VIN、车牌、设备号。
- 保存完整 payload 和 parsed JSON。
- 标准化字段向 32960/808 对齐:
- `speed_kmh`
- `total_mileage_km`
- `longitude`
- `latitude`
- `soc_percent`
- `vehicle_status`
## 统一 Envelope
所有协议接入后输出同一结构。
```json
{
"event_id": "string",
"trace_id": "string",
"protocol": "GB32960|JT808|YUTONG_MQTT",
"message_id": "string",
"vin": "string",
"vehicle_key": "string",
"phone": "string",
"device_id": "string",
"plate": "string",
"source_endpoint": "ip:port or mqtt://broker/topic",
"event_time_ms": 0,
"received_at_ms": 0,
"raw_hex": "string",
"raw_text": "string",
"parsed": {},
"fields": {
"speed_kmh": 0,
"total_mileage_km": 0,
"longitude": 0,
"latitude": 0
},
"parse_status": "OK|PARTIAL|BAD_FRAME",
"parse_error": ""
}
```
规则:
- `event_id` 使用协议、设备标识、消息流水、事件时间、RAW hash 生成,保证幂等。
- Kafka partition key 优先使用 VIN没有 VIN 时使用 `protocol:phone/device_id`
- `parsed` 保存协议完整结构化字段。
- `fields` 只保存跨协议核心字段,用于统计、位置和实时快照。
## Kafka Topic
生产只支持 Kafka。
| Topic | 内容 | Key |
|---|---|---|
| `vehicle.raw.gb32960.v1` | 32960 完整 RAW envelope | VIN 或 vehicle_key |
| `vehicle.raw.jt808.v1` | 808 完整 RAW envelope | VIN 或 phone |
| `vehicle.raw.yutong-mqtt.v1` | 宇通 MQTT 完整 RAW envelope | VIN 或 vehicle_key |
| `vehicle.event.unified.v1` | 统一轻量事件,用于业务消费 | VIN 或 vehicle_key |
接入层必须先写 RAW topic再写 unified event。后续消费者只依赖 Kafka不直接依赖接入进程内存。
发布可靠性:
- Kafka 生产端必须开启同步写入RAW 写成功后才允许写 unified event。
- Kafka 写入必须支持可配置重试、单次写超时和短退避,默认值为 `KAFKA_PUBLISH_ATTEMPTS=3``KAFKA_PUBLISH_TIMEOUT_MS=3000``KAFKA_PUBLISH_BACKOFF_MS=100`
- gateway 支持本地磁盘 spool/WAL配置 `KAFKA_SPOOL_DIR` 后启用Kafka 长时间不可用时,发布失败的 envelope 先原子写入本地 JSON 文件。
- spool 补发按文件名顺序执行,补发成功后删除文件;默认补发间隔由 `KAFKA_SPOOL_REPLAY_INTERVAL_MS=1000` 控制。
- 如果 RAW 写 Kafka 失败并已落盘,同一个 event 的 unified event 不允许抢先写 Kafka也必须进入 spool确保恢复后按 RAW -> unified 顺序补发。
- Kafka topic 不允许自动创建topic 和分区数由部署脚本或运维初始化,避免生产拼写错误造成隐性分流。
## TDengine 数据库设计
如果现有 TDengine 库不符合目标,可以新建库并迁移。建议库名:
```sql
CREATE DATABASE IF NOT EXISTS lingniu_vehicle_ts KEEP 7300 DURATION 10 BUFFER 256;
```
### raw_frames
保存完整 RAW 和完整 parsed JSON。
```sql
CREATE STABLE IF NOT EXISTS raw_frames (
ts TIMESTAMP,
frame_id NCHAR(64),
event_id NCHAR(64),
message_id INT,
event_time TIMESTAMP,
received_at TIMESTAMP,
raw_size_bytes INT,
raw_hex BINARY(16374),
raw_text BINARY(16374),
parsed_json BINARY(16374),
fields_json BINARY(4096),
parse_status NCHAR(16),
parse_error BINARY(1024),
source_endpoint NCHAR(128)
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
);
```
说明:
- 子表按 `protocol + vehicle_key hash` 创建。
- RAW 查询优先按 tags 和时间范围过滤。
- `parsed_json` 是完整协议字段;查询 API 可选择是否返回,避免默认大字段拖慢。
### vehicle_locations
只保存位置查询核心字段。
```sql
CREATE STABLE IF NOT EXISTS vehicle_locations (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
longitude DOUBLE,
latitude DOUBLE,
altitude_m DOUBLE,
speed_kmh DOUBLE,
direction_deg INT,
alarm_flag BIGINT,
status_flag BIGINT,
total_mileage_km DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
);
```
### vehicle_mileage_points
保存总里程采样点,用于回放重算和查询。
```sql
CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
total_mileage_km DOUBLE,
speed_kmh DOUBLE,
longitude DOUBLE,
latitude DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
);
```
## MySQL 数据库设计
MySQL 只保存配置、身份映射、808 注册鉴权和每日统计窄表。
### vehicle_identity_binding
用于把外部标识定位到 VIN。
```sql
CREATE TABLE vehicle_identity_binding (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vin VARCHAR(32) NOT NULL,
plate VARCHAR(32) NULL,
phone VARCHAR(32) NULL,
device_id VARCHAR(64) NULL,
remark VARCHAR(255) NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_vin (vin),
KEY idx_plate (plate),
KEY idx_phone (phone),
KEY idx_device_id (device_id)
);
```
### jt808_registration
只记录 808 独有的注册、鉴权和在线身份信息。
```sql
CREATE TABLE jt808_registration (
phone VARCHAR(32) PRIMARY KEY,
vin VARCHAR(32) NULL,
plate VARCHAR(32) NULL,
device_id VARCHAR(64) NULL,
manufacturer_id VARCHAR(32) NULL,
terminal_model VARCHAR(64) NULL,
terminal_id VARCHAR(64) NULL,
auth_code VARCHAR(128) NULL,
source_endpoint VARCHAR(128) NULL,
first_registered_at DATETIME NULL,
latest_registered_at DATETIME NULL,
latest_authed_at DATETIME NULL,
latest_seen_at DATETIME NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
KEY idx_vin (vin),
KEY idx_plate (plate),
KEY idx_device_id (device_id)
);
```
### vehicle_daily_metric
32960 和 808 的每日里程、每日总里程都进入同一张窄表。
```sql
CREATE TABLE vehicle_daily_metric (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vin VARCHAR(32) NOT NULL,
stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL,
metric_key VARCHAR(64) NOT NULL,
metric_value DECIMAL(18,3) NOT NULL,
metric_unit VARCHAR(16) NOT NULL,
first_total_mileage_km DECIMAL(18,3) NULL,
latest_total_mileage_km DECIMAL(18,3) NULL,
sample_count BIGINT NOT NULL DEFAULT 0,
calculation_method VARCHAR(64) NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
KEY idx_stat_date (stat_date),
KEY idx_protocol_metric (protocol, metric_key)
);
```
指标:
- `daily_mileage_km = max(total_mileage_km) - min(total_mileage_km)`
- `daily_total_mileage_km = max(total_mileage_km)`
统计消费者可幂等重放。每个样本 upsert 时只维护当天 `min/max total_mileage_km`,不依赖进程内状态。
## Redis 实时数据设计
Redis 保存可从 Kafka 重建的数据。
| Key | 内容 |
|---|---|
| `vehicle:online:{vin}` | 在线状态、最后活跃时间、协议列表 |
| `vehicle:latest:{vin}` | 合并后的 VIN 最新快照 |
| `vehicle:latest:{vin}:{protocol}` | 单协议最新快照 |
| `vehicle:last_seen` | VIN 到最后接收时间的 sorted set |
合并规则:
- 位置以最新 event_time 为准。
- 总里程取最新上报值,不做递增假设。
- 32960 多帧字段按 field timestamp 合并。
- 808 和 MQTT 不覆盖其他协议独有字段。
- Redis TTL 用于在线判断,不作为数据删除依据。
## 数据流
```mermaid
flowchart LR
source["车辆 / 平台"] --> gateway["Go gateway"]
gateway --> parser["protocol parser"]
parser --> identity["identity resolver"]
identity --> kafkaRaw["Kafka RAW topics"]
identity --> kafkaEvent["Kafka unified event"]
kafkaRaw --> history["history-writer"]
kafkaRaw --> stats["stat-writer"]
kafkaEvent --> realtime["realtime consumer"]
history --> td["TDengine raw_frames / locations / mileage_points"]
stats --> mysql["MySQL vehicle_daily_metric"]
realtime --> redis["Redis latest state"]
```
## 错误处理
- 协议坏帧仍写 RAW topic`parse_status=BAD_FRAME`
- 可部分解析的帧写 `parse_status=PARTIAL`,保留 parse error。
- Kafka 写失败时接入层先按配置重试;重试耗尽后不写 unified event必须打错误日志和 metrics。
- 启用 `KAFKA_SPOOL_DIR`Kafka 重试耗尽会落本地 spool如果 spool 写失败,才视为接入层最终失败。
- TDengine 写失败不提交 Kafka offset。
- MySQL 统计写失败不提交 Kafka offset。
- Redis 写失败不影响历史和统计,但要通过 metrics 暴露。
- 808 相同 phone 新连接时,旧连接应被替换并记录 latest_seen。
## 部署设计
生产 ECS 运行 Go 二进制或容器:
- `vehicle-gateway`
- `vehicle-history-writer`
- `vehicle-stat-writer`
- `vehicle-realtime-api`
配置来源:
- 环境变量优先。
- Nacos 可作为后续配置中心,但 Go 第一阶段必须能只靠环境变量启动,便于 Portainer 部署和故障恢复。
TDengine 连接:
- 优先 WebSocket `taosWS`
- 若当前 ECS 只开放原生端口,可短期兼容 `taosSql`,但代码配置必须显式标记为兼容模式。
## 验证计划
### 本地验证
- 用样例 808 报文验证:
- phone BCD 解析。
- 0200 经纬度、速度、方向、时间。
- 表 27 `0x01` GPS 总里程。
- 用真实 32960 报文验证:
- 登录应答正确。
- 实时帧和补发帧都能写 RAW。
- 位置和总里程字段进入统一 fields。
- 用宇通 MQTT payload 验证:
- payload 完整保存。
- 核心字段归一化。
- 不连 Kafka 时可输出 JSON log。
- 连接 Kafka 时 RAW topic 可消费到 envelope。
### ECS 验证
- 部署前先旁路接收或短窗口切流。
- 验证端口:
- `32960`
- `808`
- MQTT 订阅连接。
- 验证 Kafka topic 有持续写入。
- 验证 TDengine
- `raw_frames` 有三种协议数据。
- `vehicle_locations` 能按 VIN、协议、时间分页查询。
- `vehicle_mileage_points` 有 32960 和 808 总里程采样。
- 验证 MySQL
- `vehicle_daily_metric` 有 32960 和 808 的 `daily_mileage_km``daily_total_mileage_km`
- 验证 Redis
- 查询 VIN 在线。
- 查询 VIN 合并实时快照。
- 查询 VIN 分协议实时快照。
## 验收标准
- Go gateway 能在 ECS 上接收真实 32960、808、宇通 MQTT 数据。
- 三种协议 RAW 都进入 Kafka。
- Kafka 短暂写失败时gateway 按配置重试;单元测试覆盖首次失败后成功和重试耗尽返回错误。
- Kafka 长时间不可用时gateway 可把 RAW/unified 写入本地 spool单元测试覆盖 RAW 已落盘时 unified 也落盘、恢复后按顺序补发并删除文件。
- 三种协议 RAW 和 parsed JSON 都进入 TDengine。
- 32960 和 808 的位置进入 `vehicle_locations`
- 32960 和 808 的总里程采样进入 `vehicle_mileage_points`
- 32960 和 808 的每日里程、每日总里程进入 MySQL 窄表。
- Redis 可以查询 VIN 在线和实时数据。
- 任一消费者宕机后可通过 Kafka offset 继续恢复。
- 查询和统计验证使用 ECS 上的真实数据完成。
## 实施顺序
1. 创建 `go/vehicle-gateway` 单 module迁移现有 Go 原型中可用的解析、Kafka、TDengine、MySQL、Redis 代码。
2. 先写协议解析单元测试,覆盖 808 样例报文和 32960 核心字段。
3. 完成统一 envelope 和 Kafka sink。
4. 完成 TDengine schema bootstrap 与 history writer。
5. 完成 MySQL schema bootstrap 与 stat writer。
6. 完成 Redis realtime writer/API。
7. 本地启动 gateway用 ECS 转发流量验证。
8. 构建 Linux 镜像,部署 ECS。
9. 按验收标准逐项验证并记录证据。

View File

@@ -5,6 +5,8 @@ The project should stay small at runtime: protocol apps ingest and publish,
history consumes and indexes, analytics derives metrics, and business systems
read through explicit APIs or Kafka.
Only supported message backbone: Kafka.
## Active Scope
Active production protocols:
@@ -15,9 +17,12 @@ Active production protocols:
| JT/T 808 | `jt808-ingest-app` | `vehicle.event.jt808.v1` | `vehicle.raw.jt808.v1` |
| Yutong MQTT | `yutong-mqtt-app` | `vehicle.event.mqtt-yutong.v1` | `vehicle.raw.mqtt-yutong.v1` |
Xinda Push is legacy and is not part of the default production surface.
It may remain in source for reference, but new optimization work should target
the active protocols above.
Xinda Push source, Maven profile, deployment bindings, and history consumers
are removed. New optimization work should target the active protocols above.
JSATL12 attachment upload is also outside the default production reactor. Keep
it available through the `optional-attachments` profile until an attachment app
is explicitly deployed.
## Goals
@@ -28,8 +33,7 @@ the active protocols above.
- Store complete raw payload metadata and parsed JSON in TDengine `raw_frames`.
- Store query-friendly location rows separately from raw payload JSON.
- Keep derived statistics in metric tables, not protocol-specific daily tables.
- Use Redis only for hot state or streaming calculation state that can be
rebuilt from Kafka.
- Use Redis only for optional latest-state APIs that can be rebuilt from Kafka.
- Keep optional compatibility modules outside the production hot path.
## Non-Goals
@@ -37,9 +41,10 @@ the active protocols above.
- Ingest apps do not write business tables.
- Ingest apps do not serve history query APIs.
- Redis is not the long-term historical store.
- `event-file-store` Parquet/DuckDB is not the default history store.
- `telemetry_fields` parsing is not handled by this project when a separate
field parsing service owns that job.
- File-based event indexes are not part of the current build surface.
- `telemetry_fields` parsing and field-trend APIs are owned by a separate
field parsing service. This project stores RAW parsed JSON plus compact
location rows and does not expose a telemetry-fields history API.
## Runtime Responsibilities
@@ -66,7 +71,7 @@ repositories, and statistic calculators.
### Event Contract
`sink-mq` owns the Kafka protobuf envelope and consumer plumbing.
The Kafka sink owns the protobuf envelope and consumer plumbing.
Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped
envelopes, and downstream failures are converted to structured results and DLQ
records instead of blocking a partition.
@@ -104,11 +109,9 @@ Production default:
- `TDENGINE_HISTORY_ENABLED=true` in deployment.
- `EVENT_FILE_STORE_ENABLED=false`.
- `TDENGINE_TELEMETRY_FIELDS_ENABLED=false`.
`event-file-store` remains an optional compatibility path for old low-level
record APIs or local investigations. It should not be treated as the primary
history backend.
File-based event indexes have been removed; this app should keep history
queries on TDengine and raw replay on `archive://...` references.
### Analytics App
@@ -120,22 +123,25 @@ Responsibilities:
- Calculate 808 daily mileage from the reported GPS total mileage only:
```text
daily_mileage_km = last_total_mileage_km - first_total_mileage_km
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
calculation_method = JT808_TOTAL_MILEAGE_DIFF
```
- Store the metric in `vehicle_stat_metric` with
`metric_key = daily_mileage_km`; the first and latest GPS total mileage used
for the subtraction stay on the same metric row as calculation source columns.
`metric_key = daily_mileage_km`; the local-day minimum and maximum GPS total
mileage used for the subtraction stay on the same metric row as calculation
source columns.
There is no JT808-specific daily mileage table and no in-memory production
state-store mode. Restart recovery reads the same metric row, so it does not
require a separate Redis mileage state.
Runtime state: none outside `vehicle_stat_metric`. There is no JT808-specific
daily mileage table. Restart recovery reads the same metric row.
### Latest State
Module: `vehicle-state-service`
This module is outside the default production reactor. Build it with
`-Poptional-latest-state` when Redis latest-state APIs are explicitly needed.
Responsibilities:
- Consume normalized Kafka envelopes.
@@ -147,6 +153,8 @@ source of truth for historical queries.
## Data Flow
Default production flow:
```mermaid
flowchart LR
vehicle["Vehicles / Platforms"] --> ingest["Protocol apps"]
@@ -154,12 +162,14 @@ flowchart LR
ingest --> kafka["Kafka<br/>event + raw envelopes"]
kafka --> history["vehicle-history-app"]
kafka --> analytics["vehicle-analytics-app"]
kafka --> state["vehicle-state-service"]
history --> tdengine["TDengine<br/>raw_frames + locations"]
analytics --> mysql["MySQL<br/>vehicle_stat_metric"]
state --> redisState["Redis<br/>latest state"]
```
Optional latest-state flow, enabled only with `-Poptional-latest-state`, consumes
the same Kafka envelopes and rebuilds Redis snapshots outside the default
production deployment.
## Failure Handling
- Protocol apps continue accepting traffic when history, analytics, or Redis
@@ -176,13 +186,17 @@ flowchart LR
- Active production deployment contains GB32960, JT808, Yutong MQTT,
vehicle-history, and vehicle-analytics apps.
- Xinda Push is absent from default deployment and history consumer bindings.
- History APIs read from TDengine by default; Parquet/DuckDB is optional
compatibility only.
- Xinda Push source and deployment bindings are removed.
- JSATL12 is absent from the default Maven reactor and available through
`optional-attachments`.
- vehicle-state-service is absent from the default Maven reactor and available
through `optional-latest-state`.
- raw-archive-store prototype has been removed; raw bytes are written through
`sink-archive`.
- History APIs read from TDengine; file-based event indexes are not maintained.
- Raw frame queries can return complete parsed JSON through `payloadJson.parsed`.
- Location queries page over compact rows and reference raw records instead of
duplicating full raw JSON.
- JT808 daily mileage is stored only in `vehicle_stat_metric`.
- Runtime state needed for 808 mileage is also in `vehicle_stat_metric`, with no
Redis or production memory state-store mode.
- Runtime state needed for 808 mileage is also in `vehicle_stat_metric`.
- Build validation covers the active modules and their composition tests.

View File

@@ -0,0 +1,29 @@
FROM golang:1.26-bookworm AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/gateway ./cmd/gateway \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/history-writer ./cmd/history-writer \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/stat-writer ./cmd/stat-writer \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/realtime-api ./cmd/realtime-api
FROM debian:bookworm-slim
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates tzdata \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=build /out/gateway /app/gateway
COPY --from=build /out/history-writer /app/history-writer
COPY --from=build /out/stat-writer /app/stat-writer
COPY --from=build /out/realtime-api /app/realtime-api
ENV TZ=Asia/Shanghai
EXPOSE 808 32960 20210
CMD ["/app/gateway"]

View File

@@ -0,0 +1,223 @@
package main
import (
"context"
"database/sql"
"log/slog"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
_ "github.com/go-sql-driver/mysql"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/gateway"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
)
func main() {
logger := observability.NewLogger("vehicle-gateway")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
sink, err := buildSink(ctx, logger)
if err != nil {
logger.Error("build sink failed", "error", err)
os.Exit(1)
}
defer sink.Close()
resolver, closeResolver, err := buildIdentityResolver(ctx, logger)
if err != nil {
logger.Error("build identity resolver failed", "error", err)
os.Exit(1)
}
defer closeResolver()
protocols := []gateway.TCPProtocol{
{
Protocol: envelope.ProtocolGB32960,
Addr: env("GB32960_TCP_ADDR", ":32960"),
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Respond: gb32960.AutoResponse,
},
{
Protocol: envelope.ProtocolJT808,
Addr: env("JT808_TCP_ADDR", ":808"),
Extract: jt808.ExtractFrames,
Parse: jt808.ParseFrame,
Respond: jt808.NewAutoResponder(env("JT808_REGISTER_AUTH_CODE", "g7gps")).Respond,
},
}
var wg sync.WaitGroup
errs := make(chan error, len(protocols))
for _, protocol := range protocols {
server, err := gateway.NewTCPServer(gateway.TCPServerConfig{
Protocol: protocol,
Sink: sink,
Resolver: resolver,
Logger: logger,
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
})
if err != nil {
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
os.Exit(1)
}
wg.Add(1)
go func() {
defer wg.Done()
if err := server.ListenAndServe(ctx); err != nil {
errs <- err
}
}()
}
if envBool("YUTONG_MQTT_ENABLED", false) {
client, err := gateway.NewMQTTClient(gateway.MQTTClientConfig{
EndpointName: env("YUTONG_MQTT_ENDPOINT", env("YUTONG_MQTT_ENDPOINT_NAME", "yutong")),
Broker: env("YUTONG_MQTT_URI", ""),
ClientID: env("YUTONG_MQTT_CLIENT_ID", "lingniu-go-yutong-mqtt"),
Username: env("YUTONG_MQTT_USERNAME", ""),
Password: env("YUTONG_MQTT_PASSWORD", ""),
Topics: splitCSV(env("YUTONG_MQTT_TOPICS", env("YUTONG_MQTT_TOPIC", "/ytforward/shln/+"))),
QoS: byte(envInt("YUTONG_MQTT_QOS", 2)),
CleanSession: envBool("YUTONG_MQTT_CLEAN_SESSION", false),
KeepAlive: time.Duration(envInt("YUTONG_MQTT_KEEP_ALIVE_SECONDS", 20)) * time.Second,
ConnectTimeout: time.Duration(envInt("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", 10)) * time.Second,
TLSCACertPath: env("YUTONG_MQTT_TLS_CA_PEM", ""),
TLSClientCertPath: env("YUTONG_MQTT_TLS_CLIENT_PEM", ""),
TLSClientKeyPath: env("YUTONG_MQTT_TLS_CLIENT_KEY", ""),
TLSHostnameVerification: envBool("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", true),
Sink: sink,
Resolver: resolver,
Logger: logger,
})
if err != nil {
logger.Error("build yutong mqtt client failed", "error", err)
os.Exit(1)
}
if err := client.Start(ctx); err != nil {
logger.Error("start yutong mqtt client failed", "error", err)
os.Exit(1)
}
logger.Info("yutong mqtt client started")
}
logger.Info("vehicle gateway started")
select {
case <-ctx.Done():
case err := <-errs:
logger.Error("vehicle gateway listener failed", "error", err)
stop()
}
wg.Wait()
}
func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.Resolver, func(), error) {
dsn := env("IDENTITY_MYSQL_DSN", strings.TrimSpace(os.Getenv("MYSQL_DSN")))
if strings.TrimSpace(dsn) == "" {
logger.Warn("identity mysql dsn is empty; using noop identity resolver")
return identity.NoopResolver{}, func() {}, nil
}
db, err := sql.Open("mysql", dsn)
if err != nil {
return nil, nil, err
}
if err := db.PingContext(ctx); err != nil {
_ = db.Close()
return nil, nil, err
}
table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding")
logger.Info("identity mysql resolver enabled", "table", table)
return identity.NewMySQLResolver(db, table), func() { _ = db.Close() }, nil
}
func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error) {
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
if len(brokers) == 0 {
logger.Warn("KAFKA_BROKERS is empty; using log sink")
return eventbus.NewLogSink(logger), nil
}
sink, err := eventbus.NewKafkaSink(eventbus.KafkaConfig{
Brokers: brokers,
RawTopics: map[envelope.Protocol]string{
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1"),
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1"),
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1"),
},
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1"),
})
if err != nil {
return nil, err
}
var out eventbus.Sink = eventbus.NewRetryingSink(sink, eventbus.RetryConfig{
Attempts: envInt("KAFKA_PUBLISH_ATTEMPTS", 3),
Backoff: time.Duration(envInt("KAFKA_PUBLISH_BACKOFF_MS", 100)) * time.Millisecond,
AttemptTimeout: time.Duration(envInt("KAFKA_PUBLISH_TIMEOUT_MS", 3000)) * time.Millisecond,
})
spoolDir := strings.TrimSpace(os.Getenv("KAFKA_SPOOL_DIR"))
if spoolDir != "" {
durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{Directory: spoolDir})
interval := time.Duration(envInt("KAFKA_SPOOL_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond
go durable.ReplayLoop(ctx, interval, func(err error) {
logger.Warn("kafka spool replay failed", "error", err)
})
logger.Info("kafka durable spool enabled", "dir", spoolDir, "replay_interval_ms", interval.Milliseconds())
out = durable
}
return out, nil
}
func env(key string, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func envInt(key string, fallback int) int {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
var out int
for _, r := range value {
if r < '0' || r > '9' {
return fallback
}
out = out*10 + int(r-'0')
}
if out <= 0 {
return fallback
}
return out
}
func envBool(key string, fallback bool) bool {
value := strings.ToLower(strings.TrimSpace(os.Getenv(key)))
if value == "" {
return fallback
}
return value == "1" || value == "true" || value == "yes" || value == "on"
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item != "" {
out = append(out, item)
}
}
return out
}

View File

@@ -0,0 +1,123 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"os"
"os/signal"
"strings"
"syscall"
"github.com/segmentio/kafka-go"
_ "github.com/taosdata/driver-go/v3/taosWS"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
)
func main() {
logger := observability.NewLogger("vehicle-history-writer")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cfg := loadConfig()
db, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
if err != nil {
logger.Error("tdengine open failed", "error", err)
os.Exit(1)
}
defer db.Close()
if err := db.PingContext(ctx); err != nil {
logger.Error("tdengine ping failed", "error", err)
os.Exit(1)
}
writer := history.NewWriter(db)
if cfg.EnsureSchema {
if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
logger.Error("tdengine schema bootstrap failed", "error", err)
os.Exit(1)
}
}
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: cfg.KafkaBrokers,
GroupID: cfg.KafkaGroup,
GroupTopics: cfg.KafkaTopics,
MinBytes: 1,
MaxBytes: 10e6,
})
defer reader.Close()
logger.Info("history writer started",
"driver", cfg.TDengineDriver,
"group", cfg.KafkaGroup,
"topics", strings.Join(cfg.KafkaTopics, ","))
for {
message, err := reader.FetchMessage(ctx)
if err != nil {
if ctx.Err() != nil {
return
}
logger.Error("kafka fetch failed", "error", err)
continue
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = reader.CommitMessages(ctx, message)
continue
}
if err := writer.AppendAll(ctx, env); err != nil {
logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
continue
}
if err := reader.CommitMessages(ctx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
}
}
type config struct {
KafkaBrokers []string
KafkaTopics []string
KafkaGroup string
TDengineDriver string
TDengineDSN string
TDengineDatabase string
EnsureSchema bool
}
func loadConfig() config {
return config{
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1")),
KafkaGroup: env("KAFKA_GROUP", "go-history-writer"),
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
TDengineDSN: env("TDENGINE_DSN", ""),
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
EnsureSchema: env("TDENGINE_ENSURE_SCHEMA", "true") != "false",
}
}
func env(key string, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item != "" {
out = append(out, item)
}
}
return out
}

View File

@@ -0,0 +1,198 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
_ "github.com/go-sql-driver/mysql"
"github.com/redis/go-redis/v9"
"github.com/segmentio/kafka-go"
_ "github.com/taosdata/driver-go/v3/taosWS"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
func main() {
logger := observability.NewLogger("vehicle-realtime-api")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
client := redis.NewClient(&redis.Options{
Addr: env("REDIS_ADDR", "127.0.0.1:6379"),
Username: env("REDIS_USERNAME", ""),
Password: env("REDIS_PASSWORD", ""),
DB: envInt("REDIS_DB", 0),
})
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
logger.Error("redis ping failed", "error", err)
os.Exit(1)
}
repository := realtime.NewRepository(client, realtime.Config{
OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", 600)) * time.Second,
})
if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
go consumeKafka(ctx, logger, repository, brokers)
} else {
logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
}
mux := http.NewServeMux()
mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository))
closeStats := func() {}
if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" {
db, err := sql.Open("mysql", dsn)
if err != nil {
logger.Error("stats mysql open failed", "error", err)
os.Exit(1)
}
if err := db.PingContext(ctx); err != nil {
_ = db.Close()
logger.Error("stats mysql ping failed", "error", err)
os.Exit(1)
}
closeStats = func() { _ = db.Close() }
mux.Handle("/api/stats/daily-metrics", stats.NewMetricHandler(stats.NewMetricRepository(db)))
logger.Info("stats mysql query enabled")
} else {
mux.HandleFunc("/api/stats/daily-metrics", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(map[string]any{"error": "MYSQL_DSN is not configured"})
})
logger.Warn("MYSQL_DSN is empty; stats query api disabled")
}
defer closeStats()
closeHistory := func() {}
if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" {
driver := env("TDENGINE_DRIVER", "taosWS")
db, err := sql.Open(driver, dsn)
if err != nil {
logger.Error("history tdengine open failed", "driver", driver, "error", err)
os.Exit(1)
}
if err := db.PingContext(ctx); err != nil {
_ = db.Close()
logger.Error("history tdengine ping failed", "driver", driver, "error", err)
os.Exit(1)
}
closeHistory = func() { _ = db.Close() }
database := env("TDENGINE_DATABASE", history.DefaultDatabase)
mux.Handle("/api/history/raw-frames", history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)))
mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database)))
mux.Handle("/api/history/mileage-points", history.NewMileagePointHandler(history.NewMileagePointRepository(db, database)))
logger.Info("history query enabled", "driver", driver, "database", database)
} else {
historyUnavailable := func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"})
}
mux.HandleFunc("/api/history/raw-frames", historyUnavailable)
mux.HandleFunc("/api/history/locations", historyUnavailable)
mux.HandleFunc("/api/history/mileage-points", historyUnavailable)
logger.Warn("TDENGINE_DSN is empty; history query api disabled")
}
defer closeHistory()
server := &http.Server{
Addr: env("HTTP_ADDR", ":20210"),
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
}
go func() {
<-ctx.Done()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = server.Shutdown(shutdownCtx)
}()
logger.Info("realtime api started", "addr", server.Addr)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error("http server failed", "error", err)
os.Exit(1)
}
}
func consumeKafka(ctx context.Context, logger interface {
Info(string, ...any)
Error(string, ...any)
Warn(string, ...any)
}, repository *realtime.Repository, brokers []string) {
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: brokers,
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
GroupTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.event.unified.v1")),
MinBytes: 1,
MaxBytes: 10e6,
})
defer reader.Close()
logger.Info("realtime kafka consumer started", "topics", env("KAFKA_TOPICS", "vehicle.event.unified.v1"))
for {
message, err := reader.FetchMessage(ctx)
if err != nil {
if ctx.Err() != nil {
return
}
logger.Error("kafka fetch failed", "error", err)
continue
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = reader.CommitMessages(ctx, message)
continue
}
if err := repository.Update(ctx, env); err != nil {
logger.Error("redis realtime update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
continue
}
if err := reader.CommitMessages(ctx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
}
}
func env(key string, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func envInt(key string, fallback int) int {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
parsed, err := strconv.Atoi(value)
if err != nil {
return fallback
}
return parsed
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item != "" {
out = append(out, item)
}
}
return out
}

View File

@@ -0,0 +1,122 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"os"
"os/signal"
"strings"
"syscall"
"time"
_ "github.com/go-sql-driver/mysql"
"github.com/segmentio/kafka-go"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
func main() {
logger := observability.NewLogger("vehicle-stat-writer")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cfg := loadConfig()
db, err := sql.Open("mysql", cfg.MySQLDSN)
if err != nil {
logger.Error("mysql open failed", "error", err)
os.Exit(1)
}
defer db.Close()
if err := db.PingContext(ctx); err != nil {
logger.Error("mysql ping failed", "error", err)
os.Exit(1)
}
writer := stats.NewWriter(db, cfg.Location)
if cfg.EnsureSchema {
if err := writer.EnsureSchema(ctx); err != nil {
logger.Error("mysql schema bootstrap failed", "error", err)
os.Exit(1)
}
}
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: cfg.KafkaBrokers,
GroupID: cfg.KafkaGroup,
GroupTopics: cfg.KafkaTopics,
MinBytes: 1,
MaxBytes: 10e6,
})
defer reader.Close()
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","))
for {
message, err := reader.FetchMessage(ctx)
if err != nil {
if ctx.Err() != nil {
return
}
logger.Error("kafka fetch failed", "error", err)
continue
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = reader.CommitMessages(ctx, message)
continue
}
if err := writer.Append(ctx, env); err != nil {
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
continue
}
if err := reader.CommitMessages(ctx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
}
}
type config struct {
KafkaBrokers []string
KafkaTopics []string
KafkaGroup string
MySQLDSN string
EnsureSchema bool
Location *time.Location
}
func loadConfig() config {
loc, err := time.LoadLocation(env("LOCAL_TZ", "Asia/Shanghai"))
if err != nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
return config{
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1")),
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
MySQLDSN: env("MYSQL_DSN", ""),
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",
Location: loc,
}
}
func env(key string, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item != "" {
out = append(out, item)
}
}
return out
}

30
go/vehicle-gateway/go.mod Normal file
View File

@@ -0,0 +1,30 @@
module lingniu-vehicle-ingest/go/vehicle-gateway
go 1.26
require (
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/alicebob/miniredis/v2 v2.35.0
github.com/eclipse/paho.mqtt.golang v1.5.1
github.com/go-sql-driver/mysql v1.9.3
github.com/redis/go-redis/v9 v9.17.2
github.com/segmentio/kafka-go v0.4.49
github.com/taosdata/driver-go/v3 v3.8.1
golang.org/x/text v0.29.0
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
golang.org/x/net v0.44.0 // indirect
golang.org/x/sync v0.17.0 // indirect
)

73
go/vehicle-gateway/go.sum Normal file
View File

@@ -0,0 +1,73 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE=
github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/taosdata/driver-go/v3 v3.8.1 h1:kkd4ABsGiU+oXDbsw/sic985LKAvnpF9Gb/TEunTnLE=
github.com/taosdata/driver-go/v3 v3.8.1/go.mod h1:S6OGOinfR0xxxaMGsvBi9cLkYxEIW1p6qqr8QJATTlg=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,95 @@
package envelope
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
)
type Protocol string
const (
ProtocolGB32960 Protocol = "GB32960"
ProtocolJT808 Protocol = "JT808"
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
)
type ParseStatus string
const (
ParseOK ParseStatus = "OK"
ParsePartial ParseStatus = "PARTIAL"
ParseBadFrame ParseStatus = "BAD_FRAME"
)
const (
FieldSpeedKMH = "speed_kmh"
FieldTotalMileageKM = "total_mileage_km"
FieldLongitude = "longitude"
FieldLatitude = "latitude"
FieldSOCPercent = "soc_percent"
)
type FrameEnvelope struct {
EventID string `json:"event_id"`
TraceID string `json:"trace_id"`
Protocol Protocol `json:"protocol"`
MessageID string `json:"message_id"`
Sequence uint16 `json:"sequence"`
VIN string `json:"vin,omitempty"`
VehicleKeyHint string `json:"vehicle_key,omitempty"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
Plate string `json:"plate,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
Parsed map[string]any `json:"parsed,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
ParseStatus ParseStatus `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
}
func (e FrameEnvelope) VehicleKey() string {
if key := strings.TrimSpace(e.VIN); key != "" {
return key
}
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
return key
}
if key := strings.TrimSpace(e.Phone); key != "" {
return string(e.Protocol) + ":" + key
}
if key := strings.TrimSpace(e.DeviceID); key != "" {
return string(e.Protocol) + ":" + key
}
return string(e.Protocol) + ":unknown"
}
func (e FrameEnvelope) StableEventID() string {
if strings.TrimSpace(e.EventID) != "" {
return e.EventID
}
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
sum := sha256.Sum256([]byte(input))
return hex.EncodeToString(sum[:16])
}
func (e FrameEnvelope) KafkaKey() []byte {
return []byte(e.VehicleKey())
}
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
if e.EventID == "" {
e.EventID = e.StableEventID()
}
if e.ParseStatus == "" {
e.ParseStatus = ParseOK
}
return json.Marshal(e)
}

View File

@@ -0,0 +1,55 @@
package envelope
import (
"encoding/json"
"testing"
)
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
t.Fatalf("VehicleKey() = %q", got)
}
}
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
if got := e.VehicleKey(); got != "JT808:013307795425" {
t.Fatalf("VehicleKey() = %q", got)
}
}
func TestFrameEnvelopeEventIDStable(t *testing.T) {
e := FrameEnvelope{
Protocol: ProtocolJT808,
MessageID: "0x0200",
Phone: "013307795425",
Sequence: 1,
EventTimeMS: 1782745114000,
ReceivedAtMS: 1782745114999,
RawHex: "7e02000000ff7e",
}
a := e.StableEventID()
b := e.StableEventID()
if a == "" || a != b {
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
}
}
func TestFrameEnvelopeMarshalDefaults(t *testing.T) {
e := FrameEnvelope{Protocol: ProtocolGB32960, VIN: "LNBVIN00000000002", MessageID: "0x02"}
payload, err := e.MarshalJSONBytes()
if err != nil {
t.Fatalf("MarshalJSONBytes() error = %v", err)
}
var decoded FrameEnvelope
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if decoded.EventID == "" {
t.Fatal("event id should be filled")
}
if decoded.ParseStatus != ParseOK {
t.Fatalf("parse status = %q", decoded.ParseStatus)
}
}

View File

@@ -0,0 +1,187 @@
package eventbus
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type DurableConfig struct {
Directory string
}
type DurableSink struct {
delegate Sink
dir string
mu sync.Mutex
seq uint64
rawPending map[string]struct{}
}
type durableRecord struct {
Kind string `json:"kind"`
Envelope envelope.FrameEnvelope `json:"envelope"`
}
func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
if delegate == nil {
panic("durable delegate sink must not be nil")
}
return &DurableSink{
delegate: delegate,
dir: strings.TrimSpace(cfg.Directory),
rawPending: map[string]struct{}{},
}
}
func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
if err := s.delegate.PublishRaw(ctx, env); err == nil {
return nil
}
if err := s.spool("raw", env); err != nil {
return err
}
s.markRawPending(env)
return nil
}
func (s *DurableSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
if s.isRawPending(env) {
return s.spool("unified", env)
}
if err := s.delegate.PublishUnified(ctx, env); err == nil {
return nil
}
return s.spool("unified", env)
}
func (s *DurableSink) ReplayOnce(ctx context.Context) error {
files, err := filepath.Glob(filepath.Join(s.dir, "*.json"))
if err != nil {
return err
}
sort.Strings(files)
for _, file := range files {
record, err := readDurableRecord(file)
if err != nil {
return err
}
if err := s.publishRecord(ctx, record); err != nil {
return err
}
if err := os.Remove(file); err != nil {
return err
}
if record.Kind == "raw" {
s.clearRawPending(record.Envelope)
}
}
return nil
}
func (s *DurableSink) ReplayLoop(ctx context.Context, interval time.Duration, onError func(error)) {
if interval <= 0 {
interval = time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := s.ReplayOnce(ctx); err != nil && onError != nil {
onError(err)
}
}
}
}
func (s *DurableSink) Close() error {
return s.delegate.Close()
}
func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) error {
switch record.Kind {
case "raw":
return s.delegate.PublishRaw(ctx, record.Envelope)
case "unified":
return s.delegate.PublishUnified(ctx, record.Envelope)
default:
return fmt.Errorf("unknown durable record kind %q", record.Kind)
}
}
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
if s.dir == "" {
return fmt.Errorf("durable spool directory is empty")
}
if env.EventID == "" {
env.EventID = env.StableEventID()
}
if env.ParseStatus == "" {
env.ParseStatus = envelope.ParseOK
}
if err := os.MkdirAll(s.dir, 0o750); err != nil {
return err
}
payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env})
if err != nil {
return err
}
name := s.nextFileName(env, kind)
path := filepath.Join(s.dir, name)
tmp := path + ".tmp"
if err := os.WriteFile(tmp, payload, 0o640); err != nil {
return err
}
return os.Rename(tmp, path)
}
func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string {
s.mu.Lock()
defer s.mu.Unlock()
s.seq++
eventID := env.StableEventID()
if len(eventID) > 12 {
eventID = eventID[:12]
}
return fmt.Sprintf("%020d-%06d-%s-%s.json", time.Now().UnixNano(), s.seq, kind, eventID)
}
func (s *DurableSink) markRawPending(env envelope.FrameEnvelope) {
s.mu.Lock()
defer s.mu.Unlock()
s.rawPending[env.StableEventID()] = struct{}{}
}
func (s *DurableSink) clearRawPending(env envelope.FrameEnvelope) {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.rawPending, env.StableEventID())
}
func (s *DurableSink) isRawPending(env envelope.FrameEnvelope) bool {
s.mu.Lock()
defer s.mu.Unlock()
_, ok := s.rawPending[env.StableEventID()]
return ok
}
func readDurableRecord(path string) (durableRecord, error) {
var record durableRecord
payload, err := os.ReadFile(path)
if err != nil {
return record, err
}
return record, json.Unmarshal(payload, &record)
}

View File

@@ -0,0 +1,153 @@
package eventbus
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) {
dir := t.TempDir()
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
env := durableTestEnvelope()
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("PublishUnified() error = %v", err)
}
if delegate.rawCalls != 1 {
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
}
if delegate.unifiedCalls != 0 {
t.Fatalf("unified should not be delegated while raw is spooled, calls = %d", delegate.unifiedCalls)
}
files := spoolFiles(t, dir)
if len(files) != 2 {
t.Fatalf("spool files = %d, want 2: %#v", len(files), files)
}
assertSpoolKind(t, files[0], "raw")
assertSpoolKind(t, files[1], "unified")
}
func TestDurableSinkReplayPublishesInFileOrderAndDeletesFiles(t *testing.T) {
dir := t.TempDir()
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
env := durableTestEnvelope()
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("PublishUnified() error = %v", err)
}
delegate.rawErrors = nil
if err := sink.ReplayOnce(context.Background()); err != nil {
t.Fatalf("ReplayOnce() error = %v", err)
}
got := delegate.calls
want := []string{"raw", "raw", "unified"}
if len(got) != len(want) {
t.Fatalf("calls = %#v, want %#v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("calls[%d] = %q, want %q; all=%#v", i, got[i], want[i], got)
}
}
if files := spoolFiles(t, dir); len(files) != 0 {
t.Fatalf("spool files after replay = %#v, want none", files)
}
}
func durableTestEnvelope() envelope.FrameEnvelope {
return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
Phone: "013079963379",
VIN: "LKLG7C4E3NA774736",
Sequence: 183,
EventTimeMS: 1782903940000,
ReceivedAtMS: 1782917751903,
RawHex: "0200002201307996337900B7",
}
}
func spoolFiles(t *testing.T, dir string) []string {
t.Helper()
matches, err := filepath.Glob(filepath.Join(dir, "*.json"))
if err != nil {
t.Fatalf("glob spool files: %v", err)
}
return matches
}
func assertSpoolKind(t *testing.T, path string, kind string) {
t.Helper()
payload, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read spool file: %v", err)
}
if !containsString(string(payload), `"kind":"`+kind+`"`) {
t.Fatalf("spool file %s payload = %s, want kind %q", path, payload, kind)
}
}
func containsString(value string, needle string) bool {
return len(needle) == 0 || (len(value) >= len(needle) && indexString(value, needle) >= 0)
}
func indexString(value string, needle string) int {
for i := 0; i+len(needle) <= len(value); i++ {
if value[i:i+len(needle)] == needle {
return i
}
}
return -1
}
var errSpoolTest = errors.New("delegate unavailable")
type scriptedSink struct {
rawErrors []error
unifiedErrors []error
rawCalls int
unifiedCalls int
calls []string
}
func (s *scriptedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
s.rawCalls++
s.calls = append(s.calls, "raw")
if len(s.rawErrors) > 0 {
err := s.rawErrors[0]
s.rawErrors = s.rawErrors[1:]
return err
}
return nil
}
func (s *scriptedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
s.unifiedCalls++
s.calls = append(s.calls, "unified")
if len(s.unifiedErrors) > 0 {
err := s.unifiedErrors[0]
s.unifiedErrors = s.unifiedErrors[1:]
return err
}
return nil
}
func (s *scriptedSink) Close() error {
return nil
}

View File

@@ -0,0 +1,94 @@
package eventbus
import (
"context"
"errors"
"fmt"
"github.com/segmentio/kafka-go"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Sink interface {
PublishRaw(context.Context, envelope.FrameEnvelope) error
PublishUnified(context.Context, envelope.FrameEnvelope) error
Close() error
}
type KafkaSink struct {
writer kafkaWriter
rawTopics map[envelope.Protocol]string
unifiedTopic string
}
type kafkaWriter interface {
WriteMessages(context.Context, ...kafka.Message) error
Close() error
}
type KafkaConfig struct {
Brokers []string
RawTopics map[envelope.Protocol]string
UnifiedTopic string
}
func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) {
if len(cfg.Brokers) == 0 {
return nil, errors.New("kafka brokers are required")
}
return newKafkaSinkWithWriter(&kafka.Writer{
Addr: kafka.TCP(cfg.Brokers...),
Balancer: &kafka.Hash{},
AllowAutoTopicCreation: false,
}, cfg), nil
}
func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
rawTopics := map[envelope.Protocol]string{
envelope.ProtocolGB32960: "vehicle.raw.gb32960.v1",
envelope.ProtocolJT808: "vehicle.raw.jt808.v1",
envelope.ProtocolYutongMQTT: "vehicle.raw.yutong-mqtt.v1",
}
for protocol, topic := range cfg.RawTopics {
if topic != "" {
rawTopics[protocol] = topic
}
}
unifiedTopic := cfg.UnifiedTopic
if unifiedTopic == "" {
unifiedTopic = "vehicle.event.unified.v1"
}
return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic}
}
func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
topic, ok := s.rawTopics[env.Protocol]
if !ok || topic == "" {
return fmt.Errorf("raw topic not configured for protocol %s", env.Protocol)
}
return s.publish(ctx, topic, env)
}
func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
return s.publish(ctx, s.unifiedTopic, env)
}
func (s *KafkaSink) Close() error {
if s == nil || s.writer == nil {
return nil
}
return s.writer.Close()
}
func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
payload, err := env.MarshalJSONBytes()
if err != nil {
return err
}
return s.writer.WriteMessages(ctx, kafka.Message{
Topic: topic,
Key: env.KafkaKey(),
Value: payload,
})
}

View File

@@ -0,0 +1,77 @@
package eventbus
import (
"context"
"encoding/json"
"testing"
"github.com/segmentio/kafka-go"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestKafkaSinkRoutesRawTopics(t *testing.T) {
writer := &recordingWriter{}
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "013307795425", MessageID: "0x0200"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if len(writer.messages) != 1 {
t.Fatalf("messages = %d", len(writer.messages))
}
msg := writer.messages[0]
if msg.Topic != "vehicle.raw.jt808.v1" {
t.Fatalf("topic = %q", msg.Topic)
}
if string(msg.Key) != "JT808:013307795425" {
t.Fatalf("key = %q", string(msg.Key))
}
var decoded envelope.FrameEnvelope
if err := json.Unmarshal(msg.Value, &decoded); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if decoded.EventID == "" || decoded.ParseStatus != envelope.ParseOK {
t.Fatalf("unexpected payload defaults: %#v", decoded)
}
}
func TestKafkaSinkRoutesUnifiedTopic(t *testing.T) {
writer := &recordingWriter{}
sink := newKafkaSinkWithWriter(writer, KafkaConfig{UnifiedTopic: "custom.unified"})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000001", MessageID: "0x02"}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("PublishUnified() error = %v", err)
}
if writer.messages[0].Topic != "custom.unified" {
t.Fatalf("topic = %q", writer.messages[0].Topic)
}
if string(writer.messages[0].Key) != "LNBVIN00000000001" {
t.Fatalf("key = %q", string(writer.messages[0].Key))
}
}
func TestKafkaSinkRejectsUnknownProtocol(t *testing.T) {
writer := &recordingWriter{}
sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")})
if err == nil {
t.Fatal("expected unknown protocol error")
}
}
type recordingWriter struct {
messages []kafka.Message
}
func (w *recordingWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
w.messages = append(w.messages, messages...)
return nil
}
func (w *recordingWriter) Close() error {
return nil
}

View File

@@ -0,0 +1,33 @@
package eventbus
import (
"context"
"log/slog"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type LogSink struct {
logger *slog.Logger
}
func NewLogSink(logger *slog.Logger) *LogSink {
if logger == nil {
logger = slog.Default()
}
return &LogSink{logger: logger}
}
func (s *LogSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
s.logger.Info("raw envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID, "status", env.ParseStatus)
return nil
}
func (s *LogSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
s.logger.Info("unified envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID)
return nil
}
func (s *LogSink) Close() error {
return nil
}

View File

@@ -0,0 +1,80 @@
package eventbus
import (
"context"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type RetryConfig struct {
Attempts int
Backoff time.Duration
AttemptTimeout time.Duration
}
type RetryingSink struct {
delegate Sink
cfg RetryConfig
}
func NewRetryingSink(delegate Sink, cfg RetryConfig) *RetryingSink {
if delegate == nil {
panic("retry delegate sink must not be nil")
}
if cfg.Attempts <= 0 {
cfg.Attempts = 1
}
return &RetryingSink{delegate: delegate, cfg: cfg}
}
func (s *RetryingSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
return s.withRetry(ctx, func(attemptCtx context.Context) error {
return s.delegate.PublishRaw(attemptCtx, env)
})
}
func (s *RetryingSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
return s.withRetry(ctx, func(attemptCtx context.Context) error {
return s.delegate.PublishUnified(attemptCtx, env)
})
}
func (s *RetryingSink) Close() error {
return s.delegate.Close()
}
func (s *RetryingSink) withRetry(ctx context.Context, publish func(context.Context) error) error {
var lastErr error
for attempt := 1; attempt <= s.cfg.Attempts; attempt++ {
attemptCtx := ctx
cancel := func() {}
if s.cfg.AttemptTimeout > 0 {
attemptCtx, cancel = context.WithTimeout(ctx, s.cfg.AttemptTimeout)
}
err := publish(attemptCtx)
cancel()
if err == nil {
return nil
}
lastErr = err
if ctx.Err() != nil {
return ctx.Err()
}
if attempt == s.cfg.Attempts {
break
}
if s.cfg.Backoff > 0 {
timer := time.NewTimer(s.cfg.Backoff)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return ctx.Err()
case <-timer.C:
}
}
}
return lastErr
}

View File

@@ -0,0 +1,80 @@
package eventbus
import (
"context"
"errors"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestRetryingSinkRetriesRawUntilSuccess(t *testing.T) {
delegate := &flakySink{rawFailures: 1}
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 3})
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
if err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if delegate.rawCalls != 2 {
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
}
}
func TestRetryingSinkReturnsLastErrorAfterAttempts(t *testing.T) {
delegate := &flakySink{rawFailures: 3}
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 2})
err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
if !errors.Is(err, errFlakyPublish) {
t.Fatalf("PublishRaw() error = %v, want %v", err, errFlakyPublish)
}
if delegate.rawCalls != 2 {
t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
}
}
func TestRetryingSinkStopsWhenContextCanceledDuringBackoff(t *testing.T) {
delegate := &flakySink{rawFailures: 10}
sink := NewRetryingSink(delegate, RetryConfig{Attempts: 5, Backoff: time.Minute})
ctx, cancel := context.WithCancel(context.Background())
cancel()
err := sink.PublishRaw(ctx, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
if !errors.Is(err, context.Canceled) {
t.Fatalf("PublishRaw() error = %v, want context canceled", err)
}
if delegate.rawCalls != 1 {
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
}
}
var errFlakyPublish = errors.New("publish failed")
type flakySink struct {
rawFailures int
unifiedFailures int
rawCalls int
unifiedCalls int
}
func (s *flakySink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
s.rawCalls++
if s.rawCalls <= s.rawFailures {
return errFlakyPublish
}
return nil
}
func (s *flakySink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
s.unifiedCalls++
if s.unifiedCalls <= s.unifiedFailures {
return errFlakyPublish
}
return nil
}
func (s *flakySink) Close() error {
return nil
}

View File

@@ -0,0 +1,218 @@
package gateway
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"errors"
"fmt"
"log/slog"
"os"
"strings"
"time"
mqtt "github.com/eclipse/paho.mqtt.golang"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
)
type MQTTClientConfig struct {
EndpointName string
Broker string
ClientID string
Username string
Password string
Topics []string
QoS byte
CleanSession bool
KeepAlive time.Duration
ConnectTimeout time.Duration
TLSCACertPath string
TLSClientCertPath string
TLSClientKeyPath string
TLSHostnameVerification bool
Sink eventbus.Sink
Resolver identity.Resolver
Logger *slog.Logger
}
type MQTTClient struct {
cfg MQTTClientConfig
client mqtt.Client
}
func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
if strings.TrimSpace(cfg.Broker) == "" {
return nil, errors.New("mqtt broker is required")
}
if strings.TrimSpace(cfg.ClientID) == "" {
return nil, errors.New("mqtt client id is required")
}
if len(cfg.Topics) == 0 {
return nil, errors.New("mqtt topics are required")
}
if cfg.Sink == nil {
return nil, errors.New("sink is required")
}
if cfg.Logger == nil {
cfg.Logger = slog.Default()
}
if cfg.Resolver == nil {
cfg.Resolver = identity.NoopResolver{}
}
if cfg.EndpointName == "" {
cfg.EndpointName = "yutong"
}
return &MQTTClient{cfg: cfg}, nil
}
func (c *MQTTClient) Start(ctx context.Context) error {
opts, err := c.buildOptions(ctx)
if err != nil {
return err
}
c.client = mqtt.NewClient(opts)
token := c.client.Connect()
if token.Wait() && token.Error() != nil {
return token.Error()
}
go func() {
<-ctx.Done()
if c.client != nil && c.client.IsConnected() {
c.client.Disconnect(250)
}
}()
return nil
}
func (c *MQTTClient) buildOptions(ctx context.Context) (*mqtt.ClientOptions, error) {
keepAlive := c.cfg.KeepAlive
if keepAlive <= 0 {
keepAlive = 20 * time.Second
}
connectTimeout := c.cfg.ConnectTimeout
if connectTimeout <= 0 {
connectTimeout = 10 * time.Second
}
opts := mqtt.NewClientOptions().
AddBroker(c.cfg.Broker).
SetClientID(c.cfg.ClientID).
SetUsername(c.cfg.Username).
SetPassword(c.cfg.Password).
SetCleanSession(c.cfg.CleanSession).
SetAutoReconnect(true).
SetConnectRetry(true).
SetConnectRetryInterval(5 * time.Second).
SetResumeSubs(true).
SetOrderMatters(false).
SetKeepAlive(keepAlive).
SetConnectTimeout(connectTimeout)
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
c.handleMessage(ctx, message.Topic(), message.Payload())
})
opts.OnConnect = func(client mqtt.Client) {
for _, topic := range c.cfg.Topics {
token := client.Subscribe(topic, c.cfg.QoS, nil)
if token.Wait() && token.Error() != nil {
c.cfg.Logger.Error("mqtt subscribe failed", "broker", c.cfg.Broker, "topic", topic, "error", token.Error())
continue
}
c.cfg.Logger.Info("mqtt subscribed", "broker", c.cfg.Broker, "topic", topic, "qos", c.cfg.QoS)
}
}
opts.OnConnectionLost = func(_ mqtt.Client, err error) {
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
}
tlsConfig, err := c.buildTLSConfig()
if err != nil {
return nil, err
}
if tlsConfig != nil {
opts.SetTLSConfig(tlsConfig)
}
return opts, nil
}
func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
caPath := strings.TrimSpace(c.cfg.TLSCACertPath)
certPath := strings.TrimSpace(c.cfg.TLSClientCertPath)
keyPath := strings.TrimSpace(c.cfg.TLSClientKeyPath)
if caPath == "" && certPath == "" && keyPath == "" {
return nil, nil
}
config := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: !c.cfg.TLSHostnameVerification,
}
if caPath != "" {
caPEM, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("read mqtt ca certificate: %w", err)
}
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(caPEM) {
return nil, fmt.Errorf("parse mqtt ca certificate %s", caPath)
}
config.RootCAs = roots
}
if certPath != "" || keyPath != "" {
if certPath == "" || keyPath == "" {
return nil, errors.New("mqtt client certificate and key must be configured together")
}
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, fmt.Errorf("load mqtt client certificate: %w", err)
}
config.Certificates = []tls.Certificate{cert}
}
return config, nil
}
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
receivedAtMS := time.Now().UnixMilli()
env, err := yutongmqtt.ParseMessage(c.cfg.EndpointName, topic, payload, receivedAtMS)
if err != nil {
env = envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
MessageID: "MQTT",
VehicleKeyHint: "unknown",
SourceEndpoint: "mqtt://" + c.cfg.EndpointName + topic,
EventTimeMS: receivedAtMS,
ReceivedAtMS: receivedAtMS,
RawText: string(payload),
RawHex: strings.ToUpper(hex.EncodeToString(payload)),
ParseStatus: envelope.ParseBadFrame,
ParseError: err.Error(),
}
env.EventID = env.StableEventID()
} else {
resolved, resolveErr := c.cfg.Resolver.Resolve(ctx, env)
if resolveErr != nil {
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
env.ParseStatus = envelope.ParsePartial
} else {
env = resolved
}
}
if err := c.cfg.Sink.PublishRaw(ctx, env); err != nil {
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
return
}
if env.ParseStatus == envelope.ParseBadFrame {
return
}
if err := c.cfg.Sink.PublishUnified(ctx, env); err != nil {
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
}
}

View File

@@ -0,0 +1,183 @@
package gateway
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"log/slog"
"math/big"
"os"
"path/filepath"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
sink := &recordingSink{}
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
"device":"LTEST000000000001",
"time":"20260413100000",
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
}`))
if len(sink.raw) != 1 || len(sink.unified) != 1 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
t.Fatalf("unexpected raw envelope: %#v", sink.raw[0])
}
}
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
sink := &recordingSink{}
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/bad", []byte("{bad-json"))
if len(sink.raw) != 1 || len(sink.unified) != 0 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
}
}
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
dir := t.TempDir()
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "ssl://mqtt.example.test:8883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
QoS: 1,
Sink: &recordingSink{},
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
TLSCACertPath: caPath,
TLSClientCertPath: certPath,
TLSClientKeyPath: keyPath,
TLSHostnameVerification: false,
CleanSession: true,
KeepAlive: 20 * time.Second,
ConnectTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
opts, err := client.buildOptions(context.Background())
if err != nil {
t.Fatalf("buildOptions() error = %v", err)
}
if opts.TLSConfig == nil {
t.Fatal("TLSConfig is nil")
}
if opts.TLSConfig.RootCAs == nil {
t.Fatal("RootCAs is nil")
}
if len(opts.TLSConfig.Certificates) != 1 {
t.Fatalf("client certificates = %d, want 1", len(opts.TLSConfig.Certificates))
}
if !opts.TLSConfig.InsecureSkipVerify {
t.Fatal("InsecureSkipVerify should be true when hostname verification is disabled")
}
if !opts.CleanSession {
t.Fatal("CleanSession should be true")
}
if opts.Order {
t.Fatal("OrderMatters should be false so MQTT network handling is not blocked by Kafka/DB work")
}
if !opts.ResumeSubs {
t.Fatal("ResumeSubs should be true to avoid subscribe failures during reconnect churn")
}
if got := opts.KeepAlive; got != 20 {
t.Fatalf("KeepAlive = %d, want 20", got)
}
if got := opts.ConnectTimeout; got != 10*time.Second {
t.Fatalf("ConnectTimeout = %v, want 10s", got)
}
}
func writeTestTLSMaterial(t *testing.T, dir string) (string, string, string) {
t.Helper()
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate ca key: %v", err)
}
caTemplate := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "test-ca"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
BasicConstraintsValid: true,
IsCA: true,
}
caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
if err != nil {
t.Fatalf("create ca cert: %v", err)
}
clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate client key: %v", err)
}
clientTemplate := &x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{CommonName: "test-client"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey)
if err != nil {
t.Fatalf("create client cert: %v", err)
}
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "client.pem")
keyPath := filepath.Join(dir, "client-key.pem")
writePEM(t, caPath, "CERTIFICATE", caDER)
writePEM(t, certPath, "CERTIFICATE", clientDER)
keyDER := x509.MarshalPKCS1PrivateKey(clientKey)
writePEM(t, keyPath, "RSA PRIVATE KEY", keyDER)
return caPath, certPath, keyPath
}
func writePEM(t *testing.T, path, typ string, der []byte) {
t.Helper()
file, err := os.Create(path)
if err != nil {
t.Fatalf("create %s: %v", path, err)
}
defer file.Close()
if err := pem.Encode(file, &pem.Block{Type: typ, Bytes: der}); err != nil {
t.Fatalf("write pem %s: %v", path, err)
}
}

View File

@@ -0,0 +1,238 @@
package gateway
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"net"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
)
type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
type TCPProtocol struct {
Protocol envelope.Protocol
Addr string
Extract FrameExtractor
Parse FrameParser
Respond FrameResponder
}
type TCPServer struct {
protocol TCPProtocol
sink eventbus.Sink
resolver identity.Resolver
logger *slog.Logger
readBufferSize int
idleTimeout time.Duration
maxConnections int
}
type TCPServerConfig struct {
Protocol TCPProtocol
Sink eventbus.Sink
Resolver identity.Resolver
Logger *slog.Logger
ReadBufferSize int
IdleTimeout time.Duration
MaxConnections int
}
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
if cfg.Protocol.Protocol == "" {
return nil, errors.New("protocol is required")
}
if strings.TrimSpace(cfg.Protocol.Addr) == "" {
return nil, errors.New("listen addr is required")
}
if cfg.Protocol.Extract == nil {
return nil, errors.New("frame extractor is required")
}
if cfg.Protocol.Parse == nil {
return nil, errors.New("frame parser is required")
}
if cfg.Sink == nil {
return nil, errors.New("sink is required")
}
if cfg.Logger == nil {
cfg.Logger = slog.Default()
}
if cfg.Resolver == nil {
cfg.Resolver = identity.NoopResolver{}
}
if cfg.ReadBufferSize <= 0 {
cfg.ReadBufferSize = 32 * 1024
}
if cfg.IdleTimeout <= 0 {
cfg.IdleTimeout = 2 * time.Minute
}
if cfg.MaxConnections <= 0 {
cfg.MaxConnections = 10_000
}
return &TCPServer{
protocol: cfg.Protocol,
sink: cfg.Sink,
resolver: cfg.Resolver,
logger: cfg.Logger,
readBufferSize: cfg.ReadBufferSize,
idleTimeout: cfg.IdleTimeout,
maxConnections: cfg.MaxConnections,
}, nil
}
func (s *TCPServer) ListenAndServe(ctx context.Context) error {
var lc net.ListenConfig
listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr)
if err != nil {
return err
}
defer listener.Close()
go func() {
<-ctx.Done()
_ = listener.Close()
}()
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
sem := make(chan struct{}, s.maxConnections)
var wg sync.WaitGroup
defer wg.Wait()
for {
conn, err := listener.Accept()
if err != nil {
if ctx.Err() != nil {
return nil
}
s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err)
continue
}
select {
case sem <- struct{}{}:
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sem }()
s.handleConnection(ctx, conn)
}()
default:
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
_ = conn.Close()
}
}
}
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
defer conn.Close()
source := conn.RemoteAddr().String()
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
log.Info("tcp connection opened")
defer log.Info("tcp connection closed")
readBuffer := make([]byte, s.readBufferSize)
var pending []byte
for {
_ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout))
n, err := conn.Read(readBuffer)
if n > 0 {
pending = append(pending, readBuffer[:n]...)
frames, remainder, extractErr := s.protocol.Extract(pending)
if extractErr != nil {
log.Warn("frame extraction failed", "error", extractErr)
return
}
pending = remainder
for _, frame := range frames {
s.handleFrame(ctx, conn, frame, source)
}
}
if err != nil {
if errors.Is(err, io.EOF) {
return
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
log.Warn("tcp connection idle timeout")
return
}
log.Warn("tcp read failed", "error", err)
return
}
if ctx.Err() != nil {
return
}
}
}
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string) {
receivedAtMS := time.Now().UnixMilli()
env, err := s.protocol.Parse(raw, receivedAtMS, source)
if err != nil {
env = envelope.FrameEnvelope{
Protocol: s.protocol.Protocol,
SourceEndpoint: source,
ReceivedAtMS: receivedAtMS,
EventTimeMS: receivedAtMS,
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
ParseStatus: envelope.ParseBadFrame,
ParseError: err.Error(),
}
env.EventID = env.StableEventID()
} else {
resolved, resolveErr := s.resolver.Resolve(ctx, env)
if resolveErr != nil {
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
env.ParseStatus = envelope.ParsePartial
} else {
env = resolved
}
}
if err := s.sink.PublishRaw(ctx, env); err != nil {
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if env.ParseStatus == envelope.ParseBadFrame {
return
}
if err := s.sink.PublishUnified(ctx, env); err != nil {
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if s.protocol.Respond == nil {
return
}
response, ok, err := s.protocol.Respond(raw, env)
if err != nil {
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if !ok || len(response) == 0 {
return
}
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := conn.Write(response); err != nil {
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
}
}
func (p TCPProtocol) String() string {
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
}

View File

@@ -0,0 +1,177 @@
package gateway
import (
"context"
"encoding/hex"
"io"
"log/slog"
"net"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
)
func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
if err != nil {
t.Fatal(err)
}
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolJT808,
Addr: ":0",
Extract: jt808.ExtractFrames,
Parse: jt808.ParseFrame,
}, sink)
client, done := runPipe(t, server)
if _, err := client.Write(frame); err != nil {
t.Fatalf("client.Write() error = %v", err)
}
_ = client.Close()
<-done
if len(sink.raw) != 1 || len(sink.unified) != 1 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].Phone != "013307795425" {
t.Fatalf("phone = %q", sink.raw[0].Phone)
}
if sink.raw[0].Fields[envelope.FieldTotalMileageKM] != 10241.2 {
t.Fatalf("total mileage = %#v", sink.raw[0].Fields[envelope.FieldTotalMileageKM])
}
}
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
good[len(good)-1] ^= 0xff
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolGB32960,
Addr: ":0",
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
}, sink)
client, done := runPipe(t, server)
if _, err := client.Write(good); err != nil {
t.Fatalf("client.Write() error = %v", err)
}
_ = client.Close()
<-done
if len(sink.raw) != 1 || len(sink.unified) != 0 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
}
if sink.raw[0].ParseError == "" {
t.Fatal("parse error should be recorded")
}
}
func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
frame := buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil)
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolGB32960,
Addr: ":0",
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
return []byte("ACK"), true, nil
},
}, sink)
client, done := runPipe(t, server)
if _, err := client.Write(frame); err != nil {
t.Fatalf("client.Write() error = %v", err)
}
buf := make([]byte, 3)
if _, err := io.ReadFull(client, buf); err != nil {
t.Fatalf("read response error = %v", err)
}
if string(buf) != "ACK" {
t.Fatalf("response = %q", string(buf))
}
_ = client.Close()
<-done
}
func newTestServer(t *testing.T, protocol TCPProtocol, sink *recordingSink) *TCPServer {
t.Helper()
server, err := NewTCPServer(TCPServerConfig{
Protocol: protocol,
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
ReadBufferSize: 1024,
IdleTimeout: time.Second,
MaxConnections: 1,
})
if err != nil {
t.Fatalf("NewTCPServer() error = %v", err)
}
return server
}
func runPipe(t *testing.T, server *TCPServer) (net.Conn, <-chan struct{}) {
t.Helper()
client, srv := net.Pipe()
done := make(chan struct{})
go func() {
defer close(done)
server.handleConnection(context.Background(), srv)
}()
return client, done
}
type recordingSink struct {
raw []envelope.FrameEnvelope
unified []envelope.FrameEnvelope
}
func (s *recordingSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error {
s.raw = append(s.raw, env)
return nil
}
func (s *recordingSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error {
s.unified = append(s.unified, env)
return nil
}
func (s *recordingSink) Close() error {
return nil
}
type testWriter struct {
t *testing.T
}
func (w testWriter) Write(p []byte) (int, error) {
w.t.Log(string(p))
return len(p), nil
}
func buildGBFrame(command byte, response byte, vin string, body []byte) []byte {
frame := []byte{'#', '#', command, response}
vinBytes := []byte(vin)
if len(vinBytes) < 17 {
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
}
frame = append(frame, vinBytes[:17]...)
frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
frame = append(frame, body...)
var bcc byte
for _, value := range frame[2:] {
bcc ^= value
}
return append(frame, bcc)
}

View File

@@ -0,0 +1,908 @@
package history
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
)
type Queryer interface {
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
}
type RawFrameQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
MessageID string
OrderBy string
IncludeTotal bool
DateFrom string
DateTo string
Limit int
Offset int
}
type RawFrameRow struct {
TS string `json:"ts"`
FrameID string `json:"frame_id"`
EventID string `json:"event_id"`
MessageID int64 `json:"message_id"`
MessageIDHex string `json:"message_id_hex"`
EventTime string `json:"event_time"`
ReceivedAt string `json:"received_at"`
RawSizeBytes int64 `json:"raw_size_bytes"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
ParsedJSON string `json:"parsed_json,omitempty"`
FieldsJSON string `json:"fields_json,omitempty"`
ParseStatus string `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
SourceEndpoint string `json:"source_endpoint"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type LocationQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
DateFrom string
DateTo string
Limit int
Offset int
}
type LocationRow struct {
TS string `json:"ts"`
EventID string `json:"event_id"`
FrameID string `json:"frame_id"`
ReceivedAt string `json:"received_at"`
Longitude float64 `json:"longitude"`
Latitude float64 `json:"latitude"`
AltitudeM *float64 `json:"altitude_m,omitempty"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
DirectionDeg *int64 `json:"direction_deg,omitempty"`
AlarmFlag *int64 `json:"alarm_flag,omitempty"`
StatusFlag *int64 `json:"status_flag,omitempty"`
TotalMileageKM *float64 `json:"total_mileage_km,omitempty"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type MileagePointQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
DateFrom string
DateTo string
Limit int
Offset int
}
type MileagePointRow struct {
TS string `json:"ts"`
EventID string `json:"event_id"`
FrameID string `json:"frame_id"`
ReceivedAt string `json:"received_at"`
TotalMileageKM float64 `json:"total_mileage_km"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Protocol string `json:"protocol"`
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
type RawFrameRepository struct {
db Queryer
database string
}
func NewRawFrameRepository(db Queryer, database string) *RawFrameRepository {
if db == nil {
panic("raw frame query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &RawFrameRepository{db: db, database: database}
}
type LocationRepository struct {
db Queryer
database string
}
type MileagePointRepository struct {
db Queryer
database string
}
func NewLocationRepository(db Queryer, database string) *LocationRepository {
if db == nil {
panic("location query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &LocationRepository{db: db, database: database}
}
func NewMileagePointRepository(db Queryer, database string) *MileagePointRepository {
if db == nil {
panic("mileage point query db must not be nil")
}
database = strings.TrimSpace(database)
if database != "" && !safeIdentifier(database) {
database = ""
}
return &MileagePointRepository{db: db, database: database}
}
func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) {
query = normalizeRawFrameQuery(query)
sqlText, args := buildRawFrameSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]RawFrameRow, 0)
for rows.Next() {
var row RawFrameRow
var ts scanDateTime
var eventTime scanDateTime
var receivedAt scanDateTime
if err := rows.Scan(
&ts,
&row.FrameID,
&row.EventID,
&row.MessageID,
&eventTime,
&receivedAt,
&row.RawSizeBytes,
&row.RawHex,
&row.RawText,
&row.ParsedJSON,
&row.FieldsJSON,
&row.ParseStatus,
&row.ParseError,
&row.SourceEndpoint,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.EventTime = eventTime.String
row.ReceivedAt = receivedAt.String
row.MessageIDHex = fmt.Sprintf("0x%04X", row.MessageID)
out = append(out, row)
}
return out, rows.Err()
}
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
query = normalizeRawFrameQuery(query)
sqlText, args := buildRawFrameCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]LocationRow, error) {
query = normalizeLocationQuery(query)
sqlText, args := buildLocationSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]LocationRow, 0)
for rows.Next() {
var row LocationRow
var ts scanDateTime
var receivedAt scanDateTime
var altitude sql.NullFloat64
var speed sql.NullFloat64
var direction sql.NullInt64
var alarm sql.NullInt64
var status sql.NullInt64
var mileage sql.NullFloat64
if err := rows.Scan(
&ts,
&row.EventID,
&row.FrameID,
&receivedAt,
&row.Longitude,
&row.Latitude,
&altitude,
&speed,
&direction,
&alarm,
&status,
&mileage,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.ReceivedAt = receivedAt.String
row.AltitudeM = nullableFloat(altitude)
row.SpeedKMH = nullableFloat(speed)
row.DirectionDeg = nullableInt(direction)
row.AlarmFlag = nullableInt(alarm)
row.StatusFlag = nullableInt(status)
row.TotalMileageKM = nullableFloat(mileage)
out = append(out, row)
}
return out, rows.Err()
}
func (r *LocationRepository) Count(ctx context.Context, query LocationQuery) (int64, error) {
query = normalizeLocationQuery(query)
sqlText, args := buildLocationCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func (r *MileagePointRepository) Query(ctx context.Context, query MileagePointQuery) ([]MileagePointRow, error) {
query = normalizeMileagePointQuery(query)
sqlText, args := buildMileagePointSQL(r.tableName(), query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]MileagePointRow, 0)
for rows.Next() {
var row MileagePointRow
var ts scanDateTime
var receivedAt scanDateTime
var speed sql.NullFloat64
var longitude sql.NullFloat64
var latitude sql.NullFloat64
if err := rows.Scan(
&ts,
&row.EventID,
&row.FrameID,
&receivedAt,
&row.TotalMileageKM,
&speed,
&longitude,
&latitude,
&row.Protocol,
&row.VehicleKey,
&row.VIN,
&row.Phone,
&row.DeviceID,
); err != nil {
return nil, err
}
row.TS = ts.String
row.ReceivedAt = receivedAt.String
row.SpeedKMH = nullableFloat(speed)
row.Longitude = nullableFloat(longitude)
row.Latitude = nullableFloat(latitude)
out = append(out, row)
}
return out, rows.Err()
}
func (r *MileagePointRepository) Count(ctx context.Context, query MileagePointQuery) (int64, error) {
query = normalizeMileagePointQuery(query)
sqlText, args := buildMileagePointCountSQL(r.tableName(), query)
return countRows(ctx, r.db, sqlText, args...)
}
func countRows(ctx context.Context, db Queryer, sqlText string, args ...any) (int64, error) {
rows, err := db.QueryContext(ctx, sqlText, args...)
if err != nil {
return 0, err
}
defer rows.Close()
var total int64
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, rows.Err()
}
func (r *RawFrameRepository) tableName() string {
if r.database == "" {
return "raw_frames"
}
return r.database + ".raw_frames"
}
func (r *LocationRepository) tableName() string {
if r.database == "" {
return "vehicle_locations"
}
return r.database + ".vehicle_locations"
}
func (r *MileagePointRepository) tableName() string {
if r.database == "" {
return "vehicle_mileage_points"
}
return r.database + ".vehicle_mileage_points"
}
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.MessageID = strings.TrimSpace(query.MessageID)
query.OrderBy = normalizeRawFrameOrderBy(query.OrderBy)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func normalizeLocationQuery(query LocationQuery) LocationQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func normalizeMileagePointQuery(query MileagePointQuery) MileagePointQuery {
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
if query.Limit <= 0 {
query.Limit = 20
}
return query
}
func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
where := rawFrameWhere(query)
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY " + rawFrameOrderColumn(query.OrderBy) + " DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := rawFrameWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
where := locationWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildLocationCountSQL(table string, query LocationQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := locationWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func buildMileagePointSQL(table string, query MileagePointQuery) (string, []any) {
where := mileagePointWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY ts DESC LIMIT " + strconv.Itoa(query.Limit) + " OFFSET " + strconv.Itoa(query.Offset)
return sqlText, nil
}
func buildMileagePointCountSQL(table string, query MileagePointQuery) (string, []any) {
sqlText := `SELECT COUNT(*) FROM ` + table
if where := mileagePointWhere(query); len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, nil
}
func rawFrameWhere(query RawFrameQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.MessageID != "" {
if parsed, ok := parseMessageID(query.MessageID); ok {
add("message_id = " + strconv.FormatInt(parsed, 10))
}
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
func normalizeRawFrameOrderBy(value string) string {
value = strings.ToLower(strings.TrimSpace(value))
switch value {
case "receivedat", "received_at":
return "receivedAt"
default:
return "ts"
}
}
func rawFrameOrderColumn(value string) string {
if normalizeRawFrameOrderBy(value) == "receivedAt" {
return "received_at"
}
return "ts"
}
func locationWhere(query LocationQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
func mileagePointWhere(query MileagePointQuery) []string {
var where []string
add := func(clause string) {
where = append(where, clause)
}
if query.Protocol != "" {
add("protocol = '" + quote(query.Protocol) + "'")
}
if query.VehicleKey != "" {
add("vehicle_key = '" + quote(query.VehicleKey) + "'")
}
if query.VIN != "" {
add("vin = '" + quote(query.VIN) + "'")
}
if query.Phone != "" {
add("phone = '" + quote(query.Phone) + "'")
}
if query.DeviceID != "" {
add("device_id = '" + quote(query.DeviceID) + "'")
}
if query.DateFrom != "" {
add("ts >= '" + quote(normalizeDateTimeLiteral(query.DateFrom)) + "'")
}
if query.DateTo != "" {
add("ts <= '" + quote(normalizeDateTimeLiteral(query.DateTo)) + "'")
}
return where
}
type RawFrameHandler struct {
repository *RawFrameRepository
}
type LocationHandler struct {
repository *LocationRepository
}
type MileagePointHandler struct {
repository *MileagePointRepository
}
func NewRawFrameHandler(repository *RawFrameRepository) *RawFrameHandler {
if repository == nil {
panic("raw frame repository must not be nil")
}
return &RawFrameHandler{repository: repository}
}
func NewLocationHandler(repository *LocationRepository) *LocationHandler {
if repository == nil {
panic("location repository must not be nil")
}
return &LocationHandler{repository: repository}
}
func NewMileagePointHandler(repository *MileagePointRepository) *MileagePointHandler {
if repository == nil {
panic("mileage point repository must not be nil")
}
return &MileagePointHandler{repository: repository}
}
func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/raw-frames" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseRawFrameQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
var total int64
if query.IncludeTotal {
total, err = h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
if !query.IncludeTotal {
total = int64(len(rows))
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/locations" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseLocationQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func (h *MileagePointHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeHistoryError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/history/mileage-points" {
writeHistoryError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseMileagePointQuery(r)
if err != nil {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return RawFrameQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return RawFrameQuery{}, err
}
query := RawFrameQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
MessageID: values.Get("messageId"),
OrderBy: values.Get("orderBy"),
IncludeTotal: values.Get("includeTotal") != "false",
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
if strings.TrimSpace(query.MessageID) != "" {
if _, ok := parseMessageID(query.MessageID); !ok {
return RawFrameQuery{}, errors.New("messageId must be decimal or hex like 0x0200")
}
}
return normalizeRawFrameQuery(query), nil
}
func parseMileagePointQuery(r *http.Request) (MileagePointQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return MileagePointQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return MileagePointQuery{}, err
}
query := MileagePointQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return MileagePointQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
return normalizeMileagePointQuery(query), nil
}
func parseLocationQuery(r *http.Request) (LocationQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 20, 1, 500, "limit")
if err != nil {
return LocationQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return LocationQuery{}, err
}
query := LocationQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
}
return normalizeLocationQuery(query), nil
}
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return fallback, nil
}
value, err := strconv.Atoi(raw)
if err != nil || value < min || value > max {
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
}
return value, nil
}
func parseMessageID(value string) (int64, bool) {
value = strings.TrimSpace(strings.ToLower(value))
if value == "" {
return 0, false
}
if strings.HasPrefix(value, "0x") {
parsed, err := strconv.ParseInt(strings.TrimPrefix(value, "0x"), 16, 32)
return parsed, err == nil
}
parsed, err := strconv.ParseInt(value, 10, 32)
return parsed, err == nil
}
func validDateTime(value string) bool {
value = normalizeDateTimeLiteral(value)
if value == "" {
return true
}
for _, layout := range []string{"2006-01-02", "2006-01-02 15:04:05", time.RFC3339} {
if _, err := time.Parse(layout, value); err == nil {
return true
}
}
return false
}
func normalizeDateTimeLiteral(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} {
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
}
}
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
}
return value
}
func safeIdentifier(value string) bool {
if value == "" {
return false
}
for _, r := range value {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
continue
}
return false
}
return true
}
type scanDateTime struct {
String string
}
func (s *scanDateTime) Scan(value any) error {
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
return nil
}
func formatSQLTime(value any, layout string) string {
switch typed := value.(type) {
case nil:
return ""
case time.Time:
return typed.Format(layout)
case []byte:
return strings.TrimSpace(string(typed))
case string:
return strings.TrimSpace(typed)
default:
return strings.TrimSpace(fmt.Sprint(typed))
}
}
func nullableFloat(value sql.NullFloat64) *float64 {
if !value.Valid {
return nil
}
return &value.Float64
}
func nullableInt(value sql.NullInt64) *int64 {
if !value.Valid {
return nil
}
return &value.Int64
}
func writeHistoryError(w http.ResponseWriter, status int, message string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
}

View File

@@ -0,0 +1,423 @@
package history
import (
"context"
"database/sql"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
)
func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
"go_frame", "event-1", 0x0200,
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
time.Date(2026, 7, 1, 23, 25, 37, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
64, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "222.66.200.68:29646", "JT808", "013079963379", "LKLG7C4E3NA774736", "013079963379", "9963379",
))
repository := NewRawFrameRepository(db, "lingniu_vehicle_ts")
rows, err := repository.Query(context.Background(), RawFrameQuery{
Protocol: "JT808",
VIN: "LKLG7C4E3NA774736",
DateFrom: "2026-07-01 00:00:00",
DateTo: "2026-07-01 23:59:59",
Limit: 20,
})
if err != nil {
t.Fatalf("Query() error = %v", err)
}
if len(rows) != 1 {
t.Fatalf("row count = %d", len(rows))
}
if rows[0].MessageID != 512 || rows[0].MessageIDHex != "0x0200" || rows[0].ParsedJSON == "" {
t.Fatalf("unexpected row: %#v", rows[0])
}
if rows[0].TS != "2026-07-01 23:25:36" {
t.Fatalf("timestamp = %q", rows[0].TS)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(38))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-01 22:28:25", "go_frame", "event-2", 2, "2026-07-01 22:28:25", "2026-07-01 22:28:25",
128, "2323", "", `{"command":"REALTIME"}`, `{"total_mileage_km":53490.9}`,
"OK", "", "8.134.95.166:53702", "GB32960", "LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"message_id_hex":"0x0002"`, `"total":38`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(11))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "go_frame", "event-3", 0x0200, "2026-07-02 00:18:22", "2026-07-02 00:22:43",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"total_mileage_km":8792.8}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"phone":"013307811350"`, `"total":11`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=NOT_EXISTS&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
t.Fatalf("expected empty items array, got: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestLocationHandlerReturnsLocationsByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(17))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
"JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
8792.8, 8.0, 121.07764, 30.585928,
"JT808", "JT808:013307811350", "", "013307811350", "",
))
handler := NewMileagePointHandler(NewMileagePointRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/mileage-points?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811350"`, `"total_mileage_km":8792.8`, `"speed_kmh":8`, `"total":19`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
}
func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) {
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?dateFrom=2026-07-01T00:00:00&dateTo=2026-07-02T00:00:00", nil)
query, err := parseRawFrameQuery(request)
if err != nil {
t.Fatalf("parseRawFrameQuery() error = %v", err)
}
if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" {
t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo)
}
}
func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) {
for raw, want := range map[string]string{
"2026-07-01T00:00:00": "2026-06-30 16:00:00",
"2026-07-01 00:00:00": "2026-06-30 16:00:00",
"2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00",
"2026-06-30T16:00:00Z": "2026-06-30 16:00:00",
} {
if got := normalizeDateTimeLiteral(raw); got != want {
t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want)
}
}
}
func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
for raw, want := range map[string]int64{
"512": 512,
"0x0200": 512,
"0X0100": 256,
} {
got, ok := parseMessageID(raw)
if !ok || got != want {
t.Fatalf("parseMessageID(%q) = %d,%v want %d,true", raw, got, ok, want)
}
}
}
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
DateFrom: "2026-07-02 00:00:00",
DateTo: "2026-07-02 23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"ts >= '2026-07-01 16:00:00'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildLocationSQL("lingniu_vehicle_ts.vehicle_locations", LocationQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
DateFrom: "2026-07-02 00:00:00",
DateTo: "2026-07-02 23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"ts >= '2026-07-01 16:00:00'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
Protocol: "JT808",
VehicleKey: "JT808:013307811350",
VIN: "VIN'1",
MessageID: "0x0200",
DateFrom: "2026-07-01T00:00:00",
DateTo: "2026-07-01T23:59:59",
Limit: 20,
Offset: 5,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
for _, want := range []string{
"protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'",
"vin = 'VIN''1'",
"message_id = 512",
"ts >= '2026-06-30 16:00:00'",
"ts <= '2026-07-01 15:59:59'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
t.Fatalf("sql missing %s: %s", want, sqlText)
}
}
}
func TestBuildRawFrameSQLCanOrderByReceivedAt(t *testing.T) {
sqlText, args := buildRawFrameSQL("lingniu_vehicle_ts.raw_frames", RawFrameQuery{
Protocol: "JT808",
OrderBy: "receivedAt",
Limit: 1,
})
if len(args) != 0 {
t.Fatalf("expected no query args for TDengine, got %#v", args)
}
if !strings.Contains(sqlText, "ORDER BY received_at DESC LIMIT 1 OFFSET 0") {
t.Fatalf("sql should order by received_at: %s", sqlText)
}
}

View File

@@ -0,0 +1,71 @@
package history
const DefaultDatabase = "lingniu_vehicle_ts"
func SchemaStatements(database string) []string {
if database == "" {
database = DefaultDatabase
}
return []string{
"CREATE DATABASE IF NOT EXISTS " + database + " KEEP 7300 DURATION 10 BUFFER 256",
"USE " + database,
`CREATE STABLE IF NOT EXISTS raw_frames (
ts TIMESTAMP,
frame_id NCHAR(64),
event_id NCHAR(64),
message_id INT,
event_time TIMESTAMP,
received_at TIMESTAMP,
raw_size_bytes INT,
raw_hex BINARY(16374),
raw_text BINARY(16374),
parsed_json BINARY(16374),
fields_json BINARY(4096),
parse_status NCHAR(16),
parse_error BINARY(1024),
source_endpoint NCHAR(128)
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
)`,
`CREATE STABLE IF NOT EXISTS vehicle_locations (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
longitude DOUBLE,
latitude DOUBLE,
altitude_m DOUBLE,
speed_kmh DOUBLE,
direction_deg INT,
alarm_flag BIGINT,
status_flag BIGINT,
total_mileage_km DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
)`,
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
total_mileage_km DOUBLE,
speed_kmh DOUBLE,
longitude DOUBLE,
latitude DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
)`,
}
}

View File

@@ -0,0 +1,339 @@
package history
import (
"context"
"crypto/sha1"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Execer interface {
ExecContext(context.Context, string, ...any) (sql.Result, error)
}
type Writer struct {
exec Execer
cache tableCache
}
type tableCache struct {
mu sync.Mutex
seen map[string]struct{}
}
func NewWriter(exec Execer) *Writer {
if exec == nil {
panic("history execer must not be nil")
}
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
}
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
for _, statement := range SchemaStatements(database) {
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
return err
}
}
return nil
}
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
if err := w.AppendRawFrame(ctx, env); err != nil {
return err
}
if err := w.AppendLocation(ctx, env); err != nil {
return err
}
return w.AppendMileagePoint(ctx, env)
}
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
table := tableName("raw", env)
if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
return err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
VALUES (%s)`, table, joinLiterals(rawValues(env))))
return err
}
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
longitude, okLon := floatField(env, envelope.FieldLongitude)
latitude, okLat := floatField(env, envelope.FieldLatitude)
if !okLon || !okLat {
return nil
}
table := tableName("loc", env)
if err := w.ensureChild(ctx, table, "vehicle_locations", env); err != nil {
return err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh,
direction_deg, alarm_flag, status_flag, total_mileage_km)
VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
return err
}
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
if !ok {
return nil
}
table := tableName("mil", env)
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
return err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage))))
return err
}
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
key := stable + "." + table
w.cache.mu.Lock()
_, ok := w.cache.seen[key]
w.cache.mu.Unlock()
if ok {
return nil
}
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
table, stable,
quote(string(env.Protocol)),
quote(env.VehicleKey()),
quote(env.VIN),
quote(env.Phone),
quote(env.DeviceID))
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
return err
}
w.cache.mu.Lock()
w.cache.seen[key] = struct{}{}
w.cache.mu.Unlock()
return nil
}
func rawValues(env envelope.FrameEnvelope) []any {
received := millis(env.ReceivedAtMS)
eventTime := millis(env.EventTimeMS)
return []any{
received,
frameID(env),
env.StableEventID(),
messageIDInt(env.MessageID),
eventTime,
received,
rawSizeBytes(env),
env.RawHex,
env.RawText,
jsonString(env.Parsed),
jsonString(env.Fields),
string(env.ParseStatus),
env.ParseError,
env.SourceEndpoint,
}
}
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
received := millis(env.ReceivedAtMS)
return []any{
eventTimeOrReceived(env),
env.StableEventID(),
frameID(env),
received,
longitude,
latitude,
floatFieldOrNil(env, "altitude_m"),
floatFieldOrNil(env, envelope.FieldSpeedKMH),
intFieldOrNil(env, "direction_deg"),
intFieldOrNil(env, "alarm_flag"),
intFieldOrNil(env, "status_flag"),
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
}
}
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
received := millis(env.ReceivedAtMS)
return []any{
eventTimeOrReceived(env),
env.StableEventID(),
frameID(env),
received,
totalMileage,
floatFieldOrNil(env, envelope.FieldSpeedKMH),
floatFieldOrNil(env, envelope.FieldLongitude),
floatFieldOrNil(env, envelope.FieldLatitude),
}
}
func frameID(env envelope.FrameEnvelope) string {
return "go_" + env.StableEventID()
}
func tableName(prefix string, env envelope.FrameEnvelope) string {
return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
}
func hash16(value string) string {
sum := sha1.Sum([]byte(value))
return hex.EncodeToString(sum[:8])
}
func messageIDInt(value string) int64 {
value = strings.TrimSpace(strings.ToLower(value))
value = strings.TrimPrefix(value, "0x")
parsed, err := strconv.ParseInt(value, 16, 32)
if err == nil {
return parsed
}
parsed, _ = strconv.ParseInt(value, 10, 32)
return parsed
}
func rawSizeBytes(env envelope.FrameEnvelope) int {
rawHex := strings.TrimSpace(env.RawHex)
if len(rawHex)%2 != 0 {
return len(rawHex) / 2
}
if rawHex != "" {
return len(rawHex) / 2
}
return len([]byte(env.RawText))
}
func jsonString(value any) string {
data, err := json.Marshal(value)
if err != nil {
return "{}"
}
return string(data)
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
if env.Fields == nil {
return 0, false
}
value, ok := env.Fields[key]
if !ok || value == nil {
return 0, false
}
switch typed := value.(type) {
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int64:
return float64(typed), true
case uint16:
return float64(typed), true
case uint32:
return float64(typed), true
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return parsed, err == nil
default:
return 0, false
}
}
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
value, ok := floatField(env, key)
if !ok {
return nil
}
return value
}
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
if env.Fields == nil {
return nil
}
value, ok := env.Fields[key]
if !ok || value == nil {
return nil
}
switch typed := value.(type) {
case int:
return typed
case int64:
return typed
case uint16:
return int64(typed)
case uint32:
return int64(typed)
case float64:
return int64(typed)
case string:
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
if err == nil {
return parsed
}
}
return nil
}
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
if env.EventTimeMS > 0 {
return millis(env.EventTimeMS)
}
return millis(env.ReceivedAtMS)
}
func millis(value int64) time.Time {
if value <= 0 {
return time.UnixMilli(time.Now().UnixMilli()).UTC()
}
return time.UnixMilli(value).UTC()
}
func quote(value string) string {
return strings.ReplaceAll(value, "'", "''")
}
func joinLiterals(values []any) string {
out := make([]string, 0, len(values))
for _, value := range values {
out = append(out, literal(value))
}
return strings.Join(out, ", ")
}
func literal(value any) string {
if value == nil {
return "NULL"
}
switch typed := value.(type) {
case time.Time:
return "'" + typed.UTC().Format("2006-01-02 15:04:05.000") + "'"
case string:
return "'" + quote(typed) + "'"
case envelope.ParseStatus:
return "'" + quote(string(typed)) + "'"
case int:
return strconv.FormatInt(int64(typed), 10)
case int64:
return strconv.FormatInt(typed, 10)
case uint16:
return strconv.FormatUint(uint64(typed), 10)
case uint32:
return strconv.FormatUint(uint64(typed), 10)
case float64:
return strconv.FormatFloat(typed, 'f', -1, 64)
case float32:
return strconv.FormatFloat(float64(typed), 'f', -1, 64)
default:
return "'" + quote(fmt.Sprint(typed)) + "'"
}
}

View File

@@ -0,0 +1,146 @@
package history
import (
"context"
"database/sql"
"strings"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
statements := strings.Join(SchemaStatements("test_ts"), "\n")
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} {
if !strings.Contains(statements, want) {
t.Fatalf("schema missing %q:\n%s", want, statements)
}
}
}
func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
}
if got := countSQL(exec.calls, "USING raw_frames"); got != 1 {
t.Fatalf("raw child create count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
t.Fatalf("location child create count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 {
t.Fatalf("mileage child create count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
t.Fatalf("location insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 {
t.Fatalf("mileage insert count = %d", got)
}
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {
t.Fatalf("raw insert should quote string literals: %s", rawInsert)
}
if strings.Contains(rawInsert, "VALUES (?,") {
t.Fatalf("raw insert should not use placeholders for tdengine websocket plain insert: %s", rawInsert)
}
}
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
env.Fields = map[string]any{}
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
t.Fatalf("location insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
t.Fatalf("mileage insert count = %d", got)
}
}
func TestRawSizeBytesUsesRawTextWhenHexIsEmpty(t *testing.T) {
env := envelope.FrameEnvelope{RawText: `{"code":"0F80","data":{"speed":12}}`}
if got, want := rawSizeBytes(env), len([]byte(env.RawText)); got != want {
t.Fatalf("raw text size = %d, want %d", got, want)
}
env.RawHex = "7E0200"
if got, want := rawSizeBytes(env), 3; got != want {
t.Fatalf("raw hex size = %d, want %d", got, want)
}
}
func sampleEnvelope() envelope.FrameEnvelope {
return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
Sequence: 1,
VIN: "LNBVIN00000000001",
Phone: "013307795425",
SourceEndpoint: "127.0.0.1:18080",
EventTimeMS: 1782745114000,
ReceivedAtMS: 1782745114999,
RawHex: "7E0200",
Parsed: map[string]any{"message": "location"},
Fields: map[string]any{
envelope.FieldLongitude: 121.069881,
envelope.FieldLatitude: 30.590151,
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10241.2,
"direction_deg": uint16(79),
"alarm_flag": uint32(0),
"status_flag": uint32(72),
},
ParseStatus: envelope.ParseOK,
}
}
func countSQL(calls []execCall, pattern string) int {
var count int
for _, call := range calls {
if strings.Contains(call.query, pattern) {
count++
}
}
return count
}
func findSQL(calls []execCall, pattern string) string {
for _, call := range calls {
if strings.Contains(call.query, pattern) {
return call.query
}
}
return ""
}
type execCall struct {
query string
args []any
}
type recordingExec struct {
calls []execCall
}
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
e.calls = append(e.calls, execCall{query: query, args: args})
return nil, nil
}

View File

@@ -0,0 +1,237 @@
package identity
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Resolver interface {
Resolve(context.Context, envelope.FrameEnvelope) (envelope.FrameEnvelope, error)
}
type NoopResolver struct{}
func (NoopResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
return env, nil
}
type MySQLResolver struct {
db *sql.DB
table string
}
func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver {
if db == nil {
panic("identity db must not be nil")
}
table = strings.TrimSpace(table)
if table == "" || !safeIdentifier(table) {
table = "vehicle_identity_binding"
}
return &MySQLResolver{db: db, table: table}
}
func safeIdentifier(value string) bool {
if value == "" {
return false
}
for _, r := range value {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
continue
}
return false
}
return true
}
func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
if strings.TrimSpace(env.VIN) != "" {
if err := r.trackJT808(ctx, env); err != nil {
return env, err
}
return env, nil
}
candidates := CandidateKeys(env)
for _, candidate := range candidates {
vin, err := r.lookup(ctx, candidate.Column, candidate.Value)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
continue
}
return env, err
}
if strings.TrimSpace(vin) != "" {
env.VIN = strings.TrimSpace(vin)
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
env.Parsed["identity"] = map[string]any{
"resolved": true,
"source": candidate.Column,
"value": candidate.Value,
}
env.EventID = env.StableEventID()
if err := r.trackJT808(ctx, env); err != nil {
return env, err
}
return env, nil
}
}
if err := r.trackJT808(ctx, env); err != nil {
return env, err
}
return env, nil
}
func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) {
query := "SELECT vin FROM " + r.table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' ORDER BY updated_at DESC LIMIT 1"
var vin string
err := r.db.QueryRowContext(ctx, query, value).Scan(&vin)
return vin, err
}
func (r *MySQLResolver) trackJT808(ctx context.Context, env envelope.FrameEnvelope) error {
if env.Protocol != envelope.ProtocolJT808 || strings.TrimSpace(env.Phone) == "" || strings.TrimSpace(env.MessageID) == "" {
return nil
}
registration := mapValue(env.Parsed, "registration")
authentication := mapValue(env.Parsed, "authentication")
deviceID := firstNonEmpty(env.DeviceID, textValue(registration, "device_id"))
plate := firstNonEmpty(env.Plate, textValue(registration, "plate"))
vin := strings.TrimSpace(env.VIN)
if vin == "" {
vin = "unknown"
}
firstRegisteredAt := nilTime()
latestRegisteredAt := nilTime()
if env.MessageID == "0x0100" {
firstRegisteredAt = "CURRENT_TIMESTAMP"
latestRegisteredAt = "CURRENT_TIMESTAMP"
}
latestAuthenticatedAt := nilTime()
if env.MessageID == "0x0102" {
latestAuthenticatedAt = "CURRENT_TIMESTAMP"
}
query := `INSERT INTO jt808_registration
(phone, device_id, plate, vin, province, city, manufacturer, device_type, plate_color,
auth_token, auth_imei, auth_software_version, source_endpoint,
first_registered_at, latest_registered_at, latest_authenticated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ` + firstRegisteredAt + `, ` + latestRegisteredAt + `, ` + latestAuthenticatedAt + `)
ON DUPLICATE KEY UPDATE
device_id = IF(VALUES(device_id) <> '', VALUES(device_id), device_id),
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
vin = IF(VALUES(vin) <> 'unknown', VALUES(vin), vin),
province = IF(VALUES(province) <> '', VALUES(province), province),
city = IF(VALUES(city) <> '', VALUES(city), city),
manufacturer = IF(VALUES(manufacturer) <> '', VALUES(manufacturer), manufacturer),
device_type = IF(VALUES(device_type) <> '', VALUES(device_type), device_type),
plate_color = IF(VALUES(plate_color) <> '', VALUES(plate_color), plate_color),
auth_token = IF(VALUES(auth_token) <> '', VALUES(auth_token), auth_token),
auth_imei = IF(VALUES(auth_imei) <> '', VALUES(auth_imei), auth_imei),
auth_software_version = IF(VALUES(auth_software_version) <> '', VALUES(auth_software_version), auth_software_version),
source_endpoint = IF(VALUES(source_endpoint) <> '', VALUES(source_endpoint), source_endpoint),
latest_seen_at = CURRENT_TIMESTAMP,
first_registered_at = COALESCE(first_registered_at, VALUES(first_registered_at)),
latest_registered_at = COALESCE(VALUES(latest_registered_at), latest_registered_at),
latest_authenticated_at = COALESCE(VALUES(latest_authenticated_at), latest_authenticated_at)`
_, err := r.db.ExecContext(ctx, query,
env.Phone,
deviceID,
plate,
vin,
textValue(registration, "province"),
textValue(registration, "city"),
textValue(registration, "manufacturer"),
textValue(registration, "device_type"),
textValue(registration, "plate_color"),
textValue(authentication, "token"),
textValue(authentication, "imei"),
textValue(authentication, "software_version"),
env.SourceEndpoint,
)
if err != nil {
return err
}
if vin == "unknown" {
return nil
}
_, err = r.db.ExecContext(ctx, `UPDATE IGNORE `+r.table+`
SET
phone = CASE WHEN (phone IS NULL OR phone = '') AND ? <> '' THEN ? ELSE phone END,
device_id = CASE WHEN (device_id IS NULL OR device_id = '') AND ? <> '' THEN ? ELSE device_id END,
plate = CASE WHEN (plate IS NULL OR plate = '') AND ? <> '' THEN ? ELSE plate END
WHERE vin = ?`, env.Phone, env.Phone, deviceID, deviceID, plate, plate, vin)
return err
}
type CandidateKey struct {
Column string
Value string
}
func CandidateKeys(env envelope.FrameEnvelope) []CandidateKey {
var out []CandidateKey
add := func(column string, value string) {
value = strings.TrimSpace(value)
if value == "" || strings.EqualFold(value, "unknown") {
return
}
for _, existing := range out {
if existing.Column == column && existing.Value == value {
return
}
}
out = append(out, CandidateKey{Column: column, Value: value})
}
add("phone", env.Phone)
if trimmed := strings.TrimLeft(strings.TrimSpace(env.Phone), "0"); trimmed != "" {
add("phone", trimmed)
}
add("device_id", env.DeviceID)
add("plate", env.Plate)
add("vin", env.VehicleKeyHint)
return out
}
func mapValue(values map[string]any, key string) map[string]any {
if values == nil {
return nil
}
nested, _ := values[key].(map[string]any)
return nested
}
func textValue(values map[string]any, key string) string {
if values == nil {
return ""
}
value, ok := values[key]
if !ok || value == nil {
return ""
}
switch typed := value.(type) {
case string:
return strings.TrimSpace(typed)
default:
return strings.TrimSpace(fmt.Sprint(typed))
}
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
}
return ""
}
func nilTime() string {
return "NULL"
}

View File

@@ -0,0 +1,171 @@
package identity
import (
"context"
"database/sql"
"testing"
"github.com/DATA-DOG/go-sqlmock"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestCandidateKeysPrioritizesPhoneDevicePlate(t *testing.T) {
keys := CandidateKeys(envelope.FrameEnvelope{
Phone: "013307795425",
DeviceID: "D1",
Plate: "豫A12345",
VehicleKeyHint: "D1",
})
got := []string{}
for _, key := range keys {
got = append(got, key.Column+"="+key.Value)
}
want := []string{"phone=013307795425", "phone=13307795425", "device_id=D1", "plate=豫A12345", "vin=D1"}
if len(got) != len(want) {
t.Fatalf("candidate count = %d got=%#v", len(got), got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
}
}
}
func TestCandidateKeysAddsTrimmedPhoneVariant(t *testing.T) {
keys := CandidateKeys(envelope.FrameEnvelope{
Phone: "013079963379",
})
got := []string{}
for _, key := range keys {
got = append(got, key.Column+"="+key.Value)
}
want := []string{"phone=013079963379", "phone=13079963379"}
if len(got) != len(want) {
t.Fatalf("candidate count = %d got=%#v", len(got), got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("candidate[%d] = %q, want %q", i, got[i], want[i])
}
}
}
func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
db, mock := newMockDB(t)
defer db.Close()
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("013307795425").
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001"))
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307795425",
Parsed: map[string]any{},
})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if env.VIN != "LNBVIN00000000001" {
t.Fatalf("vin = %q", env.VIN)
}
identity, ok := env.Parsed["identity"].(map[string]any)
if !ok || identity["source"] != "phone" {
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMySQLResolverFallsBackToDeviceID(t *testing.T) {
db, mock := newMockDB(t)
defer db.Close()
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("013307795425").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("13307795425").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
WithArgs("D1").
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000002"))
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307795425",
DeviceID: "D1",
})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if env.VIN != "LNBVIN00000000002" {
t.Fatalf("vin = %q", env.VIN)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMySQLResolverTracksJT808RegistrationAndBackfillsBinding(t *testing.T) {
db, mock := newMockDB(t)
defer db.Close()
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("013079963379").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("13079963379").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE device_id = \\?").
WithArgs("DEV0001").
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
WithArgs("TEST123").
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LKLG7C4E3NA774736"))
mock.ExpectExec("INSERT INTO jt808_registration").
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE IGNORE vehicle_identity_binding").
WithArgs("013079963379", "013079963379", "DEV0001", "DEV0001", "TEST123", "TEST123", "LKLG7C4E3NA774736").
WillReturnResult(sqlmock.NewResult(0, 1))
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0100",
Phone: "013079963379",
DeviceID: "DEV0001",
Plate: "TEST123",
SourceEndpoint: "115.231.168.135:43625",
Parsed: map[string]any{
"registration": map[string]any{
"province": uint16(16),
"city": uint16(32),
"manufacturer": "YUTNG",
"device_type": "ZK6105CHEVNPG4",
"device_id": "DEV0001",
"plate_color": uint8(2),
"plate": "TEST123",
},
},
})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if env.VIN != "LKLG7C4E3NA774736" {
t.Fatalf("vin = %q", env.VIN)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) {
t.Helper()
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
return db, mock
}

View File

@@ -0,0 +1,13 @@
package observability
import (
"log/slog"
"os"
)
// NewLogger returns a JSON logger with a stable service field so ECS logs from
// different Go processes can be filtered without relying on container names.
func NewLogger(service string) *slog.Logger {
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
return slog.New(handler).With("service", service)
}

View File

@@ -0,0 +1,840 @@
package gb32960
import (
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var (
ErrFrameTooShort = errors.New("gb32960 frame too short")
ErrBadStartSymbol = errors.New("gb32960 bad start symbol")
ErrBodyLength = errors.New("gb32960 body length mismatch")
ErrBCC = errors.New("gb32960 bcc mismatch")
)
const (
minFrameLen = 25
headerLen = 24
)
// ExtractFrames splits GB/T 32960 TCP streams using the protocol length field.
// Returned frames include the start symbols and trailing BCC byte.
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
offset := 0
for {
start := indexStart(stream[offset:])
if start < 0 {
return frames, nil, nil
}
offset += start
remaining := stream[offset:]
if len(remaining) < headerLen {
return frames, append([]byte(nil), remaining...), nil
}
bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
frameLen := frameLength(remaining, bodyLen)
if len(remaining) < frameLen {
return frames, append([]byte(nil), remaining...), nil
}
frames = append(frames, append([]byte(nil), remaining[:frameLen]...))
offset += frameLen
if offset >= len(stream) {
return frames, nil, nil
}
}
}
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
if len(raw) < minFrameLen {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
}
version, ok := version(raw[0], raw[1])
if !ok {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
}
bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
actualBodyLen := len(raw) - headerLen - 1
if len(raw) != headerLen+bodyLen+1 && !isExtendedPlatformLogin(raw[2], bodyLen, actualBodyLen) {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
}
if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrBCC, got, want)
}
command := raw[2]
responseFlag := raw[3]
vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
body := raw[headerLen : headerLen+actualBodyLen]
parsed := map[string]any{
"header": map[string]any{
"version": version,
"command": fmt.Sprintf("0x%02X", command),
"response_flag": fmt.Sprintf("0x%02X", responseFlag),
"vin": vin,
"encrypt": raw[21],
"body_length": bodyLen,
"actual_body_length": actualBodyLen,
},
}
fields := map[string]any{}
eventTimeMS := receivedAtMS
if command == 0x02 || command == 0x03 {
eventTime, units := parseDataBody(version, body, fields)
if !eventTime.IsZero() {
eventTimeMS = eventTime.UnixMilli()
fields["device_time"] = eventTime.Format(time.RFC3339)
parsed["device_time"] = eventTime.Format(time.RFC3339)
}
parsed["data_units"] = units
} else if command == 0x05 {
login := parsePlatformLogin(body)
parsed["platform_login"] = login
if deviceTime, ok := login["login_time"].(string); ok && deviceTime != "" {
fields["device_time"] = deviceTime
parsed["device_time"] = deviceTime
}
if account, ok := login["username"].(string); ok && account != "" {
fields["platform_account"] = account
}
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
MessageID: fmt.Sprintf("0x%02X", command),
VIN: vin,
SourceEndpoint: sourceEndpoint,
EventTimeMS: eventTimeMS,
ReceivedAtMS: receivedAtMS,
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
Parsed: parsed,
Fields: fields,
ParseStatus: envelope.ParseOK,
}
env.EventID = env.StableEventID()
return env, nil
}
func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
if len(body) < 6 {
return time.Time{}, nil
}
eventTime := parseGBTime(body[:6])
cursor := 6
var units []map[string]any
for cursor < len(body) {
unitType := body[cursor]
cursor++
switch unitType {
case 0x01:
size := 20
if version == "V2025" {
size = 18
}
if len(body[cursor:]) < size {
units = append(units, map[string]any{"type": "0x01", "error": "truncated"})
return eventTime, units
}
unit := parseVehicleData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x01", "name": "vehicle", "value": unit})
fields["vehicle_status"] = unit["vehicle_status"]
fields["charge_status"] = unit["charge_status"]
fields["running_mode"] = unit["running_mode"]
fields[envelope.FieldSpeedKMH] = unit["speed_kmh"]
fields[envelope.FieldTotalMileageKM] = unit["total_mileage_km"]
fields[envelope.FieldSOCPercent] = unit["soc_percent"]
cursor += size
case 0x05:
if len(body[cursor:]) < 9 {
units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
return eventTime, units
}
unit := parsePositionData(body[cursor : cursor+9])
units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
fields["position_status"] = unit["position_status"]
fields[envelope.FieldLongitude] = unit["longitude"]
fields[envelope.FieldLatitude] = unit["latitude"]
cursor += 9
case 0x02:
if len(body[cursor:]) < 1 {
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
return eventTime, units
}
size := 1 + int(body[cursor])*12
if len(body[cursor:]) < size {
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
return eventTime, units
}
unit := parseDriveMotorData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x02", "name": "drive_motor", "value": unit})
cursor += size
case 0x03:
if len(body[cursor:]) < 8 {
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
return eventTime, units
}
probeCount := int(binary.BigEndian.Uint16(body[cursor+6 : cursor+8]))
size := 8 + probeCount + 10
if len(body[cursor:]) < size {
units = append(units, map[string]any{"type": "0x03", "error": "truncated"})
return eventTime, units
}
unit := parseFuelCellData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x03", "name": "fuel_cell", "value": unit})
fields["fuel_cell_hydrogen_consumption_kg_per_100km"] = unit["hydrogen_consumption_kg_per_100km"]
fields["fuel_cell_voltage_v"] = unit["fuel_cell_voltage_v"]
fields["fuel_cell_current_a"] = unit["fuel_cell_current_a"]
cursor += size
case 0x04:
if len(body[cursor:]) < 5 {
units = append(units, map[string]any{"type": "0x04", "error": "truncated"})
return eventTime, units
}
unit := parseEngineData(body[cursor : cursor+5])
units = append(units, map[string]any{"type": "0x04", "name": "engine", "value": unit})
cursor += 5
case 0x06:
if len(body[cursor:]) < 14 {
units = append(units, map[string]any{"type": "0x06", "error": "truncated"})
return eventTime, units
}
unit := parseExtremeData(body[cursor : cursor+14])
units = append(units, map[string]any{"type": "0x06", "name": "extreme", "value": unit})
cursor += 14
case 0x07:
size, ok := alarmDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x07", "error": "truncated"})
return eventTime, units
}
unit := parseAlarmData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x07", "name": "alarm", "value": unit})
cursor += size
case 0x08:
size, ok := voltageDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x08", "error": "truncated"})
return eventTime, units
}
unit := parseVoltageData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x08", "name": "voltage", "value": unit})
cursor += size
case 0x09:
size, ok := temperatureDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x09", "error": "truncated"})
return eventTime, units
}
unit := parseTemperatureData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x09", "name": "temperature", "value": unit})
cursor += size
case 0x30:
size, ok := gdFCStackDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x30", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCStackData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x30", "name": "gd_fc_stack", "value": unit})
if summaries, ok := unit["summaries"].([]map[string]any); ok && len(summaries) > 0 {
fields["gd_fc_stack_hydrogen_inlet_pressure_kpa"] = summaries[0]["hydrogen_inlet_pressure_kpa"]
fields["gd_fc_stack_water_outlet_temp_c"] = summaries[0]["stack_water_outlet_temp_c"]
fields["gd_fc_stack_cell_count"] = summaries[0]["cell_count"]
fields["gd_fc_stack_frame_cell_count"] = summaries[0]["frame_cell_count"]
}
cursor += size
case 0x31:
size, ok := gdFCAuxiliaryDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x31", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCAuxiliaryData(body[cursor : cursor+size])
units = append(units, map[string]any{"type": "0x31", "name": "gd_fc_auxiliary", "value": unit})
cursor += size
case 0x32:
if len(body[cursor:]) < 9 {
units = append(units, map[string]any{"type": "0x32", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCDcDcData(body[cursor : cursor+9])
units = append(units, map[string]any{"type": "0x32", "name": "gd_fc_dcdc", "value": unit})
fields["gd_fc_dcdc_controller_temp_c"] = unit["controller_temp_c"]
cursor += 9
case 0x33:
if len(body[cursor:]) < 5 {
units = append(units, map[string]any{"type": "0x33", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCAirConditionerData(body[cursor : cursor+5])
units = append(units, map[string]any{"type": "0x33", "name": "gd_fc_air_conditioner", "value": unit})
cursor += 5
case 0x34:
if len(body[cursor:]) < 7 {
units = append(units, map[string]any{"type": "0x34", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCVehicleInfoData(body[cursor : cursor+7])
units = append(units, map[string]any{"type": "0x34", "name": "gd_fc_vehicle_info", "value": unit})
fields["gd_fc_vehicle_hydrogen_mass_kg"] = unit["hydrogen_mass_kg"]
cursor += 7
case 0x80:
if len(body[cursor:]) < 11 {
units = append(units, map[string]any{"type": "0x80", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCDemoExtensionData(body[cursor : cursor+11])
units = append(units, map[string]any{"type": "0x80", "name": "gd_fc_demo_extension", "value": unit})
fields["gd_fc_demo_stack_temp_c"] = unit["stack_temp_c"]
cursor += 11
case 0x83:
size, ok := gdFCVendorTLVDataSize(body[cursor:])
if !ok {
units = append(units, map[string]any{"type": "0x83", "error": "truncated"})
return eventTime, units
}
unit := parseGdFCVendorTLVData(0x83, body[cursor:cursor+size])
units = append(units, map[string]any{"type": "0x83", "name": "gd_fc_vendor_tlv", "value": unit})
cursor += size
default:
units = append(units, map[string]any{
"type": fmt.Sprintf("0x%02X", unitType),
"raw_tail": strings.ToUpper(hex.EncodeToString(body[cursor:])),
"parse": "unknown_unit",
"byte_size": len(body) - cursor,
})
return eventTime, units
}
}
return eventTime, units
}
func frameLength(frameStart []byte, bodyLen int) int {
standardLen := headerLen + bodyLen + 1
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
return standardLen
}
// Some field platform-login frames declare the standard 41-byte body but
// carry a 32-byte password, making the actual body 53 bytes. Only consume
// the extended form when its own BCC verifies so the splitter does not eat
// bytes from the next frame.
extendedLen := headerLen + 53 + 1
if len(frameStart) < extendedLen {
return standardLen
}
extended := frameStart[:extendedLen]
if bcc(extended[2:len(extended)-1]) == extended[len(extended)-1] {
return extendedLen
}
return standardLen
}
func isExtendedPlatformLogin(command byte, declaredBodyLen int, actualBodyLen int) bool {
return command == 0x05 && declaredBodyLen == 41 && actualBodyLen == 53
}
func parsePlatformLogin(body []byte) map[string]any {
login := map[string]any{
"raw_body_length": len(body),
}
if len(body) < 9 {
login["error"] = "truncated"
return login
}
loginTime := parseGBTime(body[:6])
if !loginTime.IsZero() {
login["login_time"] = loginTime.Format(time.RFC3339)
}
login["serial_no"] = binary.BigEndian.Uint16(body[6:8])
if len(body) >= 21 {
login["username"] = trimASCII(body[8:20])
}
if len(body) > 21 {
passwordEnd := len(body) - 1
login["password"] = trimASCII(body[20:passwordEnd])
login["encrypt_rule"] = body[passwordEnd]
}
return login
}
func parseVehicleData(data []byte) map[string]any {
return map[string]any{
"vehicle_status": int(data[0]),
"charge_status": int(data[1]),
"running_mode": int(data[2]),
"speed_kmh": float64(binary.BigEndian.Uint16(data[3:5])) / 10,
"total_mileage_km": float64(binary.BigEndian.Uint32(data[5:9])) / 10,
"total_voltage_v": float64(binary.BigEndian.Uint16(data[9:11])) / 10,
"total_current_a": float64(binary.BigEndian.Uint16(data[11:13]))/10 - 1000,
"soc_percent": int(data[13]),
"dc_dc_status": int(data[14]),
"gear": int(data[15]),
"insulation_kohm": binary.BigEndian.Uint16(data[16:18]),
"accelerator_pct": int(data[18]),
"brake_pct": int(data[19]),
}
}
func parseDriveMotorData(data []byte) map[string]any {
count := int(data[0])
motors := make([]map[string]any, 0, count)
cursor := 1
for i := 0; i < count; i++ {
motors = append(motors, map[string]any{
"serial_no": int(data[cursor]),
"state": int(data[cursor+1]),
"controller_temperature_c": int(data[cursor+2]) - 40,
"speed_rpm": int(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) - 20000,
"torque_nm": float64(int(binary.BigEndian.Uint16(data[cursor+5:cursor+7]))-20000) / 10,
"motor_temperature_c": int(data[cursor+7]) - 40,
"controller_voltage_v": float64(binary.BigEndian.Uint16(data[cursor+8:cursor+10])) / 10,
"controller_current_a": float64(binary.BigEndian.Uint16(data[cursor+10:cursor+12]))/10 - 1000,
})
cursor += 12
}
return map[string]any{
"count": count,
"motors": motors,
}
}
func parseFuelCellData(data []byte) map[string]any {
probeCount := int(binary.BigEndian.Uint16(data[6:8]))
probes := make([]int, 0, probeCount)
cursor := 8
for i := 0; i < probeCount; i++ {
probes = append(probes, int(data[cursor])-40)
cursor++
}
out := map[string]any{
"fuel_cell_voltage_v": float64(binary.BigEndian.Uint16(data[0:2])) / 10,
"fuel_cell_current_a": float64(binary.BigEndian.Uint16(data[2:4])) / 10,
"hydrogen_consumption_kg_per_100km": float64(binary.BigEndian.Uint16(data[4:6])) / 100,
"temperature_probe_count": probeCount,
"temperature_probe_values_c": probes,
"max_hydrogen_temperature_c": float64(binary.BigEndian.Uint16(data[cursor:cursor+2]))/10 - 40,
"max_hydrogen_temperature_probe_id": int(data[cursor+2]),
"max_hydrogen_concentration_fraction": float64(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) * 0.000001,
"max_hydrogen_concentration_probe_id": int(data[cursor+5]),
"max_hydrogen_pressure_mpa": float64(binary.BigEndian.Uint16(data[cursor+6:cursor+8])) / 10,
"max_hydrogen_pressure_probe_id": int(data[cursor+8]),
"dc_dc_status": int(data[cursor+9]),
}
return out
}
func parseEngineData(data []byte) map[string]any {
return map[string]any{
"engine_status": int(data[0]),
"crank_speed_rpm": binary.BigEndian.Uint16(data[1:3]),
"fuel_rate": binary.BigEndian.Uint16(data[3:5]),
}
}
func parseExtremeData(data []byte) map[string]any {
return map[string]any{
"max_voltage_subsystem_no": int(data[0]),
"max_voltage_cell_no": int(data[1]),
"max_voltage_v": float64(binary.BigEndian.Uint16(data[2:4])) / 1000,
"min_voltage_subsystem_no": int(data[4]),
"min_voltage_cell_no": int(data[5]),
"min_voltage_v": float64(binary.BigEndian.Uint16(data[6:8])) / 1000,
"max_temp_subsystem_no": int(data[8]),
"max_temp_probe_no": int(data[9]),
"max_temp_c": int(data[10]) - 40,
"min_temp_subsystem_no": int(data[11]),
"min_temp_probe_no": int(data[12]),
"min_temp_c": int(data[13]) - 40,
}
}
func alarmDataSize(data []byte) (int, bool) {
if len(data) < 5 {
return 0, false
}
cursor := 5
for i := 0; i < 4; i++ {
if len(data[cursor:]) < 1 {
return 0, false
}
count := int(data[cursor])
cursor++
cursor += count * 4
if cursor > len(data) {
return 0, false
}
}
return cursor, true
}
func parseAlarmData(data []byte) map[string]any {
cursor := 5
readList := func() []string {
count := int(data[cursor])
cursor++
values := make([]string, 0, count)
for i := 0; i < count; i++ {
values = append(values, fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[cursor:cursor+4])))
cursor += 4
}
return values
}
return map[string]any{
"max_alarm_level": int(data[0]),
"general_alarm_flag": fmt.Sprintf("0x%08X", binary.BigEndian.Uint32(data[1:5])),
"battery_faults": readList(),
"motor_faults": readList(),
"engine_faults": readList(),
"other_faults": readList(),
}
}
func voltageDataSize(data []byte) (int, bool) {
if len(data) < 1 {
return 0, false
}
cursor := 1
for i := 0; i < int(data[0]); i++ {
if len(data[cursor:]) < 10 {
return 0, false
}
frameCellCount := int(data[cursor+9])
cursor += 10 + frameCellCount*2
if cursor > len(data) {
return 0, false
}
}
return cursor, true
}
func parseVoltageData(data []byte) map[string]any {
subCount := int(data[0])
cursor := 1
totalVoltage := 0.0
maxCell := 0.0
minCell := 0.0
cellSeen := false
for i := 0; i < subCount; i++ {
totalVoltage += float64(binary.BigEndian.Uint16(data[cursor+1:cursor+3])) / 10
cellCount := int(binary.BigEndian.Uint16(data[cursor+5 : cursor+7]))
frameCellCount := int(data[cursor+9])
cursor += 10
for c := 0; c < frameCellCount; c++ {
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
if !cellSeen || voltage > maxCell {
maxCell = voltage
}
if !cellSeen || voltage < minCell {
minCell = voltage
}
cellSeen = true
cursor += 2
}
_ = cellCount
}
return map[string]any{
"subsystem_count": subCount,
"total_voltage_v": totalVoltage,
"max_cell_v": maxCell,
"min_cell_v": minCell,
}
}
func temperatureDataSize(data []byte) (int, bool) {
if len(data) < 1 {
return 0, false
}
cursor := 1
for i := 0; i < int(data[0]); i++ {
if len(data[cursor:]) < 3 {
return 0, false
}
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
cursor += 3 + probeCount
if cursor > len(data) {
return 0, false
}
}
return cursor, true
}
func parseTemperatureData(data []byte) map[string]any {
subCount := int(data[0])
cursor := 1
maxTemp := 0
minTemp := 0
seen := false
for i := 0; i < subCount; i++ {
probeCount := int(binary.BigEndian.Uint16(data[cursor+1 : cursor+3]))
cursor += 3
for p := 0; p < probeCount; p++ {
temp := int(data[cursor]) - 40
if !seen || temp > maxTemp {
maxTemp = temp
}
if !seen || temp < minTemp {
minTemp = temp
}
seen = true
cursor++
}
}
return map[string]any{
"subsystem_count": subCount,
"max_temp_c": maxTemp,
"min_temp_c": minTemp,
}
}
func gdFCStackDataSize(data []byte) (int, bool) {
if len(data) < 1 {
return 0, false
}
cursor := 1
for i := 0; i < int(data[0]); i++ {
if len(data[cursor:]) < 22 {
return 0, false
}
frameCellCount := int(data[cursor+21])
cursor += 22 + frameCellCount*2
if cursor > len(data) {
return 0, false
}
}
return cursor, true
}
func parseGdFCStackData(data []byte) map[string]any {
count := int(data[0])
cursor := 1
summaries := make([]map[string]any, 0, count)
for i := 0; i < count; i++ {
frameCellCount := int(data[cursor+21])
summary := map[string]any{
"engine_work_state": int(data[cursor]),
"stack_water_outlet_temp_c": nullableTempOffset40(data[cursor+1]),
"hydrogen_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+2:cursor+4], 0.1, -100),
"air_inlet_pressure_kpa": nullableScaledU16WithOffset(data[cursor+4:cursor+6], 0.1, -100),
"air_inlet_temp_c": nullableTempOffset40(data[cursor+6]),
"max_cell_voltage_id": nullableU16(data[cursor+7 : cursor+9]),
"min_cell_voltage_id": nullableU16(data[cursor+9 : cursor+11]),
"max_cell_voltage_v": nullableScaledU16(data[cursor+11:cursor+13], 0.001),
"min_cell_voltage_v": nullableScaledU16(data[cursor+13:cursor+15], 0.001),
"avg_cell_voltage_v": nullableScaledU16(data[cursor+15:cursor+17], 0.001),
"cell_count": nullableU16(data[cursor+17 : cursor+19]),
"frame_cell_start": nullableU16(data[cursor+19 : cursor+21]),
"frame_cell_count": frameCellCount,
}
cursor += 22
maxCell := 0.0
minCell := 0.0
seen := false
for c := 0; c < frameCellCount; c++ {
voltage := float64(binary.BigEndian.Uint16(data[cursor:cursor+2])) / 1000
if !seen || voltage > maxCell {
maxCell = voltage
}
if !seen || voltage < minCell {
minCell = voltage
}
seen = true
cursor += 2
}
if seen {
summary["frame_max_cell_voltage_v"] = maxCell
summary["frame_min_cell_voltage_v"] = minCell
}
summaries = append(summaries, summary)
}
return map[string]any{
"stack_count": count,
"summaries": summaries,
}
}
func gdFCAuxiliaryDataSize(data []byte) (int, bool) {
if len(data) < 1 {
return 0, false
}
size := 1 + int(data[0])*16
return size, len(data) >= size
}
func parseGdFCAuxiliaryData(data []byte) map[string]any {
count := int(data[0])
cursor := 1
subsystems := make([]map[string]any, 0, count)
for i := 0; i < count; i++ {
subsystems = append(subsystems, map[string]any{
"air_compressor_motor_voltage_v": nullableScaledU16(data[cursor:cursor+2], 0.1),
"air_compressor_power_kw": nullableU16WithOffset(data[cursor+2:cursor+4], -5),
"hydrogen_pump_motor_voltage_v": nullableScaledU16(data[cursor+4:cursor+6], 0.2),
"hydrogen_pump_power_kw": nullableU16WithOffset(data[cursor+6:cursor+8], -5),
"water_pump_voltage_v": nullableScaledU16(data[cursor+8:cursor+10], 0.1),
"ptc_voltage_v": nullableScaledU16(data[cursor+10:cursor+12], 0.1),
"ptc_power_kw": nullableU16WithOffset(data[cursor+12:cursor+14], -5),
"low_voltage_battery_voltage_v": nullableScaledU16(data[cursor+14:cursor+16], 0.1),
})
cursor += 16
}
return map[string]any{
"subsystem_count": count,
"subsystems": subsystems,
}
}
func parseGdFCDcDcData(data []byte) map[string]any {
return map[string]any{
"input_voltage_v": nullableScaledU16(data[0:2], 0.1),
"input_current_a": nullableScaledU16(data[2:4], 0.1),
"output_voltage_v": nullableScaledU16(data[4:6], 0.1),
"output_current_a": nullableScaledU16(data[6:8], 0.1),
"controller_temp_c": nullableTempOffset40(data[8]),
}
}
func parseGdFCAirConditionerData(data []byte) map[string]any {
return map[string]any{
"status": int(data[0]),
"power_kw": nullableU16WithOffset(data[1:3], -5),
"compressor_input_voltage_v": nullableScaledU16(data[3:5], 0.1),
}
}
func parseGdFCVehicleInfoData(data []byte) map[string]any {
return map[string]any{
"collision_alarm": int(data[0]),
"ambient_temp_c": nullableU16WithOffset(data[1:3], -40),
"ambient_pressure_kpa": nullableScaledU16WithOffset(data[3:5], 0.1, -100),
"hydrogen_mass_kg": nullableScaledU16(data[5:7], 0.1),
}
}
func parseGdFCDemoExtensionData(data []byte) map[string]any {
return map[string]any{
"declared_length": int(binary.BigEndian.Uint16(data[0:2])),
"stack_temp_c": nullableTempOffset40(data[2]),
"air_compressor_voltage_v": nullableScaledU16(data[3:5], 0.1),
"air_compressor_current_a": nullableScaledU16WithOffset(data[5:7], 0.1, -1000),
"hydrogen_pump_voltage_v": nullableScaledU16(data[7:9], 0.1),
"hydrogen_pump_current_a": nullableScaledU16WithOffset(data[9:11], 0.1, -1000),
}
}
func gdFCVendorTLVDataSize(data []byte) (int, bool) {
if len(data) < 2 {
return 0, false
}
size := 2 + int(binary.BigEndian.Uint16(data[0:2]))
return size, len(data) >= size
}
func parseGdFCVendorTLVData(typeCode byte, data []byte) map[string]any {
length := int(binary.BigEndian.Uint16(data[0:2]))
return map[string]any{
"type": fmt.Sprintf("0x%02X", typeCode),
"declared_length": length,
"payload_hex": strings.ToUpper(hex.EncodeToString(data[2:])),
}
}
func nullableU16(data []byte) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
return nil
}
return int(value)
}
func nullableU16WithOffset(data []byte, offset int) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
return nil
}
return int(value) + offset
}
func nullableScaledU16(data []byte, scale float64) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
return nil
}
return float64(value) * scale
}
func nullableScaledU16WithOffset(data []byte, scale float64, offset float64) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
return nil
}
return float64(value)*scale + offset
}
func nullableTempOffset40(value byte) any {
if value == 0xfe || value == 0xff {
return nil
}
return int(value) - 40
}
func parsePositionData(data []byte) map[string]any {
return map[string]any{
"position_status": int(data[0]),
"longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
"latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
}
}
func indexStart(data []byte) int {
for i := 0; i+1 < len(data); i++ {
if _, ok := version(data[i], data[i+1]); ok {
return i
}
}
return -1
}
func version(a byte, b byte) (string, bool) {
switch {
case a == '#' && b == '#':
return "V2016", true
case a == '$' && b == '$':
return "V2025", true
default:
return "", false
}
}
func parseGBTime(data []byte) time.Time {
if len(data) != 6 {
return time.Time{}
}
year := 2000 + int(data[0])
month := time.Month(data[1])
day := int(data[2])
hour := int(data[3])
minute := int(data[4])
second := int(data[5])
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
return time.Time{}
}
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
}
func trimASCII(data []byte) string {
return strings.TrimRight(string(data), "\x00 ")
}
func bcc(data []byte) byte {
var out byte
for _, value := range data {
out ^= value
}
return out
}

View File

@@ -0,0 +1,258 @@
package gb32960
import (
"encoding/hex"
"fmt"
"math"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
const realFuelCellFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39})
second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
stream := append([]byte{0x00, 0x01}, first...)
stream = append(stream, second[:len(second)-3]...)
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 {
t.Fatalf("expected one complete frame, got %d", len(frames))
}
if string(remainder) != string(second[:len(second)-3]) {
t.Fatalf("expected partial second frame as remainder, got %s", hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782745114999, "115.29.187.205:32960")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x02" {
t.Fatalf("unexpected envelope header: %#v", env)
}
if env.VIN != "LNBSCB3D4R1234567" {
t.Fatalf("unexpected vin: %q", env.VIN)
}
}
func TestParseFrameRejectsBadBCC(t *testing.T) {
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
frame[len(frame)-1] ^= 0xff
if _, err := ParseFrame(frame, 1782745114999, ""); err == nil {
t.Fatal("expected bad bcc error")
}
}
func TestExtractFramesSupportsExtendedPlatformLogin(t *testing.T) {
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x1f, 0x00, 0x01}
body = append(body, fixedASCII("Hyundai", 12)...)
body = append(body, fixedASCII("f2e3445d7cda409fb4f278f6fb890734", 32)...)
body = append(body, 0x01)
frame := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
frames, remainder, err := ExtractFrames(append(frame, []byte{0x23, 0x23, 0x05}...))
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 3 {
t.Fatalf("unexpected split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
if len(frames[0]) != headerLen+53+1 {
t.Fatalf("extended frame len = %d", len(frames[0]))
}
env, err := ParseFrame(frames[0], 1782745114999, "8.134.95.166:39376")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x05" || env.ParseStatus != envelope.ParseOK {
t.Fatalf("unexpected env: %#v", env)
}
header := env.Parsed["header"].(map[string]any)
if header["body_length"] != 41 || header["actual_body_length"] != 53 {
t.Fatalf("unexpected length header: %#v", header)
}
login := env.Parsed["platform_login"].(map[string]any)
if login["username"] != "Hyundai" || login["password"] != "f2e3445d7cda409fb4f278f6fb890734" {
t.Fatalf("unexpected platform login: %#v", login)
}
if login["login_time"] != "2026-07-02T00:38:31+08:00" {
t.Fatalf("unexpected login time: %#v", login)
}
if env.Fields["platform_account"] != "Hyundai" {
t.Fatalf("platform account not exposed: %#v", env.Fields)
}
}
func TestAutoResponseEchoesRawVINAndOriginalTime(t *testing.T) {
body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x0f, 0x00, 0x01}
body = append(body, fixedASCII("Hyundai", 12)...)
body = append(body, fixedASCII("ack-test-password", 32)...)
body = append(body, 0x01)
request := buildFrameWithDeclaredLength(0x05, 0xfe, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", body, 41)
env, err := ParseFrame(request, 1782914969584, "8.134.95.166:39376")
if err != nil {
t.Fatal(err)
}
response, ok, err := AutoResponse(request, env)
if err != nil {
t.Fatalf("AutoResponse() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
if response[2] != 0x05 || response[3] != 0x01 {
t.Fatalf("unexpected response header: %x", response[:4])
}
if hex.EncodeToString(response[4:21]) != "0000000000000000000000000000000000" {
t.Fatalf("raw vin not echoed: %x", response[4:21])
}
if hex.EncodeToString(response[24:30]) != "1a070200260f" {
t.Fatalf("timestamp body not preserved: %x", response[24:30])
}
if got, want := bcc(response[2:len(response)-1]), response[len(response)-1]; got != want {
t.Fatalf("response bcc got=0x%02x want=0x%02x", got, want)
}
}
func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
body := []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39}
body = append(body, 0x01)
body = append(body,
0x01, // vehicle status
0x03, // charge status
0x02, // running mode
0x01, 0x2c, // speed: 30.0km/h
0x00, 0x01, 0x86, 0xa0, // total mileage: 10000.0km
0x15, 0xe5, // total voltage: 560.5V
0x27, 0x10, // total current: 0A after -1000 offset
85, 0x01, 0x00,
0x27, 0x10,
0x00, 0x00)
body = append(body, 0x05)
body = append(body,
0x00,
0x07, 0x36, 0x50, 0x40, // longitude: 121.000000
0x01, 0xd2, 0x4f, 0x00) // latitude: 30.560000
frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", body)
env, err := ParseFrame(frame, 1782745114999, "115.29.187.205:32960")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
assertFloatField(t, env, envelope.FieldSpeedKMH, 30.0)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10000.0)
assertFloatField(t, env, envelope.FieldLongitude, 121.0)
assertFloatField(t, env, envelope.FieldLatitude, 30.56)
if env.Fields[envelope.FieldSOCPercent] != 85 {
t.Fatalf("unexpected soc: %#v", env.Fields[envelope.FieldSOCPercent])
}
if env.EventTimeMS == 1782745114999 {
t.Fatal("event time should come from GB32960 device timestamp")
}
}
func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testing.T) {
frame, err := hex.DecodeString(realFuelCellFrame)
if err != nil {
t.Fatal(err)
}
env, err := ParseFrame(frame, 1782914969584, "115.29.187.205:32960")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.VIN != "LB9A32A21R0LS1707" {
t.Fatalf("vin = %q", env.VIN)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 53490.9)
assertFloatField(t, env, "fuel_cell_hydrogen_consumption_kg_per_100km", 1.8)
assertFloatField(t, env, envelope.FieldLongitude, 120.800326)
assertFloatField(t, env, envelope.FieldLatitude, 31.634907)
assertFloatField(t, env, "gd_fc_stack_hydrogen_inlet_pressure_kpa", 97.0)
assertFloatField(t, env, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
assertIntField(t, env, "gd_fc_demo_stack_temp_c", 65)
units, ok := env.Parsed["data_units"].([]map[string]any)
if !ok {
t.Fatalf("data_units missing: %#v", env.Parsed["data_units"])
}
gotTypes := make([]string, 0, len(units))
for _, unit := range units {
gotTypes = append(gotTypes, fmt.Sprint(unit["type"]))
}
wantTypes := []string{"0x01", "0x02", "0x03", "0x04", "0x05", "0x06", "0x07", "0x08", "0x09", "0x30", "0x31", "0x32", "0x33", "0x34", "0x80", "0x83"}
if fmt.Sprint(gotTypes) != fmt.Sprint(wantTypes) {
t.Fatalf("unit types = %v, want %v", gotTypes, wantTypes)
}
if units[1]["name"] != "drive_motor" {
t.Fatalf("unexpected motor unit: %#v", units[1])
}
if units[2]["name"] != "fuel_cell" {
t.Fatalf("unexpected fuel cell unit: %#v", units[2])
}
if units[9]["name"] != "gd_fc_stack" {
t.Fatalf("unexpected gd stack unit: %#v", units[9])
}
if units[10]["name"] != "gd_fc_auxiliary" {
t.Fatalf("unexpected gd auxiliary unit: %#v", units[10])
}
if units[15]["name"] != "gd_fc_vendor_tlv" {
t.Fatalf("unexpected gd vendor tlv unit: %#v", units[15])
}
tlv := units[15]["value"].(map[string]any)
if tlv["declared_length"] != 36 {
t.Fatalf("unexpected vendor tlv: %#v", tlv)
}
}
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
t.Helper()
got, ok := env.Fields[key].(float64)
if !ok {
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
}
if math.Abs(got-want) > 0.000001 {
t.Fatalf("field %s = %v, want %v", key, got, want)
}
}
func assertIntField(t *testing.T, env envelope.FrameEnvelope, key string, want int) {
t.Helper()
got, ok := env.Fields[key].(int)
if !ok {
t.Fatalf("field %s missing or not int: %#v", key, env.Fields[key])
}
if got != want {
t.Fatalf("field %s = %v, want %v", key, got, want)
}
}
func buildFrame(command byte, response byte, vin string, body []byte) []byte {
return buildFrameWithDeclaredLength(command, response, vin, body, len(body))
}
func buildFrameWithDeclaredLength(command byte, response byte, vin string, body []byte, declaredBodyLen int) []byte {
frame := []byte{'#', '#', command, response}
vinBytes := []byte(vin)
if len(vinBytes) < 17 {
vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
}
frame = append(frame, vinBytes[:17]...)
frame = append(frame, 0x01, byte(declaredBodyLen>>8), byte(declaredBodyLen))
frame = append(frame, body...)
return append(frame, bcc(frame[2:]))
}
func fixedASCII(value string, size int) []byte {
out := make([]byte, size)
copy(out, []byte(value))
return out
}

View File

@@ -0,0 +1,80 @@
package gb32960
import (
"encoding/binary"
"errors"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrResponseFrameTooShort = errors.New("gb32960 response frame too short")
const (
responseSuccess = byte(0x01)
responseCommand = byte(0xfe)
encryptNone = byte(0x01)
)
// AutoResponse builds the protocol ACK for accepted upstream GB/T 32960 frames.
// The ACK is written only after the caller has durably published the frame.
func AutoResponse(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if env.ParseStatus == envelope.ParseBadFrame {
return nil, false, nil
}
if len(raw) < headerLen+1 {
return nil, false, ErrResponseFrameTooShort
}
if raw[3] != responseCommand {
return nil, false, nil
}
command := raw[2]
if !shouldRespond(command) {
return nil, false, nil
}
body := []byte(nil)
if timestamp := responseTime(raw, command); !timestamp.IsZero() {
body = encodeGBTime(timestamp)
}
return buildResponse(raw[0], command, responseSuccess, raw[4:21], body), true, nil
}
func shouldRespond(command byte) bool {
switch command {
case 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08:
return true
default:
return false
}
}
func responseTime(raw []byte, command byte) time.Time {
bodyLen := len(raw) - headerLen - 1
if bodyLen >= 6 {
return parseGBTime(raw[headerLen : headerLen+6])
}
return time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600))
}
func encodeGBTime(t time.Time) []byte {
local := t.In(time.FixedZone("Asia/Shanghai", 8*3600))
return []byte{
byte(local.Year() - 2000),
byte(local.Month()),
byte(local.Day()),
byte(local.Hour()),
byte(local.Minute()),
byte(local.Second()),
}
}
func buildResponse(start byte, command byte, responseFlag byte, vin17 []byte, body []byte) []byte {
out := make([]byte, 0, headerLen+len(body)+1)
out = append(out, start, start, command, responseFlag)
out = append(out, vin17[:17]...)
out = append(out, encryptNone, 0, 0)
binary.BigEndian.PutUint16(out[22:24], uint16(len(body)))
out = append(out, body...)
return append(out, bcc(out[2:]))
}

View File

@@ -0,0 +1,600 @@
package jt808
import (
"bytes"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"golang.org/x/text/encoding/simplifiedchinese"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var (
ErrFrameTooShort = errors.New("jt808 frame too short")
ErrChecksum = errors.New("jt808 checksum mismatch")
ErrEscape = errors.New("jt808 invalid escape sequence")
)
type Location struct {
AlarmFlag uint32
StatusFlag uint32
Latitude float64
Longitude float64
AltitudeM uint16
SpeedKMH float64
DirectionDeg uint16
Time time.Time
TotalMileageKM *float64
Additional []AdditionalItem
}
type AdditionalItem struct {
ID byte
Length int
ValueHex string
Parsed any
}
// ExtractFrames splits JT/T 808 TCP streams by 0x7e delimiters and unescapes
// complete frame payloads. Returned frames exclude the leading and trailing 0x7e.
func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
start := -1
for i, value := range stream {
if value != 0x7e {
continue
}
if start < 0 {
start = i
continue
}
if i == start+1 {
start = i
continue
}
frame, err := unescape(stream[start+1 : i])
if err != nil {
return nil, nil, err
}
frames = append(frames, frame)
start = i
}
if start >= 0 && start < len(stream)-1 {
return frames, append([]byte(nil), stream[start:]...), nil
}
return frames, nil, nil
}
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
if len(raw) < 13 {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
}
if got, want := checksum(raw[:len(raw)-1]), raw[len(raw)-1]; got != want {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrChecksum, got, want)
}
messageID := binary.BigEndian.Uint16(raw[0:2])
props := binary.BigEndian.Uint16(raw[2:4])
bodySize := props & 0x03ff
versioned := props&0x4000 != 0
headerLen := 12
phoneStart := 4
phoneEnd := 10
serialStart := 10
packageInfo := map[string]any(nil)
protocolVersion := byte(0)
if versioned {
headerLen = 17
if len(raw) < headerLen {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: versioned header len=%d", ErrFrameTooShort, len(raw))
}
protocolVersion = raw[4]
phoneStart = 5
phoneEnd = 15
serialStart = 15
}
subpackage := props&0x2000 != 0
if subpackage {
headerLen += 4
}
if len(raw) < headerLen+int(bodySize)+1 {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
}
if subpackage {
packageInfo = map[string]any{
"total": binary.BigEndian.Uint16(raw[serialStart+2 : serialStart+4]),
"index": binary.BigEndian.Uint16(raw[serialStart+4 : serialStart+6]),
}
}
phoneBCD := raw[phoneStart:phoneEnd]
phone := bcdString(phoneBCD)
phoneTrimZero := strings.TrimLeft(phone, "0")
if versioned && phoneTrimZero != "" {
phone = phoneTrimZero
}
body := raw[headerLen : headerLen+int(bodySize)]
parsed := map[string]any{
"header": map[string]any{
"message_id": fmt.Sprintf("0x%04X", messageID),
"protocol_version": protocolVersion,
"versioned": versioned,
"properties": props,
"body_size": bodySize,
"phone_bcd_hex": strings.ToUpper(hex.EncodeToString(phoneBCD)),
"phone": phone,
"phone_trim_zero": phoneTrimZero,
"sequence": binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
"subpackage": subpackage,
"package_info": packageInfo,
"encryption_flags": (props >> 10) & 0x07,
},
}
fields := map[string]any{}
eventTimeMS := receivedAtMS
if messageID == 0x0200 {
location, err := parseLocation(body)
if err != nil {
return envelope.FrameEnvelope{}, err
}
parsed["location"] = locationToMap(location)
fields[envelope.FieldLatitude] = location.Latitude
fields[envelope.FieldLongitude] = location.Longitude
fields[envelope.FieldSpeedKMH] = location.SpeedKMH
fields["altitude_m"] = location.AltitudeM
fields["direction_deg"] = location.DirectionDeg
fields["alarm_flag"] = location.AlarmFlag
fields["status_flag"] = location.StatusFlag
if !location.Time.IsZero() {
eventTimeMS = location.Time.UnixMilli()
fields["device_time"] = location.Time.Format(time.RFC3339)
}
if location.TotalMileageKM != nil {
fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
}
} else if messageID == 0x0100 {
registration, err := parseRegistration(body, versioned)
if err != nil {
return envelope.FrameEnvelope{}, err
}
parsed["registration"] = registration
if deviceID, ok := registration["device_id"].(string); ok {
fields["device_id"] = deviceID
}
if plate, ok := registration["plate"].(string); ok {
fields["plate"] = plate
}
} else if messageID == 0x0102 {
authentication := parseAuthentication(body, versioned)
parsed["authentication"] = authentication
if token, ok := authentication["token"].(string); ok {
fields["auth_token"] = token
}
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: fmt.Sprintf("0x%04X", messageID),
Sequence: binary.BigEndian.Uint16(raw[serialStart : serialStart+2]),
Phone: phone,
SourceEndpoint: sourceEndpoint,
EventTimeMS: eventTimeMS,
ReceivedAtMS: receivedAtMS,
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
Parsed: parsed,
Fields: fields,
ParseStatus: envelope.ParseOK,
}
if registration, ok := parsed["registration"].(map[string]any); ok {
env.DeviceID, _ = registration["device_id"].(string)
env.Plate, _ = registration["plate"].(string)
}
env.EventID = env.StableEventID()
return env, nil
}
func parseRegistration(body []byte, versioned bool) (map[string]any, error) {
schemaVersion := "2011_2013"
manufacturerSize := 5
deviceTypeSize := 20
deviceIDSize := 7
if versioned {
schemaVersion = "2019"
manufacturerSize = 11
deviceTypeSize = 30
deviceIDSize = 30
}
minLen := 2 + 2 + manufacturerSize + deviceTypeSize + deviceIDSize + 1
if len(body) < minLen {
return nil, fmt.Errorf("%w: registration body len=%d", ErrFrameTooShort, len(body))
}
manufacturerStart := 4
deviceTypeStart := manufacturerStart + manufacturerSize
deviceIDStart := deviceTypeStart + deviceTypeSize
plateColorStart := deviceIDStart + deviceIDSize
registration := map[string]any{
"schema_version": schemaVersion,
"province": binary.BigEndian.Uint16(body[0:2]),
"city": binary.BigEndian.Uint16(body[2:4]),
"manufacturer": fixedText(body[manufacturerStart:deviceTypeStart]),
"device_type": fixedText(body[deviceTypeStart:deviceIDStart]),
"device_id": fixedText(body[deviceIDStart:plateColorStart]),
"plate_color": body[plateColorStart],
"plate": freeText(body[plateColorStart+1:]),
}
return registration, nil
}
func parseAuthentication(body []byte, versioned bool) map[string]any {
if versioned {
if out, ok := parseVersionedAuthentication(body); ok {
return out
}
if out, ok := parseDelimitedVersionedAuthentication(body); ok {
return out
}
}
return map[string]any{"token": freeText(body)}
}
func parseVersionedAuthentication(body []byte) (map[string]any, bool) {
if len(body) < 17 {
return nil, false
}
tokenLen := int(body[0])
imeiStart := 1 + tokenLen
softwareLenStart := imeiStart + 15
if tokenLen <= 0 || len(body) < softwareLenStart+1 {
return nil, false
}
softwareLen := int(body[softwareLenStart])
softwareStart := softwareLenStart + 1
if len(body) < softwareStart+softwareLen {
return nil, false
}
return map[string]any{
"schema_version": "2019",
"token": freeText(body[1:imeiStart]),
"imei": fixedText(body[imeiStart:softwareLenStart]),
"software_version": freeText(body[softwareStart : softwareStart+softwareLen]),
}, true
}
func parseDelimitedVersionedAuthentication(body []byte) (map[string]any, bool) {
parts := bytes.Split(body, []byte{0xff, 0xff})
if len(parts) < 2 {
return nil, false
}
out := map[string]any{
"schema_version": "2019_delimited",
"token": freeText(parts[0]),
"imei": freeText(parts[1]),
}
if len(parts) > 2 {
out["software_version"] = freeText(parts[2])
}
return out, true
}
func parseLocation(body []byte) (Location, error) {
if len(body) < 28 {
return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
}
status := binary.BigEndian.Uint32(body[4:8])
latitude := float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000
longitude := float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000
if status&(1<<2) != 0 {
latitude = -latitude
}
if status&(1<<3) != 0 {
longitude = -longitude
}
location := Location{
AlarmFlag: binary.BigEndian.Uint32(body[0:4]),
StatusFlag: status,
Latitude: latitude,
Longitude: longitude,
AltitudeM: binary.BigEndian.Uint16(body[16:18]),
SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10,
DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
Time: parseBCDTime(body[22:28]),
}
location.Additional = parseAdditional(body[28:], &location)
return location, nil
}
func parseAdditional(data []byte, location *Location) []AdditionalItem {
var out []AdditionalItem
for len(data) >= 2 {
id := data[0]
size := int(data[1])
data = data[2:]
if len(data) < size {
out = append(out, AdditionalItem{
ID: id,
Length: size,
ValueHex: strings.ToUpper(hex.EncodeToString(data)),
Parsed: "truncated",
})
return out
}
value := data[:size]
item := AdditionalItem{
ID: id,
Length: size,
ValueHex: strings.ToUpper(hex.EncodeToString(value)),
}
switch {
case id == 0x01 && size == 4:
rawMileage := binary.BigEndian.Uint32(value)
mileage := float64(rawMileage) / 10
location.TotalMileageKM = &mileage
item.Parsed = map[string]any{
"name": envelope.FieldTotalMileageKM,
"raw_value_tenth_km": rawMileage,
"value": mileage,
"unit": "km",
}
case id == 0x02 && size == 2:
rawFuel := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "fuel_l",
"raw_value_tenth_l": rawFuel,
"value": float64(rawFuel) / 10,
"unit": "L",
}
case id == 0x03 && size == 2:
rawSpeed := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "recorder_speed_kmh",
"raw_value_tenth_kmh": rawSpeed,
"value": float64(rawSpeed) / 10,
"unit": "km/h",
}
case id == 0x04 && size == 2:
item.Parsed = map[string]any{
"name": "manual_alarm_event_id",
"value": binary.BigEndian.Uint16(value),
}
case id == 0x05 && size == 30:
item.Parsed = map[string]any{
"name": "tire_pressure",
"values": parseTirePressure(value),
}
case id == 0x06 && size == 2:
item.Parsed = map[string]any{
"name": "carriage_temperature_c",
"value": int16(binary.BigEndian.Uint16(value)),
"unit": "C",
}
case id == 0x11 && (size == 1 || size == 5):
parsed := map[string]any{
"name": "overspeed_alarm",
"location_type": value[0],
}
if size == 5 {
parsed["area_id"] = binary.BigEndian.Uint32(value[1:5])
}
item.Parsed = parsed
case id == 0x12 && size == 6:
item.Parsed = map[string]any{
"name": "area_route_alarm",
"location_type": value[0],
"area_id": binary.BigEndian.Uint32(value[1:5]),
"direction": value[5],
}
case id == 0x13 && size == 7:
item.Parsed = map[string]any{
"name": "road_section_driving_time_alarm",
"road_section_id": binary.BigEndian.Uint32(value[0:4]),
"driving_time_seconds": binary.BigEndian.Uint16(value[4:6]),
"result": value[6],
"result_description": roadSectionDrivingTimeResult(value[6]),
"driving_time_result_source": "jt808_0x0200_additional_0x13",
}
case id == 0x25 && size == 4:
rawStatus := binary.BigEndian.Uint32(value)
item.Parsed = map[string]any{
"name": "extended_vehicle_signal_status",
"value": rawStatus,
"bits": parseExtendedVehicleSignal(rawStatus),
}
case id == 0x2a && size == 2:
rawStatus := binary.BigEndian.Uint16(value)
item.Parsed = map[string]any{
"name": "io_status",
"value": rawStatus,
"bits": parseIOStatus(rawStatus),
}
case id == 0x2b && size == 4:
analog := binary.BigEndian.Uint32(value)
item.Parsed = map[string]any{
"name": "analog",
"ad0": analog & 0xffff,
"ad1": analog >> 16,
"value": analog,
}
case id == 0x30 && size == 1:
item.Parsed = map[string]any{
"name": "network_signal_strength",
"value": value[0],
}
case id == 0x31 && size == 1:
item.Parsed = map[string]any{
"name": "gnss_satellite_count",
"value": value[0],
}
}
out = append(out, item)
data = data[size:]
}
return out
}
func locationToMap(location Location) map[string]any {
out := map[string]any{
"alarm_flag": location.AlarmFlag,
"status_flag": location.StatusFlag,
"latitude": location.Latitude,
"longitude": location.Longitude,
"altitude_m": location.AltitudeM,
"speed_kmh": location.SpeedKMH,
"direction_deg": location.DirectionDeg,
"additional": additionalToMaps(location.Additional),
}
if !location.Time.IsZero() {
out["device_time"] = location.Time.Format(time.RFC3339)
}
if location.TotalMileageKM != nil {
out[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
}
return out
}
func additionalToMaps(items []AdditionalItem) []map[string]any {
out := make([]map[string]any, 0, len(items))
for _, item := range items {
value := map[string]any{
"id": fmt.Sprintf("0x%02X", item.ID),
"length": item.Length,
"value_hex": item.ValueHex,
}
if item.Parsed != nil {
value["parsed"] = item.Parsed
}
out = append(out, value)
}
return out
}
func parseTirePressure(data []byte) []map[string]any {
out := make([]map[string]any, 0, len(data))
for index, value := range data {
out = append(out, map[string]any{
"position": index,
"value": value,
"valid": value != 0xff,
})
}
return out
}
func parseExtendedVehicleSignal(value uint32) map[string]bool {
return map[string]bool{
"low_beam": value&0x0001 != 0,
"high_beam": value&0x0002 != 0,
"right_turn": value&0x0004 != 0,
"left_turn": value&0x0008 != 0,
"brake": value&0x0010 != 0,
"reverse_gear": value&0x0020 != 0,
"fog_light": value&0x0040 != 0,
"clearance_light": value&0x0080 != 0,
"horn": value&0x0100 != 0,
"air_conditioner": value&0x0200 != 0,
"neutral": value&0x0400 != 0,
"retarder": value&0x0800 != 0,
"abs": value&0x1000 != 0,
"heater": value&0x2000 != 0,
"clutch": value&0x4000 != 0,
}
}
func parseIOStatus(value uint16) map[string]bool {
return map[string]bool{
"deep_sleep": value&0x0001 != 0,
"sleep": value&0x0002 != 0,
}
}
func roadSectionDrivingTimeResult(value byte) string {
if value == 0 {
return "insufficient"
}
if value == 1 {
return "too_long"
}
return "unknown"
}
func bcdString(data []byte) string {
out := make([]byte, 0, len(data)*2)
for _, value := range data {
out = append(out, '0'+((value>>4)&0x0f), '0'+(value&0x0f))
}
return string(out)
}
func fixedText(data []byte) string {
return strings.Trim(freeText(data), "\x00 ")
}
func freeText(data []byte) string {
data = bytes.Trim(data, "\x00 ")
if len(data) == 0 {
return ""
}
if utf8.Valid(data) {
return string(data)
}
decoded, err := simplifiedchinese.GBK.NewDecoder().Bytes(data)
if err == nil && utf8.Valid(decoded) {
return string(decoded)
}
return strings.ToValidUTF8(string(data), "")
}
func parseBCDTime(data []byte) time.Time {
if len(data) != 6 {
return time.Time{}
}
year := 2000 + bcdByte(data[0])
month := time.Month(bcdByte(data[1]))
day := bcdByte(data[2])
hour := bcdByte(data[3])
minute := bcdByte(data[4])
second := bcdByte(data[5])
if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
return time.Time{}
}
return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
}
func bcdByte(value byte) int {
return int(value>>4)*10 + int(value&0x0f)
}
func unescape(data []byte) ([]byte, error) {
out := make([]byte, 0, len(data))
for i := 0; i < len(data); i++ {
if data[i] != 0x7d {
out = append(out, data[i])
continue
}
if i+1 >= len(data) {
return nil, ErrEscape
}
i++
switch data[i] {
case 0x01:
out = append(out, 0x7d)
case 0x02:
out = append(out, 0x7e)
default:
return nil, ErrEscape
}
}
return out, nil
}
func checksum(data []byte) byte {
var out byte
for _, value := range data {
out ^= value
}
return out
}

View File

@@ -0,0 +1,503 @@
package jt808
import (
"encoding/binary"
"encoding/hex"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
const real2019LocationFrame = "7E02004046010000000001489413506007CB00000000004C000301D276AB06FBFF4B000D014E00E826070122092214040000000017020000010400013361030201402504000000012A02000030011D310112EA0402008000C57E"
const fullAdditionalLocationFrame = "7e020000800123456789017fff000004000000080006eeb6ad02633df701380003006320070719235901040000000b02020016030200210402002c051e3737370000000000000000000000000000000000000000000000000000001105420000004212064d0000004d4d1307000000580058582504000000632a02000a2b040000001430011e31012806020001927e"
func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
if err != nil {
t.Fatal(err)
}
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 {
t.Fatalf("expected one complete frame, got %d", len(frames))
}
if hex.EncodeToString(remainder) != "7e02000000" {
t.Fatalf("expected partial remainder, got %s", hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782745114999, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Protocol != envelope.ProtocolJT808 {
t.Fatalf("protocol = %q", env.Protocol)
}
if env.MessageID != "0x0200" {
t.Fatalf("message id = %q", env.MessageID)
}
if env.Phone != "013307795425" {
t.Fatalf("phone = %q", env.Phone)
}
if env.Sequence != 1 {
t.Fatalf("sequence = %d", env.Sequence)
}
assertFloatField(t, env, envelope.FieldLatitude, 30.590151)
assertFloatField(t, env, envelope.FieldLongitude, 121.069881)
assertFloatField(t, env, envelope.FieldSpeedKMH, 23.0)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 10241.2)
if env.Fields["direction_deg"] != uint16(79) {
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
}
location, ok := env.Parsed["location"].(map[string]any)
if !ok {
t.Fatalf("parsed location missing: %#v", env.Parsed)
}
additional, ok := location["additional"].([]map[string]any)
if !ok || len(additional) == 0 {
t.Fatalf("additional fields missing: %#v", location["additional"])
}
if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
t.Fatalf("unexpected first additional item: %#v", additional[0])
}
parsed, ok := additional[0]["parsed"].(map[string]any)
if !ok {
t.Fatalf("mileage parsed field missing: %#v", additional[0])
}
if parsed["raw_value_tenth_km"] != uint32(102412) || parsed["value"] != 10241.2 {
t.Fatalf("unexpected mileage parsed value: %#v", parsed)
}
}
func TestParseFrameSupportsJT8082019VersionedHeader(t *testing.T) {
stream, err := hex.DecodeString(real2019LocationFrame)
if err != nil {
t.Fatal(err)
}
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782914967713, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x0200" {
t.Fatalf("message id = %q", env.MessageID)
}
if env.Phone != "14894135060" {
t.Fatalf("phone = %q", env.Phone)
}
if env.Sequence != 1995 {
t.Fatalf("sequence = %d", env.Sequence)
}
assertFloatField(t, env, envelope.FieldLatitude, 30.570155)
assertFloatField(t, env, envelope.FieldLongitude, 117.178187)
assertFloatField(t, env, envelope.FieldSpeedKMH, 33.4)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 7868.9)
if env.Fields["altitude_m"] != uint16(13) {
t.Fatalf("altitude = %#v", env.Fields["altitude_m"])
}
if env.Fields["direction_deg"] != uint16(232) {
t.Fatalf("direction = %#v", env.Fields["direction_deg"])
}
header, ok := env.Parsed["header"].(map[string]any)
if !ok {
t.Fatalf("parsed header missing: %#v", env.Parsed)
}
if header["versioned"] != true || header["protocol_version"] != byte(1) {
t.Fatalf("unexpected versioned header: %#v", header)
}
if header["phone_bcd_hex"] != "00000000014894135060" {
t.Fatalf("phone bcd = %#v", header["phone_bcd_hex"])
}
}
func TestParseFrameParsesCommonLocationAdditionalItems(t *testing.T) {
stream, err := hex.DecodeString(fullAdditionalLocationFrame)
if err != nil {
t.Fatal(err)
}
frames, remainder, err := ExtractFrames(stream)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
env, err := ParseFrame(frames[0], 1782914967713, "127.0.0.1:808")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Phone != "012345678901" {
t.Fatalf("phone = %q", env.Phone)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.1)
location := env.Parsed["location"].(map[string]any)
additional := location["additional"].([]map[string]any)
byID := map[string]map[string]any{}
for _, item := range additional {
byID[item["id"].(string)] = item["parsed"].(map[string]any)
}
if byID["0x01"]["raw_value_tenth_km"] != uint32(11) || byID["0x01"]["value"] != 1.1 {
t.Fatalf("mileage parsed = %#v", byID["0x01"])
}
if byID["0x05"]["name"] != "tire_pressure" {
t.Fatalf("tire pressure missing: %#v", byID["0x05"])
}
tireValues := byID["0x05"]["values"].([]map[string]any)
if tireValues[0]["value"] != byte(0x37) || tireValues[0]["valid"] != true {
t.Fatalf("tire pressure first value = %#v", tireValues[0])
}
if byID["0x11"]["location_type"] != byte(0x42) || byID["0x11"]["area_id"] != uint32(0x42) {
t.Fatalf("overspeed parsed = %#v", byID["0x11"])
}
if byID["0x12"]["location_type"] != byte(0x4d) || byID["0x12"]["area_id"] != uint32(0x4d) || byID["0x12"]["direction"] != byte(0x4d) {
t.Fatalf("area route parsed = %#v", byID["0x12"])
}
if byID["0x13"]["road_section_id"] != uint32(0x58) || byID["0x13"]["driving_time_seconds"] != uint16(0x58) {
t.Fatalf("road section parsed = %#v", byID["0x13"])
}
ioBits := byID["0x2A"]["bits"].(map[string]bool)
if ioBits["deep_sleep"] || !ioBits["sleep"] {
t.Fatalf("io bits = %#v", ioBits)
}
}
func TestParseFrameRejectsTruncatedSubpackageHeader(t *testing.T) {
payload := []byte{0x02, 0x00, 0x20, 0x00}
payload = append(payload, encodeBCD("013307795425", 6)...)
payload = binary.BigEndian.AppendUint16(payload, 1)
payload = append(payload, checksum(payload))
_, err := ParseFrame(payload, 1782914967713, "115.231.168.135:43625")
if err == nil {
t.Fatal("expected truncated subpackage header to fail")
}
}
func TestParseLocationAppliesSouthWestStatusBits(t *testing.T) {
body := make([]byte, 28)
binary.BigEndian.PutUint32(body[4:8], (1<<2)|(1<<3))
binary.BigEndian.PutUint32(body[8:12], 12345678)
binary.BigEndian.PutUint32(body[12:16], 98765432)
location, err := parseLocation(body)
if err != nil {
t.Fatalf("parseLocation() error = %v", err)
}
if location.Latitude != -12.345678 {
t.Fatalf("latitude = %v", location.Latitude)
}
if location.Longitude != -98.765432 {
t.Fatalf("longitude = %v", location.Longitude)
}
}
func TestParseFrameParsesRegistrationIdentity(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTNG", 5)
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
body = appendFixedASCII(body, "DEV0001", 7)
body = append(body, 2)
body = append(body, []byte("TEST123")...)
env, err := ParseFrame(buildFrame(0x0100, "013079963379", 7, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x0100" {
t.Fatalf("message id = %q", env.MessageID)
}
if env.Phone != "013079963379" {
t.Fatalf("phone = %q", env.Phone)
}
if env.DeviceID != "DEV0001" {
t.Fatalf("device id = %q", env.DeviceID)
}
if env.Plate != "TEST123" {
t.Fatalf("plate = %q", env.Plate)
}
registration, ok := env.Parsed["registration"].(map[string]any)
if !ok {
t.Fatalf("registration missing: %#v", env.Parsed)
}
if registration["manufacturer"] != "YUTNG" || registration["device_type"] != "ZK6105CHEVNPG4" {
t.Fatalf("registration fields = %#v", registration)
}
if registration["plate_color"] != uint8(2) {
t.Fatalf("plate color = %#v", registration["plate_color"])
}
}
func TestParseFrameParsesJT8082019RegistrationIdentity(t *testing.T) {
body := make([]byte, 0, 84)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTONG00001", 11)
body = appendFixedASCII(body, "ZK6105CHEVNPG4-2019", 30)
body = appendFixedASCII(body, "DEV201900000000000000000000001", 30)
body = append(body, 2)
body = append(body, []byte("TEST2019")...)
env, err := ParseFrame(buildVersionedFrame(0x0100, "14894135060", 11, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Phone != "14894135060" {
t.Fatalf("phone = %q", env.Phone)
}
if env.DeviceID != "DEV201900000000000000000000001" {
t.Fatalf("device id = %q", env.DeviceID)
}
if env.Plate != "TEST2019" {
t.Fatalf("plate = %q", env.Plate)
}
registration, ok := env.Parsed["registration"].(map[string]any)
if !ok {
t.Fatalf("registration missing: %#v", env.Parsed)
}
if registration["schema_version"] != "2019" || registration["manufacturer"] != "YUTONG00001" {
t.Fatalf("registration fields = %#v", registration)
}
}
func TestParseFrameDecodesGBKRegistrationPlate(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTNG", 5)
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
body = appendFixedASCII(body, "DEV0002", 7)
body = append(body, 2)
body = append(body, []byte{0xBB, 0xA6, 'A', '5', '3', '3', '0', '1'}...)
env, err := ParseFrame(buildFrame(0x0100, "013079963380", 9, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.Plate != "沪A53301" {
t.Fatalf("plate = %q", env.Plate)
}
}
func TestParseFrameParsesJT8082019Authentication(t *testing.T) {
body := []byte{5}
body = append(body, []byte("TOKEN")...)
body = appendFixedASCII(body, "123456789012345", 15)
body = append(body, 4)
body = append(body, []byte("v1.2")...)
env, err := ParseFrame(buildVersionedFrame(0x0102, "14894135060", 12, body), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
auth, ok := env.Parsed["authentication"].(map[string]any)
if !ok {
t.Fatalf("authentication missing: %#v", env.Parsed)
}
if auth["token"] != "TOKEN" || auth["imei"] != "123456789012345" || auth["software_version"] != "v1.2" {
t.Fatalf("authentication = %#v", auth)
}
}
func TestParseFrameParsesAuthenticationToken(t *testing.T) {
env, err := ParseFrame(buildFrame(0x0102, "064646848757", 8, []byte("g7gps")), 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
if env.MessageID != "0x0102" {
t.Fatalf("message id = %q", env.MessageID)
}
auth, ok := env.Parsed["authentication"].(map[string]any)
if !ok {
t.Fatalf("authentication missing: %#v", env.Parsed)
}
if auth["token"] != "g7gps" {
t.Fatalf("token = %#v", auth["token"])
}
}
func TestAutoResponderBuildsRegisterAck(t *testing.T) {
body := make([]byte, 0, 46)
body = binary.BigEndian.AppendUint16(body, 16)
body = binary.BigEndian.AppendUint16(body, 32)
body = appendFixedASCII(body, "YUTNG", 5)
body = appendFixedASCII(body, "ZK6105CHEVNPG4", 20)
body = appendFixedASCII(body, "DEV0001", 7)
body = append(body, 2)
body = append(body, []byte("TEST123")...)
request := buildFrame(0x0100, "013079963379", 9, body)
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatal(err)
}
response, ok, err := NewAutoResponder("g7gps").Respond(request, env)
if err != nil {
t.Fatalf("Respond() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
frames, remainder, err := ExtractFrames(response)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
payload := frames[0]
if binary.BigEndian.Uint16(payload[0:2]) != 0x8100 {
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
}
if hex.EncodeToString(payload[4:10]) != "013079963379" {
t.Fatalf("phone bcd = %x", payload[4:10])
}
if binary.BigEndian.Uint16(payload[12:14]) != 9 || payload[14] != 0 || string(payload[15:20]) != "g7gps" {
t.Fatalf("unexpected register ack body: %x", payload[12:len(payload)-1])
}
}
func TestAutoResponderBuildsVersionedGeneralAck(t *testing.T) {
request := buildVersionedFrame(0x0200, "14894135060", 12, make([]byte, 28))
env, err := ParseFrame(request, 1782918600000, "115.231.168.135:43625")
if err != nil {
t.Fatal(err)
}
response, ok, err := NewAutoResponder("").Respond(request, env)
if err != nil {
t.Fatalf("Respond() error = %v", err)
}
if !ok {
t.Fatal("expected response")
}
frames, remainder, err := ExtractFrames(response)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected response split frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
payload := frames[0]
if binary.BigEndian.Uint16(payload[0:2]) != 0x8001 {
t.Fatalf("message id = %04x", binary.BigEndian.Uint16(payload[0:2]))
}
if binary.BigEndian.Uint16(payload[2:4])&0x4000 == 0 || payload[4] != 1 {
t.Fatalf("versioned header not preserved: %x", payload[:5])
}
if hex.EncodeToString(payload[5:15]) != "00000000014894135060" {
t.Fatalf("phone bcd = %x", payload[5:15])
}
bodyStart := 17
if binary.BigEndian.Uint16(payload[bodyStart:bodyStart+2]) != 12 ||
binary.BigEndian.Uint16(payload[bodyStart+2:bodyStart+4]) != 0x0200 ||
payload[bodyStart+4] != 0 {
t.Fatalf("unexpected general ack body: %x", payload[bodyStart:len(payload)-1])
}
}
func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
frame = append(frame, 0x7e)
frames, remainder, err := ExtractFrames(frame)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(frames) != 1 || len(remainder) != 0 {
t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
}
if hex.EncodeToString(frames[0][:len(payload)]) != hex.EncodeToString(payload) {
t.Fatalf("frame was not unescaped: %s", hex.EncodeToString(frames[0]))
}
}
func buildFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
payload := make([]byte, 0, 12+len(body)+1)
payload = binary.BigEndian.AppendUint16(payload, messageID)
payload = binary.BigEndian.AppendUint16(payload, uint16(len(body)))
payload = append(payload, encodeBCD(phone, 6)...)
payload = binary.BigEndian.AppendUint16(payload, sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return payload
}
func buildVersionedFrame(messageID uint16, phone string, sequence uint16, body []byte) []byte {
payload := make([]byte, 0, 17+len(body)+1)
payload = binary.BigEndian.AppendUint16(payload, messageID)
payload = binary.BigEndian.AppendUint16(payload, 0x4000|uint16(len(body)))
payload = append(payload, 1)
payload = append(payload, encodeBCD(phone, 10)...)
payload = binary.BigEndian.AppendUint16(payload, sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return payload
}
func appendFixedASCII(out []byte, value string, size int) []byte {
start := len(out)
out = append(out, make([]byte, size)...)
copy(out[start:], []byte(value))
return out
}
func encodeBCD(value string, size int) []byte {
out := make([]byte, size)
if len(value)%2 == 1 {
value = "0" + value
}
if len(value) > size*2 {
value = value[len(value)-size*2:]
}
for len(value) < size*2 {
value = "0" + value
}
for i := 0; i < size; i++ {
out[i] = (value[i*2]-'0')<<4 | (value[i*2+1] - '0')
}
return out
}
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
t.Helper()
got, ok := env.Fields[key].(float64)
if !ok {
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
}
if got != want {
t.Fatalf("field %s = %v, want %v", key, got, want)
}
}
func escape(payload []byte) []byte {
var out []byte
for _, value := range payload {
switch value {
case 0x7e:
out = append(out, 0x7d, 0x02)
case 0x7d:
out = append(out, 0x7d, 0x01)
default:
out = append(out, value)
}
}
return out
}

View File

@@ -0,0 +1,130 @@
package jt808
import (
"encoding/binary"
"errors"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrResponseHeaderTooShort = errors.New("jt808 response header too short")
const (
msgTerminalGeneralResponse = uint16(0x0001)
msgTerminalRegister = uint16(0x0100)
msgPlatformGeneralResponse = uint16(0x8001)
msgPlatformRegisterAck = uint16(0x8100)
)
type AutoResponder struct {
authCode string
}
func NewAutoResponder(authCode string) AutoResponder {
if authCode == "" {
authCode = "g7gps"
}
return AutoResponder{authCode: authCode}
}
func (r AutoResponder) Respond(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if env.ParseStatus == envelope.ParseBadFrame {
return nil, false, nil
}
header, err := parseResponseHeader(raw)
if err != nil {
return nil, false, err
}
if header.messageID == msgTerminalGeneralResponse {
return nil, false, nil
}
if header.messageID == msgTerminalRegister {
body := make([]byte, 3, 3+len(r.authCode))
binary.BigEndian.PutUint16(body[0:2], header.sequence)
body[2] = 0
body = append(body, []byte(r.authCode)...)
return encodeResponse(msgPlatformRegisterAck, header, body), true, nil
}
body := make([]byte, 5)
binary.BigEndian.PutUint16(body[0:2], header.sequence)
binary.BigEndian.PutUint16(body[2:4], header.messageID)
body[4] = 0
return encodeResponse(msgPlatformGeneralResponse, header, body), true, nil
}
type responseHeader struct {
messageID uint16
versioned bool
protocolVersion byte
phoneBCD []byte
sequence uint16
}
func parseResponseHeader(raw []byte) (responseHeader, error) {
if len(raw) < 12 {
return responseHeader{}, ErrResponseHeaderTooShort
}
props := binary.BigEndian.Uint16(raw[2:4])
versioned := props&0x4000 != 0
phoneStart := 4
phoneLen := 6
protocolVersion := byte(0)
if versioned {
if len(raw) < 17 {
return responseHeader{}, ErrResponseHeaderTooShort
}
protocolVersion = raw[4]
phoneStart = 5
phoneLen = 10
}
sequenceStart := phoneStart + phoneLen
if len(raw) < sequenceStart+2 {
return responseHeader{}, ErrResponseHeaderTooShort
}
phone := append([]byte(nil), raw[phoneStart:sequenceStart]...)
return responseHeader{
messageID: binary.BigEndian.Uint16(raw[0:2]),
versioned: versioned,
protocolVersion: protocolVersion,
phoneBCD: phone,
sequence: binary.BigEndian.Uint16(raw[sequenceStart : sequenceStart+2]),
}, nil
}
func encodeResponse(messageID uint16, header responseHeader, body []byte) []byte {
props := uint16(len(body)) & 0x03ff
if header.versioned {
props |= 0x4000
}
payload := make([]byte, 0, 4+1+len(header.phoneBCD)+2+len(body)+1)
payload = appendU16(payload, messageID)
payload = appendU16(payload, props)
if header.versioned {
payload = append(payload, header.protocolVersion)
}
payload = append(payload, header.phoneBCD...)
payload = appendU16(payload, header.sequence)
payload = append(payload, body...)
payload = append(payload, checksum(payload))
return frame(payload)
}
func appendU16(out []byte, value uint16) []byte {
return append(out, byte(value>>8), byte(value))
}
func frame(payload []byte) []byte {
out := []byte{0x7e}
for _, value := range payload {
switch value {
case 0x7e:
out = append(out, 0x7d, 0x02)
case 0x7d:
out = append(out, 0x7d, 0x01)
default:
out = append(out, value)
}
}
out = append(out, 0x7e)
return out
}

View File

@@ -0,0 +1,201 @@
package yutongmqtt
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var ErrInvalidJSON = errors.New("yutong mqtt invalid json")
func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS int64) (envelope.FrameEnvelope, error) {
root := map[string]any{}
decoder := json.NewDecoder(bytes.NewReader(payload))
decoder.UseNumber()
if err := decoder.Decode(&root); err != nil {
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %v", ErrInvalidJSON, err)
}
data := objectValue(root, "data")
if data == nil {
data = root
}
deviceID := firstText(root, "device", "deviceId", "terminalId", "terminalID", "imei", "IMEI", "mqttDeviceId")
externalVIN := firstText(root, "vin", "VIN", "externalVin", "vehicleVin")
vin := externalVIN
if vin == "" && looksLikeVIN(deviceID) {
vin = deviceID
}
plate := firstText(root, "plateNo", "carNo", "plate", "vehicleNo")
phone := firstText(root, "sim", "phone", "mobile")
eventTimeMS := parseTimeMS(firstText(root, "time", "deviceTime", "eventTime", "gpsTime"), receivedAtMS)
fields := fieldsFromData(data)
parsed := map[string]any{
"endpoint": endpoint,
"topic": topic,
"root": root,
"data": data,
}
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
MessageID: "MQTT",
VIN: vin,
VehicleKeyHint: deviceID,
Phone: phone,
DeviceID: deviceID,
Plate: plate,
SourceEndpoint: "mqtt://" + endpoint + topic,
EventTimeMS: eventTimeMS,
ReceivedAtMS: receivedAtMS,
RawText: string(payload),
Parsed: parsed,
Fields: fields,
ParseStatus: envelope.ParseOK,
}
env.EventID = env.StableEventID()
return env, nil
}
func fieldsFromData(data map[string]any) map[string]any {
fields := map[string]any{}
if speed, ok := firstFloat(data, "METER_SPEED", "speed", "speed_kmh"); ok {
fields[envelope.FieldSpeedKMH] = speed
}
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
fields[envelope.FieldTotalMileageKM] = normalizeTotalMileageKM(mileage)
}
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
fields[envelope.FieldSOCPercent] = soc
}
if longitude, ok := firstCoord(data, "LONGITUDE", "longitude", "lng"); ok {
fields[envelope.FieldLongitude] = longitude
}
if latitude, ok := firstCoord(data, "LATITUDE", "latitude", "lat"); ok {
fields[envelope.FieldLatitude] = latitude
}
if direction, ok := firstFloat(data, "GPSDirection", "direction", "direction_deg"); ok {
fields["direction_deg"] = direction
}
if gear, ok := firstFloat(data, "ON_GEAR", "gear"); ok {
fields["gear"] = gear
}
return fields
}
func objectValue(root map[string]any, key string) map[string]any {
value, ok := root[key]
if !ok {
return nil
}
typed, ok := value.(map[string]any)
if !ok {
return nil
}
return typed
}
func firstText(root map[string]any, keys ...string) string {
for _, key := range keys {
if value, ok := root[key]; ok && value != nil {
text := strings.TrimSpace(fmt.Sprint(value))
if text != "" && text != "<nil>" {
return text
}
}
}
return ""
}
func firstFloat(root map[string]any, keys ...string) (float64, bool) {
for _, key := range keys {
value, ok := root[key]
if !ok || value == nil {
continue
}
if parsed, ok := toFloat(value); ok {
return parsed, true
}
}
return 0, false
}
func firstCoord(root map[string]any, keys ...string) (float64, bool) {
value, ok := firstFloat(root, keys...)
if !ok {
return 0, false
}
if value == 0 || int64(value) == 999999 {
return 0, false
}
if value > 1000 || value < -1000 {
value = value / 1_000_000
}
return value, true
}
func toFloat(value any) (float64, bool) {
switch typed := value.(type) {
case json.Number:
parsed, err := typed.Float64()
return parsed, err == nil
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int64:
return float64(typed), true
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return parsed, err == nil
default:
return 0, false
}
}
func parseTimeMS(raw string, fallback int64) int64 {
raw = strings.TrimSpace(raw)
if raw == "" {
return fallback
}
for _, layout := range []string{
"2006-01-02 15:04:05.000",
"2006-01-02 15:04:05",
} {
if parsed, err := time.ParseInLocation(layout, raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
return parsed.UnixMilli()
}
}
if len(raw) == 14 {
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
return parsed.UnixMilli()
}
}
if parsed, err := time.Parse(time.RFC3339, raw); err == nil {
return parsed.UnixMilli()
}
if epoch, err := strconv.ParseInt(raw, 10, 64); err == nil {
if epoch > 10_000_000_000 {
return epoch
}
return epoch * 1000
}
return fallback
}
func looksLikeVIN(value string) bool {
value = strings.TrimSpace(value)
return len(value) == 17
}
func normalizeTotalMileageKM(value float64) float64 {
return value / 1000
}

View File

@@ -0,0 +1,112 @@
package yutongmqtt
import (
"errors"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
payload := []byte(`{
"device": "LTEST000000000001",
"time": "20260413100000",
"plateNo": "豫A12345",
"data": {
"METER_SPEED": 52.3,
"TOTAL_MILEAGE": 123456.7,
"BATTERY_CAPACITY_SOC": 70,
"LONGITUDE": 116397128,
"LATITUDE": 39916527,
"GPSDirection": 88
}
}`)
env, err := ParseMessage("endpoint-a", "/ytforward/shln/dev1", payload, 1782745114999)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if env.Protocol != envelope.ProtocolYutongMQTT {
t.Fatalf("protocol = %q", env.Protocol)
}
if env.VIN != "LTEST000000000001" || env.DeviceID != "LTEST000000000001" {
t.Fatalf("identity fields = vin:%q device:%q", env.VIN, env.DeviceID)
}
if env.Plate != "豫A12345" {
t.Fatalf("plate = %q", env.Plate)
}
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123.4567)
assertFloatField(t, env, envelope.FieldSOCPercent, 70)
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
assertFloatField(t, env, "direction_deg", 88)
if env.EventTimeMS == 1782745114999 {
t.Fatal("event time should come from device time")
}
if env.RawText == "" || env.Parsed["data"] == nil {
t.Fatalf("raw/parsed missing: %#v", env)
}
}
func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
payload := []byte(`{"device":"YT-DEVICE-001","time":"20260413100000","data":{"METER_SPEED":"12.3"}}`)
env, err := ParseMessage("endpoint-a", "/ytforward/shln/YT-DEVICE-001", payload, 1782745114999)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if env.VIN != "" {
t.Fatalf("non-vin device should not be treated as vin: %q", env.VIN)
}
if env.VehicleKey() != "YT-DEVICE-001" {
t.Fatalf("vehicle key = %q", env.VehicleKey())
}
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
}
func TestParseMessageNormalizesProductionMileageMeters(t *testing.T) {
payload := []byte(`{
"device":"LMRKH9AC3R1004101",
"time":"2026-07-02 01:02:04.085",
"data":{"TOTAL_MILEAGE":56905000,"METER_SPEED":19}
}`)
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 56905)
if env.EventTimeMS != 1782925324085 {
t.Fatalf("event time = %d, want 1782925324085", env.EventTimeMS)
}
}
func TestParseMessageTreatsSmallYutongMileageAsMeters(t *testing.T) {
payload := []byte(`{
"device":"LMRKH9AC3R1004101",
"time":"2026-07-02 01:02:04.085",
"data":{"TOTAL_MILEAGE":1024,"METER_SPEED":19}
}`)
env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782940000000)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
assertFloatField(t, env, envelope.FieldTotalMileageKM, 1.024)
}
func TestParseMessageRejectsMalformedJSON(t *testing.T) {
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
if !errors.Is(err, ErrInvalidJSON) {
t.Fatalf("error = %v", err)
}
}
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
t.Helper()
got, ok := env.Fields[key].(float64)
if !ok {
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
}
if got != want {
t.Fatalf("field %s = %v, want %v", key, got, want)
}
}

View File

@@ -0,0 +1,69 @@
package realtime
import (
"encoding/json"
"errors"
"net/http"
"strings"
"github.com/redis/go-redis/v9"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Handler struct {
repository *Repository
}
func NewHandler(repository *Repository) *Handler {
if repository == nil {
panic("realtime repository must not be nil")
}
return &Handler{repository: repository}
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
path := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/realtime/vehicles/"), "/")
parts := strings.Split(path, "/")
if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" {
writeError(w, http.StatusNotFound, "vehicle not found")
return
}
vin := parts[0]
switch {
case len(parts) == 1:
snapshot, err := h.repository.GetMerged(r.Context(), vin)
writeResult(w, snapshot, err)
case len(parts) == 2 && parts[1] == "online":
status, err := h.repository.IsOnline(r.Context(), vin)
writeResult(w, status, err)
case len(parts) == 3 && parts[1] == "protocols":
snapshot, err := h.repository.GetProtocol(r.Context(), vin, envelope.Protocol(strings.ToUpper(parts[2])))
writeResult(w, snapshot, err)
default:
writeError(w, http.StatusNotFound, "route not found")
}
}
func writeResult(w http.ResponseWriter, value any, err error) {
if err != nil {
if errors.Is(err, redis.Nil) {
writeError(w, http.StatusNotFound, "not found")
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(value)
}
func writeError(w http.ResponseWriter, status int, message string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
}

View File

@@ -0,0 +1,41 @@
package realtime
import (
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Snapshot struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Protocol envelope.Protocol `json:"protocol,omitempty"`
Protocols []envelope.Protocol `json:"protocols,omitempty"`
EventID string `json:"event_id,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
UpdatedAtMS int64 `json:"updated_at_ms"`
}
type OnlineStatus struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
Online bool `json:"online"`
LastSeenMS int64 `json:"last_seen_ms"`
Protocols []envelope.Protocol `json:"protocols,omitempty"`
TTLSeconds int64 `json:"ttl_seconds"`
}
type Config struct {
OnlineTTL time.Duration
}
func (c Config) ttl() time.Duration {
if c.OnlineTTL <= 0 {
return 10 * time.Minute
}
return c.OnlineTTL
}

View File

@@ -0,0 +1,232 @@
package realtime
import (
"context"
"encoding/json"
"errors"
"sort"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type Repository struct {
client *redis.Client
cfg Config
}
func NewRepository(client *redis.Client, cfg Config) *Repository {
if client == nil {
panic("redis client must not be nil")
}
return &Repository{client: client, cfg: cfg}
}
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
vin := strings.TrimSpace(env.VIN)
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil
}
nowMS := time.Now().UnixMilli()
eventMS := env.EventTimeMS
if eventMS <= 0 {
eventMS = env.ReceivedAtMS
}
protocolSnapshot := Snapshot{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
EventID: env.StableEventID(),
EventTimeMS: eventMS,
ReceivedAtMS: env.ReceivedAtMS,
SourceEndpoint: env.SourceEndpoint,
Fields: cloneFields(env.Fields),
FieldTimesMS: fieldTimes(env.Fields, eventMS),
UpdatedAtMS: nowMS,
}
if err := r.setJSON(ctx, protocolKey(vehicleKey, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
return err
}
protocols, err := r.addProtocol(ctx, vehicleKey, env.Protocol)
if err != nil {
return err
}
merged, err := r.GetMerged(ctx, vehicleKey)
if err != nil && !errors.Is(err, redis.Nil) {
return err
}
if merged.VehicleKey == "" {
merged = Snapshot{VehicleKey: vehicleKey, VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
} else if merged.VIN == "" && vin != "" {
merged.VIN = vin
}
mergeFields(&merged, env.Fields, eventMS)
if eventMS >= merged.EventTimeMS {
merged.EventTimeMS = eventMS
merged.EventID = env.StableEventID()
merged.ReceivedAtMS = env.ReceivedAtMS
merged.SourceEndpoint = env.SourceEndpoint
}
merged.Protocols = protocols
merged.UpdatedAtMS = nowMS
if err := r.setJSON(ctx, mergedKey(vehicleKey), merged, r.cfg.ttl()); err != nil {
return err
}
online := OnlineStatus{
VehicleKey: vehicleKey,
VIN: vin,
Online: true,
LastSeenMS: env.ReceivedAtMS,
Protocols: protocols,
TTLSeconds: int64(r.cfg.ttl().Seconds()),
}
if err := r.setJSON(ctx, onlineKey(vehicleKey), online, r.cfg.ttl()); err != nil {
return err
}
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vehicleKey}).Err()
}
func (r *Repository) GetMerged(ctx context.Context, vehicleKey string) (Snapshot, error) {
return r.getSnapshot(ctx, mergedKey(vehicleKey))
}
func (r *Repository) GetProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) (Snapshot, error) {
return r.getSnapshot(ctx, protocolKey(vehicleKey, protocol))
}
func (r *Repository) IsOnline(ctx context.Context, vehicleKey string) (OnlineStatus, error) {
vehicleKey = strings.TrimSpace(vehicleKey)
var status OnlineStatus
payload, err := r.client.Get(ctx, onlineKey(vehicleKey)).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}, nil
}
return status, err
}
if err := json.Unmarshal(payload, &status); err != nil {
return status, err
}
ttl := r.client.TTL(ctx, onlineKey(vehicleKey)).Val()
status.Online = ttl > 0
status.TTLSeconds = int64(ttl.Seconds())
return status, nil
}
func (r *Repository) getSnapshot(ctx context.Context, key string) (Snapshot, error) {
var snapshot Snapshot
payload, err := r.client.Get(ctx, key).Bytes()
if err != nil {
return snapshot, err
}
return snapshot, json.Unmarshal(payload, &snapshot)
}
func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl time.Duration) error {
payload, err := json.Marshal(value)
if err != nil {
return err
}
return r.client.Set(ctx, key, payload, ttl).Err()
}
func (r *Repository) addProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
key := protocolsKey(vehicleKey)
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
return nil, err
}
_ = r.client.Expire(ctx, key, r.cfg.ttl()).Err()
values, err := r.client.SMembers(ctx, key).Result()
if err != nil {
return nil, err
}
sort.Strings(values)
out := make([]envelope.Protocol, 0, len(values))
for _, value := range values {
out = append(out, envelope.Protocol(value))
}
return out, nil
}
func mergeFields(snapshot *Snapshot, fields map[string]any, eventMS int64) {
if snapshot.Fields == nil {
snapshot.Fields = map[string]any{}
}
if snapshot.FieldTimesMS == nil {
snapshot.FieldTimesMS = map[string]int64{}
}
for key, value := range fields {
if key == envelope.FieldTotalMileageKM && !positiveNumber(value) {
continue
}
if eventMS >= snapshot.FieldTimesMS[key] {
snapshot.Fields[key] = value
snapshot.FieldTimesMS[key] = eventMS
}
}
}
func positiveNumber(value any) bool {
switch typed := value.(type) {
case float64:
return typed > 0
case float32:
return typed > 0
case int:
return typed > 0
case int64:
return typed > 0
case uint16:
return typed > 0
case uint32:
return typed > 0
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return err == nil && parsed > 0
default:
return false
}
}
func cloneFields(fields map[string]any) map[string]any {
if len(fields) == 0 {
return map[string]any{}
}
out := make(map[string]any, len(fields))
for key, value := range fields {
out[key] = value
}
return out
}
func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
out := make(map[string]int64, len(fields))
for key := range fields {
out[key] = eventMS
}
return out
}
func mergedKey(vehicleKey string) string {
return "vehicle:latest:" + strings.TrimSpace(vehicleKey)
}
func protocolKey(vehicleKey string, protocol envelope.Protocol) string {
return "vehicle:latest:" + strings.TrimSpace(vehicleKey) + ":" + string(protocol)
}
func onlineKey(vehicleKey string) string {
return "vehicle:online:" + strings.TrimSpace(vehicleKey)
}
func protocolsKey(vehicleKey string) string {
return "vehicle:protocols:" + strings.TrimSpace(vehicleKey)
}

View File

@@ -0,0 +1,250 @@
package realtime
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{
envelope.FieldLongitude: 121.1,
envelope.FieldTotalMileageKM: 10.5,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "VIN001",
EventTimeMS: 900,
ReceivedAtMS: 1200,
Fields: map[string]any{
envelope.FieldLongitude: 120.0,
envelope.FieldLatitude: 30.5,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
merged, err := repo.GetMerged(ctx, "VIN001")
if err != nil {
t.Fatalf("GetMerged() error = %v", err)
}
if merged.Fields[envelope.FieldLongitude] != 121.1 {
t.Fatalf("older longitude overwrote newer value: %#v", merged.Fields)
}
if merged.Fields[envelope.FieldLatitude] != 30.5 {
t.Fatalf("new latitude missing: %#v", merged.Fields)
}
if got := protocolNames(merged.Protocols); strings.Join(got, ",") != "GB32960,JT808" {
t.Fatalf("merged protocols = %#v", got)
}
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
if err != nil {
t.Fatalf("GetProtocol() error = %v", err)
}
if protocol.Fields[envelope.FieldTotalMileageKM] != 10.5 {
t.Fatalf("protocol snapshot = %#v", protocol)
}
}
func TestRepositoryOnlineStatus(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
status, err := repo.IsOnline(ctx, "VIN001")
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if status.Online {
t.Fatal("empty vin should be offline")
}
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
status, err = repo.IsOnline(ctx, "VIN001")
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if !status.Online || status.LastSeenMS != 1100 {
t.Fatalf("online status = %#v", status)
}
}
func TestRepositoryUpdatesPhoneOnlyVehicleKey(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811170",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10003.7,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
vehicleKey := "JT808:013307811170"
merged, err := repo.GetMerged(ctx, vehicleKey)
if err != nil {
t.Fatalf("GetMerged() error = %v", err)
}
if merged.VehicleKey != vehicleKey || merged.VIN != "" {
t.Fatalf("unexpected identity: %#v", merged)
}
if merged.Fields[envelope.FieldTotalMileageKM] != 10003.7 {
t.Fatalf("merged fields = %#v", merged.Fields)
}
status, err := repo.IsOnline(ctx, vehicleKey)
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if !status.Online || status.VehicleKey != vehicleKey {
t.Fatalf("online status = %#v", status)
}
}
func TestRepositoryDoesNotOverwritePositiveMileageWithZero(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{
envelope.FieldTotalMileageKM: 12345.6,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 2000,
ReceivedAtMS: 2100,
Fields: map[string]any{
envelope.FieldSpeedKMH: 22.0,
envelope.FieldTotalMileageKM: 0,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
merged, err := repo.GetMerged(ctx, "VIN001")
if err != nil {
t.Fatalf("GetMerged() error = %v", err)
}
if merged.Fields[envelope.FieldTotalMileageKM] != 12345.6 {
t.Fatalf("zero mileage overwrote positive value: %#v", merged.Fields)
}
if merged.Fields[envelope.FieldSpeedKMH] != 22.0 {
t.Fatalf("new speed should still merge: %#v", merged.Fields)
}
}
func TestHandlerReturnsMergedSnapshot(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/VIN001", nil)
rec := httptest.NewRecorder()
NewHandler(repo).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
if !stringsContains(rec.Body.String(), `"vin":"VIN001"`) {
t.Fatalf("unexpected body: %s", rec.Body.String())
}
}
func TestHandlerReturnsPhoneOnlyVehicleKeySnapshot(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811170",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/JT808:013307811170", nil)
rec := httptest.NewRecorder()
NewHandler(repo).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
if !stringsContains(rec.Body.String(), `"vehicle_key":"JT808:013307811170"`) {
t.Fatalf("unexpected body: %s", rec.Body.String())
}
}
func newTestRepository(t *testing.T) (*Repository, func()) {
t.Helper()
server, err := miniredis.Run()
if err != nil {
t.Fatalf("miniredis.Run() error = %v", err)
}
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
return NewRepository(client, Config{OnlineTTL: time.Minute}), func() {
_ = client.Close()
server.Close()
}
}
func stringsContains(value string, pattern string) bool {
return strings.Contains(value, pattern)
}
func protocolNames(protocols []envelope.Protocol) []string {
names := make([]string, 0, len(protocols))
for _, protocol := range protocols {
names = append(names, string(protocol))
}
return names
}

View File

@@ -0,0 +1,275 @@
package stats
import (
"context"
"database/sql"
"errors"
"strconv"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
const (
MetricDailyMileageKM = "daily_mileage_km"
MetricDailyTotalMileageKM = "daily_total_mileage_km"
)
type Execer interface {
ExecContext(context.Context, string, ...any) (sql.Result, error)
}
type Writer struct {
exec Execer
loc *time.Location
}
type MetricSample struct {
VehicleKey string
VIN string
Protocol envelope.Protocol
StatDate string
MetricKey string
MetricValue float64
TotalMileageKM float64
}
func NewWriter(exec Execer, loc *time.Location) *Writer {
if exec == nil {
panic("stats execer must not be nil")
}
if loc == nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
return &Writer{exec: exec, loc: loc}
}
func (w *Writer) EnsureSchema(ctx context.Context) error {
if _, err := w.exec.ExecContext(ctx, DailyMetricTableSQL); err != nil {
return err
}
if db, ok := w.exec.(metadataDB); ok {
return migrateDailyMetricTable(ctx, db)
}
return nil
}
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
samples, err := SamplesFromEnvelope(env, w.loc)
if err != nil {
return err
}
for _, sample := range samples {
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL,
sample.VehicleKey,
sample.VIN,
sample.StatDate,
string(sample.Protocol),
sample.MetricKey,
sample.MetricValue,
sample.TotalMileageKM,
sample.TotalMileageKM); err != nil {
return err
}
}
return nil
}
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
vin := strings.TrimSpace(env.VIN)
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil, nil
}
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
if !ok {
return nil, nil
}
if totalMileage <= 0 {
return nil, nil
}
if loc == nil {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
eventMS := env.EventTimeMS
if eventMS <= 0 {
eventMS = env.ReceivedAtMS
}
if eventMS <= 0 {
return nil, errors.New("event or received time is required")
}
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
return []MetricSample{
{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
StatDate: statDate,
MetricKey: MetricDailyMileageKM,
MetricValue: 0,
TotalMileageKM: totalMileage,
},
{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
StatDate: statDate,
MetricKey: MetricDailyTotalMileageKM,
MetricValue: totalMileage,
TotalMileageKM: totalMileage,
},
}, nil
}
const upsertDailyMetricSQL = `
INSERT INTO vehicle_daily_metric
(vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit,
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
VALUES (?, ?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
ON DUPLICATE KEY UPDATE
vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
first_total_mileage_km = CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END,
latest_total_mileage_km = CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
END,
metric_value = CASE
WHEN metric_key = 'daily_mileage_km'
THEN GREATEST(
CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE latest_total_mileage_km
END,
VALUES(latest_total_mileage_km)
)
- CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END
WHEN metric_key = 'daily_total_mileage_km'
THEN GREATEST(
CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE latest_total_mileage_km
END,
VALUES(latest_total_mileage_km)
)
ELSE VALUES(metric_value)
END,
sample_count = sample_count + 1,
updated_at = CURRENT_TIMESTAMP
`
type metadataDB interface {
Execer
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func migrateDailyMetricTable(ctx context.Context, db metadataDB) error {
columnExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND column_name = 'vehicle_key'`)
if err != nil {
return err
}
if !columnExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD COLUMN vehicle_key VARCHAR(96) NOT NULL DEFAULT '' AFTER id`); err != nil {
return err
}
}
if _, err := db.ExecContext(ctx, `UPDATE vehicle_daily_metric SET vehicle_key = vin WHERE vehicle_key = ''`); err != nil {
return err
}
vehicleIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric_vehicle'`)
if err != nil {
return err
}
if !vehicleIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key)`); err != nil {
return err
}
}
oldIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric'`)
if err != nil {
return err
}
if oldIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric DROP INDEX uk_daily_metric`); err != nil {
return err
}
}
vinIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'idx_vin'`)
if err != nil {
return err
}
if !vinIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD KEY idx_vin (vin)`); err != nil {
return err
}
}
return nil
}
func informationSchemaExists(ctx context.Context, db metadataDB, query string) (bool, error) {
var count int
if err := db.QueryRowContext(ctx, query).Scan(&count); err != nil {
return false, err
}
return count > 0, nil
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
if env.Fields == nil {
return 0, false
}
value, ok := env.Fields[key]
if !ok || value == nil {
return 0, false
}
switch typed := value.(type) {
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int64:
return float64(typed), true
case uint16:
return float64(typed), true
case uint32:
return float64(typed), true
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return parsed, err == nil
default:
return 0, false
}
}

View File

@@ -0,0 +1,162 @@
package stats
import (
"context"
"database/sql"
"strings"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "LNBVIN00000000001",
EventTimeMS: time.Date(2026, 7, 1, 8, 30, 0, 0, loc).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: 10241.2,
},
}
samples, err := SamplesFromEnvelope(env, loc)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 2 {
t.Fatalf("sample count = %d", len(samples))
}
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
}
if samples[0].VehicleKey != "LNBVIN00000000001" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 {
t.Fatalf("unexpected daily total sample: %#v", samples[1])
}
if samples[0].StatDate != "2026-07-01" {
t.Fatalf("stat date = %q", samples[0].StatDate)
}
}
func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307811254",
EventTimeMS: time.Date(2026, 7, 2, 0, 3, 0, 0, loc).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: 10985.7,
},
}, loc)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 2 {
t.Fatalf("sample count = %d", len(samples))
}
if samples[0].VehicleKey != "JT808:013307811254" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[0].VIN != "" {
t.Fatalf("vin should stay empty for unresolved JT808 identity, got %q", samples[0].VIN)
}
}
func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
}, nil)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 0 {
t.Fatalf("expected no samples without vehicle key, got %#v", samples)
}
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "LNBVIN00000000002",
Fields: map[string]any{},
}, nil)
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 0 {
t.Fatalf("expected no samples without mileage, got %#v", samples)
}
}
func TestSamplesFromEnvelopeSkipsNonPositiveMileage(t *testing.T) {
for _, value := range []any{0, 0.0, -1.0, "0"} {
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "LNBVIN00000000001",
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: value,
},
}, nil)
if err != nil {
t.Fatalf("SamplesFromEnvelope(%#v) error = %v", value, err)
}
if len(samples) != 0 {
t.Fatalf("expected no samples for non-positive mileage %#v, got %#v", value, samples)
}
}
}
func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
if err := writer.EnsureSchema(context.Background()); err != nil {
t.Fatalf("EnsureSchema() error = %v", err)
}
if err := writer.Append(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "LNBVIN00000000002",
EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
Fields: map[string]any{
envelope.FieldTotalMileageKM: "10000.0",
},
}); err != nil {
t.Fatalf("Append() error = %v", err)
}
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") {
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
}
if !strings.Contains(exec.calls[0].query, "vehicle_key") {
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
}
if len(exec.calls) != 3 {
t.Fatalf("exec calls = %d", len(exec.calls))
}
if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query)
}
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
}
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
t.Fatalf("first upsert arg should be vehicle_key, got %#v", got)
}
}
type execCall struct {
query string
args []any
}
type recordingExec struct {
calls []execCall
}
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
e.calls = append(e.calls, execCall{query: query, args: args})
return nil, nil
}

View File

@@ -0,0 +1,326 @@
package stats
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
)
type Queryer interface {
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
}
type MetricQuery struct {
VehicleKey string
VIN string
Protocol string
MetricKey string
DateFrom string
DateTo string
Limit int
Offset int
}
type MetricRow struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
MetricKey string `json:"metric_key"`
MetricValue float64 `json:"metric_value"`
MetricUnit string `json:"metric_unit"`
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
SampleCount int64 `json:"sample_count"`
CalculationMethod string `json:"calculation_method"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type MetricRepository struct {
db Queryer
}
func NewMetricRepository(db Queryer) *MetricRepository {
if db == nil {
panic("metric query db must not be nil")
}
return &MetricRepository{db: db}
}
func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]MetricRow, error) {
query = normalizeMetricQuery(query)
sqlText, args := buildMetricSQL(query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]MetricRow, 0)
for rows.Next() {
var row MetricRow
var statDate scanDate
var createdAt scanDateTime
var updatedAt scanDateTime
var first sql.NullFloat64
var latest sql.NullFloat64
if err := rows.Scan(
&row.VehicleKey,
&row.VIN,
&statDate,
&row.Protocol,
&row.MetricKey,
&row.MetricValue,
&row.MetricUnit,
&first,
&latest,
&row.SampleCount,
&row.CalculationMethod,
&createdAt,
&updatedAt,
); err != nil {
return nil, err
}
row.StatDate = statDate.String
row.CreatedAt = createdAt.String
row.UpdatedAt = updatedAt.String
if first.Valid {
row.FirstTotalMileageKM = &first.Float64
}
if latest.Valid {
row.LatestTotalMileageKM = &latest.Float64
}
out = append(out, row)
}
return out, rows.Err()
}
func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, error) {
query = normalizeMetricQuery(query)
sqlText, args := buildMetricCountSQL(query)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
if err != nil {
return 0, err
}
defer rows.Close()
var total int64
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, rows.Err()
}
func normalizeMetricQuery(query MetricQuery) MetricQuery {
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.MetricKey = strings.TrimSpace(query.MetricKey)
query.DateFrom = strings.TrimSpace(query.DateFrom)
query.DateTo = strings.TrimSpace(query.DateTo)
if query.Limit <= 0 {
query.Limit = 50
}
return query
}
func buildMetricSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?"
args = append(args, query.Limit, query.Offset)
return sqlText, args
}
func buildMetricCountSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT COUNT(*) FROM vehicle_daily_metric`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
return sqlText, args
}
func buildMetricWhere(query MetricQuery) ([]string, []any) {
var where []string
var args []any
add := func(clause string, value any) {
where = append(where, clause)
args = append(args, value)
}
if query.VIN != "" {
add("vin = ?", query.VIN)
}
if query.VehicleKey != "" {
add("vehicle_key = ?", query.VehicleKey)
}
if query.Protocol != "" {
add("protocol = ?", query.Protocol)
}
if query.MetricKey != "" {
add("metric_key = ?", query.MetricKey)
}
if query.DateFrom != "" {
add("stat_date >= ?", query.DateFrom)
}
if query.DateTo != "" {
add("stat_date <= ?", query.DateTo)
}
return where, args
}
type MetricHandler struct {
repository *MetricRepository
}
func NewMetricHandler(repository *MetricRepository) *MetricHandler {
if repository == nil {
panic("metric repository must not be nil")
}
return &MetricHandler{repository: repository}
}
func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeMetricError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
if strings.Trim(r.URL.Path, "/") != "api/stats/daily-metrics" {
writeMetricError(w, http.StatusNotFound, "route not found")
return
}
query, err := parseMetricQuery(r)
if err != nil {
writeMetricError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeMetricError(w, http.StatusInternalServerError, err.Error())
return
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeMetricError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
"total": total,
"limit": query.Limit,
"offset": query.Offset,
})
}
func parseMetricQuery(r *http.Request) (MetricQuery, error) {
values := r.URL.Query()
limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit")
if err != nil {
return MetricQuery{}, err
}
offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset")
if err != nil {
return MetricQuery{}, err
}
query := MetricQuery{
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Protocol: values.Get("protocol"),
MetricKey: values.Get("metricKey"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDate(query.DateFrom) || !validDate(query.DateTo) {
return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD")
}
return normalizeMetricQuery(query), nil
}
func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return fallback, nil
}
value, err := strconv.Atoi(raw)
if err != nil || value < min || value > max {
return 0, fmt.Errorf("%s must be between %d and %d", name, min, max)
}
return value, nil
}
func validDate(value string) bool {
value = strings.TrimSpace(value)
if value == "" {
return true
}
if len(value) != len("2006-01-02") {
return false
}
for i, r := range value {
switch i {
case 4, 7:
if r != '-' {
return false
}
default:
if r < '0' || r > '9' {
return false
}
}
}
return true
}
func writeMetricError(w http.ResponseWriter, status int, message string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
}
type scanDate struct {
String string
}
func (s *scanDate) Scan(value any) error {
s.String = formatSQLTime(value, "2006-01-02")
return nil
}
type scanDateTime struct {
String string
}
func (s *scanDateTime) Scan(value any) error {
s.String = formatSQLTime(value, "2006-01-02 15:04:05")
return nil
}
func formatSQLTime(value any, layout string) string {
switch typed := value.(type) {
case nil:
return ""
case time.Time:
return typed.Format(layout)
case []byte:
return strings.TrimSpace(string(typed))
case string:
return strings.TrimSpace(typed)
default:
return strings.TrimSpace(fmt.Sprint(typed))
}
}

View File

@@ -0,0 +1,185 @@
package stats
import (
"context"
"database/sql"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
)
func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km",
12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
))
repository := NewMetricRepository(db)
rows, err := repository.Query(context.Background(), MetricQuery{
VIN: "LKLG7C4E3NA774736",
Protocol: "JT808",
DateFrom: "2026-07-01",
DateTo: "2026-07-01",
Limit: 20,
})
if err != nil {
t.Fatalf("Query() error = %v", err)
}
if len(rows) != 1 {
t.Fatalf("row count = %d", len(rows))
}
if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 {
t.Fatalf("unexpected row: %#v", rows[0])
}
if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
t.Fatalf("vehicle key = %q", rows[0].VehicleKey)
}
if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" {
t.Fatalf("unexpected time formatting: %#v", rows[0])
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km",
53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25",
))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":42`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("JT808:013307811254", "JT808").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("JT808:013307811254", "JT808", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}).AddRow(
"JT808:013307811254", "", "2026-07-02", "JT808", MetricDailyTotalMileageKM, 10985.7, "km",
10985.7, 10985.7, 1, "TOTAL_MILEAGE_DIFF", "2026-07-02 00:03:00", "2026-07-02 00:03:00",
))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vehicleKey=JT808:013307811254&protocol=JT808", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"metric_value":10985.7`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric").
WithArgs("YUTONG_MQTT").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric").
WithArgs("YUTONG_MQTT", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method",
"created_at", "updated_at",
}))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?protocol=YUTONG_MQTT", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"items":[]`) || strings.Contains(body, `"items":null`) {
t.Fatalf("expected empty items array, got: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerRejectsInvalidPagination(t *testing.T) {
handler := NewMetricHandler(NewMetricRepository(&sql.DB{}))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?limit=2001", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
}

View File

@@ -0,0 +1,22 @@
package stats
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vehicle_key VARCHAR(96) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '',
stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL,
metric_key VARCHAR(64) NOT NULL,
metric_value DECIMAL(18,3) NOT NULL,
metric_unit VARCHAR(16) NOT NULL,
first_total_mileage_km DECIMAL(18,3) NULL,
latest_total_mileage_km DECIMAL(18,3) NULL,
sample_count BIGINT NOT NULL DEFAULT 0,
calculation_method VARCHAR(64) NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key),
KEY idx_vin (vin),
KEY idx_stat_date (stat_date),
KEY idx_protocol_metric (protocol, metric_key)
)`

View File

@@ -35,7 +35,7 @@
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>sink-mq</artifactId>
<artifactId>sink-kafka</artifactId>
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
@@ -49,10 +49,6 @@
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
</dependency>
<dependency>
<groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
@@ -66,17 +62,6 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
<goal>build-info</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

View File

@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
@SpringBootApplication
public class Gb32960IngestApplication {
private static final Logger log = LoggerFactory.getLogger(Gb32960IngestApplication.class);

View File

@@ -28,12 +28,6 @@ spring:
server:
port: ${HTTP_PORT:20100}
springdoc:
swagger-ui:
path: /swagger-ui.html
api-docs:
path: /v3/api-docs
lingniu:
ingest:
gb32960:
@@ -49,10 +43,15 @@ lingniu:
whitelist: []
platforms:
- username: ${GB32960_PLATFORM_USER_HYUNDAI:Hyundai}
password: ${GB32960_PLATFORM_PWD_HYUNDAI:f2e3445d7cda409fb4f278f6fb890734}
password: ${GB32960_PLATFORM_PWD_HYUNDAI:}
allowed-ips:
- ${GB32960_PLATFORM_IP_HYUNDAI:}
description: 外部下级平台 - Hyundai
- username: ${GB32960_PLATFORM_USER_YUEJIN:YueJin}
password: ${GB32960_PLATFORM_PWD_YUEJIN:}
allowed-ips:
- ${GB32960_PLATFORM_IP_YUEJIN:}
description: 外部下级平台 - YueJin
tls:
enabled: ${GB32960_TLS_ENABLED:false}
cert-path: ${GB32960_TLS_CERT:}
@@ -64,6 +63,7 @@ lingniu:
match:
platform-accounts:
- Hyundai
- YueJin
vin-prefixes: []
vins: []
parse:
@@ -82,10 +82,8 @@ lingniu:
rate-limit:
per-vin-qps: 50
session:
store: ${SESSION_STORE:redis}
ttl: ${SESSION_TTL:30m}
identity:
store: ${VEHICLE_IDENTITY_STORE:mysql}
mysql:
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
@@ -93,9 +91,8 @@ lingniu:
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
enabled: ${KAFKA_ENABLED:true}
type: kafka
kafka:
enabled: true
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd
linger-ms: 20
@@ -117,18 +114,16 @@ lingniu:
enabled: ${SINK_ARCHIVE_ENABLED:true}
type: local
path: ${SINK_ARCHIVE_PATH:./archive/}
event-history:
enabled: false
vehicle-state:
enabled: false
vehicle-stat:
enabled: false
management:
endpoints:
web:
exposure:
include: health,info,metrics,prometheus,env,gb32960sessions
include: health,info,metrics,prometheus,gb32960sessions
endpoint:
health:
probes:
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
health:
redis:
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}

View File

@@ -6,13 +6,14 @@ import com.lingniu.ingest.identity.config.VehicleIdentityAutoConfiguration;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
import com.lingniu.ingest.session.SessionStore;
import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -30,27 +31,23 @@ class Gb32960IngestAppCompositionTest {
IngestCoreAutoConfiguration.class,
SessionCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class,
Gb32960AutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true)
.withBean("kafkaProducer", KafkaProducer.class, Gb32960IngestAppCompositionTest::kafkaProducer)
.withBean(SessionStore.class, () -> mock(SessionStore.class))
.withPropertyValues(
"lingniu.ingest.gb32960.enabled=true",
"lingniu.ingest.gb32960.server.enabled=true",
"lingniu.ingest.gb32960.port=0",
"lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.session.store=memory",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-gb32960",
"lingniu.ingest.event-history.enabled=false",
"lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=false");
"lingniu.ingest.event-history.enabled=false");
@Test
void createsGb32960ListenerWithoutHistoryStorageBoundaries() {
@@ -63,7 +60,8 @@ class Gb32960IngestAppCompositionTest {
assertThat(context).hasSingleBean(ArchiveStore.class);
assertThat(context).hasSingleBean(RawArchiveEventSink.class);
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
});
}

View File

@@ -15,6 +15,8 @@ class Gb32960IngestAppDefaultsTest {
@Test
void applicationDefaultsKeepGb32960IngestAsTcpKafkaProducerOnly() throws IOException {
Properties properties = applicationProperties();
String legacyKafkaPrefix = "lingniu.ingest.sink." + "mq.";
String kafkaTypeProperty = "lingniu.ingest.sink.kafka." + "type";
assertThat(properties)
.containsEntry("spring.application.name", "gb32960-ingest-app")
@@ -28,23 +30,48 @@ class Gb32960IngestAppDefaultsTest {
.containsEntry("lingniu.ingest.gb32960.enabled", true)
.containsEntry("lingniu.ingest.gb32960.server.enabled", true)
.containsEntry("lingniu.ingest.gb32960.port", "${GB32960_PORT:32960}")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
.containsEntry("lingniu.ingest.gb32960.auth.platforms[0].username",
"${GB32960_PLATFORM_USER_HYUNDAI:Hyundai}")
.containsEntry("lingniu.ingest.gb32960.auth.platforms[0].password",
"${GB32960_PLATFORM_PWD_HYUNDAI:}")
.containsEntry("lingniu.ingest.gb32960.auth.platforms[1].username",
"${GB32960_PLATFORM_USER_YUEJIN:YueJin}")
.containsEntry("lingniu.ingest.gb32960.auth.platforms[1].password",
"${GB32960_PLATFORM_PWD_YUEJIN:}")
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[0]",
"Hyundai")
.containsEntry("lingniu.ingest.gb32960.vendor-extensions[0].match.platform-accounts[1]",
"YueJin")
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
.containsEntry("lingniu.ingest.identity.mysql.table-name",
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
.containsEntry("lingniu.ingest.event-history.enabled", false)
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
.containsEntry("management.endpoints.web.exposure.include",
"health,info,metrics,prometheus,env,gb32960sessions");
"health,info,metrics,prometheus,gb32960sessions");
assertThat(properties.stringPropertyNames())
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."));
assertThat(applicationYaml()).doesNotContain("event-file-store:");
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
.noneMatch(name -> name.startsWith(legacyKafkaPrefix))
.noneMatch(name -> name.equals(kafkaTypeProperty))
.noneMatch(name -> name.equals("lingniu.ingest.session.store"))
.noneMatch(name -> name.equals("lingniu.ingest.identity.store"));
assertThat(applicationYaml())
.doesNotContain("event-history:")
.doesNotContain("event-file-store:")
.doesNotContain("vehicle-state:")
.doesNotContain("vehicle-stat:")
.doesNotContain("springdoc:");
assertThat(applicationYaml())
.contains("password: ${GB32960_PLATFORM_PWD_HYUNDAI:}")
.contains("password: ${GB32960_PLATFORM_PWD_YUEJIN:}")
.doesNotContain("SESSION_STORE")
.doesNotContain("VEHICLE_IDENTITY_STORE")
.doesNotContain("KAFKA_ENABLED");
}
private static Properties applicationProperties() {

View File

@@ -35,7 +35,7 @@
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>sink-mq</artifactId>
<artifactId>sink-kafka</artifactId>
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
@@ -49,10 +49,6 @@
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
</dependency>
<dependency>
<groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
@@ -66,17 +62,6 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
<goal>build-info</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

View File

@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
@SpringBootApplication
public class Jt808IngestApplication {
private static final Logger log = LoggerFactory.getLogger(Jt808IngestApplication.class);

View File

@@ -28,12 +28,6 @@ spring:
server:
port: ${HTTP_PORT:20400}
springdoc:
swagger-ui:
path: /swagger-ui.html
api-docs:
path: /v3/api-docs
lingniu:
ingest:
jt808:
@@ -52,10 +46,8 @@ lingniu:
rate-limit:
per-vin-qps: 50
session:
store: ${SESSION_STORE:redis}
ttl: ${SESSION_TTL:30m}
identity:
store: ${VEHICLE_IDENTITY_STORE:mysql}
mysql:
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
@@ -63,9 +55,8 @@ lingniu:
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
enabled: ${KAFKA_ENABLED:true}
type: kafka
kafka:
enabled: true
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
compression-type: zstd
linger-ms: 20
@@ -83,18 +74,16 @@ lingniu:
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
consumer:
enabled: false
event-history:
enabled: false
vehicle-state:
enabled: false
vehicle-stat:
enabled: false
management:
endpoints:
web:
exposure:
include: health,info,metrics,prometheus,env
include: health,info,metrics,prometheus
endpoint:
health:
probes:
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
health:
redis:
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}

View File

@@ -6,13 +6,14 @@ import com.lingniu.ingest.identity.config.VehicleIdentityAutoConfiguration;
import com.lingniu.ingest.protocol.jt808.codec.Jt808MessageDecoder;
import com.lingniu.ingest.protocol.jt808.config.Jt808AutoConfiguration;
import com.lingniu.ingest.protocol.jt808.inbound.Jt808NettyServer;
import com.lingniu.ingest.session.SessionStore;
import com.lingniu.ingest.session.config.SessionCoreAutoConfiguration;
import com.lingniu.ingest.sink.archive.ArchiveStore;
import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
import com.lingniu.ingest.sink.archive.config.SinkArchiveAutoConfiguration;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -30,26 +31,22 @@ class Jt808IngestAppCompositionTest {
IngestCoreAutoConfiguration.class,
SessionCoreAutoConfiguration.class,
VehicleIdentityAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
SinkArchiveAutoConfiguration.class,
Jt808AutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true)
.withBean("kafkaProducer", KafkaProducer.class, Jt808IngestAppCompositionTest::kafkaProducer)
.withBean(SessionStore.class, () -> mock(SessionStore.class))
.withPropertyValues(
"lingniu.ingest.jt808.enabled=true",
"lingniu.ingest.jt808.port=0",
"lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.initialize-schema=false",
"lingniu.ingest.session.store=memory",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.sink.archive.enabled=true",
"lingniu.ingest.sink.archive.path=target/test-archive-jt808",
"lingniu.ingest.event-history.enabled=false",
"lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=false");
"lingniu.ingest.event-history.enabled=false");
@Test
void createsJt808ListenerWithoutHistoryStorageBoundaries() {
@@ -62,7 +59,8 @@ class Jt808IngestAppCompositionTest {
assertThat(context).hasSingleBean(ArchiveStore.class);
assertThat(context).hasSingleBean(RawArchiveEventSink.class);
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
});
}

View File

@@ -28,13 +28,11 @@ class Jt808IngestAppDefaultsTest {
.containsEntry("server.port", "${HTTP_PORT:20400}")
.containsEntry("lingniu.ingest.jt808.enabled", true)
.containsEntry("lingniu.ingest.jt808.port", "${JT808_PORT:808}")
.containsEntry("lingniu.ingest.sink.mq.enabled", "${KAFKA_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry("lingniu.ingest.session.store", "${SESSION_STORE:redis}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:mysql}")
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", false)
.containsEntry("lingniu.ingest.sink.kafka.topics.realtime", "${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}")
.containsEntry("lingniu.ingest.sink.kafka.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry("lingniu.ingest.sink.kafka.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry("lingniu.ingest.identity.mysql.table-name",
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url",
@@ -43,13 +41,24 @@ class Jt808IngestAppDefaultsTest {
.containsEntry("lingniu.ingest.identity.mysql.password", "${VEHICLE_IDENTITY_MYSQL_PASSWORD:}")
.containsEntry("lingniu.ingest.identity.mysql.initialize-schema",
"${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}")
.containsEntry("lingniu.ingest.event-history.enabled", false)
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false);
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
assertThat(properties.stringPropertyNames())
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."));
assertThat(applicationYaml()).doesNotContain("event-file-store:");
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-stat."))
.noneMatch(name -> name.equals("lingniu.ingest.session.store"))
.noneMatch(name -> name.equals("lingniu.ingest.identity.store"));
assertThat(applicationYaml())
.doesNotContain("event-history:")
.doesNotContain("event-file-store:")
.doesNotContain("vehicle-state:")
.doesNotContain("vehicle-stat:")
.doesNotContain("springdoc:")
.doesNotContain("SESSION_STORE")
.doesNotContain("VEHICLE_IDENTITY_STORE")
.doesNotContain("KAFKA_ENABLED");
}
private static Properties applicationProperties() {

View File

@@ -23,7 +23,7 @@
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>sink-mq</artifactId>
<artifactId>sink-kafka</artifactId>
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
@@ -58,17 +58,6 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
<goal>build-info</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

View File

@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
@SpringBootApplication
public class VehicleAnalyticsApplication {
private static final Logger log = LoggerFactory.getLogger(VehicleAnalyticsApplication.class);

View File

@@ -24,7 +24,7 @@ spring:
driver-class-name: com.mysql.cj.jdbc.Driver
server:
port: ${HTTP_PORT:20300}
port: ${HTTP_PORT:20310}
springdoc:
swagger-ui:
@@ -34,17 +34,12 @@ springdoc:
lingniu:
ingest:
gb32960:
enabled: false
sink:
mq:
enabled: ${KAFKA_ENABLED:true}
type: kafka
kafka:
enabled: true
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
consumer:
dlq-topic: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
enabled: ${KAFKA_CONSUMER_ENABLED:true}
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
@@ -52,14 +47,10 @@ lingniu:
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:900000}
bindings:
vehicleStatEnvelopeConsumerProcessor:
enabled: ${VEHICLE_STAT_ENABLED:true}
enabled: true
group-id: ${KAFKA_GROUP_STAT:vehicle-stat}
topics:
- ${KAFKA_TOPIC_JT808_EVENT:vehicle.event.jt808.v1}
archive:
enabled: false
event-history:
enabled: false
vehicle-stat:
enabled: ${VEHICLE_STAT_ENABLED:true}
zone-id: ${VEHICLE_STAT_ZONE_ID:Asia/Shanghai}
@@ -70,7 +61,11 @@ management:
endpoints:
web:
exposure:
include: health,info,metrics,prometheus,env
include: health,info,metrics,prometheus
endpoint:
health:
probes:
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
metrics:
tags:
application: vehicle-analytics-app

View File

@@ -2,9 +2,9 @@ package com.lingniu.ingest.analyticsapp;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import com.lingniu.ingest.vehiclestat.JdbcVehicleStatMetricRepository;
import com.lingniu.ingest.vehiclestat.VehicleStatController;
import com.lingniu.ingest.vehiclestat.VehicleStatEnvelopeIngestor;
@@ -25,24 +25,21 @@ import static org.mockito.Mockito.mock;
class VehicleAnalyticsAppCompositionTest {
@Test
void createsStatsBeansWithoutProtocolListenerOrEventFileStore() {
void createsStatsBeansWithoutProtocolListenerOrFileHistoryStore() {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
VehicleStatAutoConfiguration.class))
.withAllowBeanDefinitionOverriding(true)
.withBean("kafkaProducer", KafkaProducer.class, VehicleAnalyticsAppCompositionTest::kafkaProducer)
.withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class))
.withBean(ObjectMapper.class, ObjectMapper::new)
.withPropertyValues(
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=true",
"lingniu.ingest.vehicle-stat.jt808.enabled=true",
"lingniu.ingest.event-history.enabled=false",
"lingniu.ingest.gb32960.enabled=false")
"lingniu.ingest.vehicle-stat.jt808.enabled=true")
.run(context -> {
assertThat(context).hasSingleBean(VehicleStatRepository.class);
assertThat(context).hasSingleBean(JdbcVehicleStatMetricRepository.class);
@@ -58,7 +55,9 @@ class VehicleAnalyticsAppCompositionTest {
assertTypeNotPresent(context, "com.lingniu.ingest.vehiclestate.VehicleStateEnvelopeIngestor");
assertTypeNotPresent(context, "com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration");
assertTypeNotPresent(context, "com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
assertTypeNotPresent(context,
"com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
});
}

View File

@@ -22,32 +22,42 @@ class VehicleAnalyticsAppDefaultsTest {
"spring.config.import[0]",
"optional:nacos:${spring.application.name}.${NACOS_CONFIG_FILE_EXTENSION:yml}?group=${NACOS_GROUP:DEFAULT_GROUP}&refreshEnabled=${NACOS_REFRESH_ENABLED:true}")
.containsEntry("spring.cloud.nacos.config.server-addr", "${NACOS_SERVER_ADDR:127.0.0.1:8848}")
.containsEntry("lingniu.ingest.gb32960.enabled", false)
.containsEntry("lingniu.ingest.sink.archive.enabled", false)
.containsEntry("lingniu.ingest.event-history.enabled", false)
.containsEntry("server.port", "${HTTP_PORT:20310}")
.containsEntry("lingniu.ingest.vehicle-stat.enabled", "${VEHICLE_STAT_ENABLED:true}")
.containsEntry(
"lingniu.ingest.vehicle-stat.jt808.enabled",
"${VEHICLE_STAT_JT808_MILEAGE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.enabled", true)
.containsEntry("lingniu.ingest.sink.kafka.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.kafka.consumer.dlq-topic",
"${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
"${VEHICLE_STAT_ENABLED:true}")
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.enabled",
true)
.containsEntry(
"lingniu.ingest.sink.mq.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_STAT:vehicle-stat}");
"lingniu.ingest.sink.kafka.consumer.bindings.vehicleStatEnvelopeConsumerProcessor.group-id",
"${KAFKA_GROUP_STAT:vehicle-stat}")
.containsEntry("management.endpoints.web.exposure.include", "health,info,metrics,prometheus");
assertThat(properties.stringPropertyNames())
.noneMatch(name -> name.startsWith("lingniu.ingest.gb32960."))
.noneMatch(name -> name.startsWith("lingniu.ingest.event-history."))
.noneMatch(name -> name.startsWith("lingniu.ingest.sink.archive."))
.noneMatch(name -> name.startsWith("lingniu.ingest.sink.kafka.topics."))
.noneMatch(name -> name.startsWith("lingniu.ingest.event-file-store."))
.noneMatch(name -> name.startsWith("lingniu.ingest.vehicle-state."))
.noneMatch(name -> name.contains("vehicleStateEnvelopeConsumerProcessor"))
.noneMatch(name -> name.startsWith("spring.data.redis."))
.noneMatch(name -> name.startsWith("management.health.redis."));
assertThat(applicationYaml())
.doesNotContain("gb32960:")
.doesNotContain("event-history:")
.doesNotContain("archive:")
.doesNotContain("event-file-store:")
.doesNotContain("vehicle-state:")
.doesNotContain("vehicleStateEnvelopeConsumerProcessor")
.doesNotContain("redis:");
.doesNotContain("redis:")
.doesNotContain("KAFKA_ENABLED");
}
private static Properties applicationProperties() {

View File

@@ -23,7 +23,7 @@
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>sink-mq</artifactId>
<artifactId>sink-kafka</artifactId>
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>
@@ -62,17 +62,6 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<jvmArguments>--sun-misc-unsafe-memory-access=allow</jvmArguments>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
<goal>build-info</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

View File

@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackages = "com.lingniu.ingest")
@SpringBootApplication
public class VehicleHistoryApplication {
private static final Logger log = LoggerFactory.getLogger(VehicleHistoryApplication.class);

View File

@@ -1,10 +1,10 @@
package com.lingniu.ingest.historyapp;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerFactory;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerWorker;
import com.lingniu.ingest.sink.mq.SinkMqProperties;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerFactory;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerWorker;
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -20,11 +20,11 @@ public class VehicleHistoryKafkaConsumerConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.kafka.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner vehicleHistoryKafkaEnvelopeConsumerRunner(
@Qualifier("eventHistoryEnvelopeConsumerProcessor") EnvelopeConsumerProcessor processor,
@Qualifier("eventHistoryRawEnvelopeConsumerProcessor") EnvelopeConsumerProcessor rawProcessor,
SinkMqProperties props) {
KafkaSinkProperties props) {
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(
Map.of(
"eventHistoryGb32960EnvelopeConsumerProcessor", processor,

View File

@@ -29,25 +29,17 @@ springdoc:
lingniu:
ingest:
gb32960:
enabled: true
server:
enabled: false
sink:
mq:
enabled: ${KAFKA_ENABLED:true}
type: kafka
kafka:
enabled: true
bootstrap-servers: ${KAFKA_BROKERS:114.55.58.251:9092}
topics:
realtime: ${KAFKA_TOPIC_GB32960_EVENT:vehicle.event.gb32960.v1}
raw-archive: ${KAFKA_TOPIC_GB32960_RAW:vehicle.raw.gb32960.v1}
dlq: ${KAFKA_TOPIC_GB32960_DLQ:vehicle.dlq.gb32960.v1}
consumer:
dlq-topic: ${KAFKA_TOPIC_HISTORY_DLQ:vehicle.dlq.history.v1}
enabled: ${KAFKA_CONSUMER_ENABLED:true}
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:1800000}
concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3}
bindings:
eventHistoryGb32960EnvelopeConsumerProcessor:
@@ -80,8 +72,6 @@ lingniu:
group-id: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:${KAFKA_GROUP_HISTORY:vehicle-history}-yutong-mqtt-raw}
topics:
- ${KAFKA_TOPIC_YUTONG_MQTT_RAW:vehicle.raw.mqtt-yutong.v1}
archive:
enabled: false
tdengine-history:
enabled: ${TDENGINE_HISTORY_ENABLED:false}
database: ${TDENGINE_HISTORY_DATABASE:vehicle_history}
@@ -93,21 +83,18 @@ lingniu:
minimum-idle: ${TDENGINE_MIN_IDLE:0}
connection-timeout-millis: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:5000}
initialization-fail-timeout-millis: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:-1}
telemetry-fields-enabled: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:false}
event-history:
enabled: true
api:
specialized-enabled: ${EVENT_HISTORY_SPECIALIZED_API_ENABLED:false}
vehicle-state:
enabled: false
vehicle-stat:
enabled: false
management:
endpoints:
web:
exposure:
include: health,info,metrics,prometheus,env
include: health,info,metrics,prometheus
endpoint:
health:
probes:
enabled: ${MANAGEMENT_HEALTH_PROBES_ENABLED:true}
health:
redis:
enabled: ${MANAGEMENT_HEALTH_REDIS_ENABLED:false}

View File

@@ -10,11 +10,19 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
class MavenModuleProfileTest {
private static final List<String> ACTIVE_APP_POMS = List.of(
"modules/apps/gb32960-ingest-app/pom.xml",
"modules/apps/jt808-ingest-app/pom.xml",
"modules/apps/yutong-mqtt-app/pom.xml",
"modules/apps/vehicle-history-app/pom.xml",
"modules/apps/vehicle-analytics-app/pom.xml");
@Test
void commandGatewayIsOutsideDefaultProductionReactor() throws Exception {
Document pom = rootPom();
@@ -30,23 +38,607 @@ class MavenModuleProfileTest {
}
@Test
void xindaIsLegacyOnlyAndOutsideDefaultProductionReactor() throws Exception {
void jsatl12AttachmentProtocolIsOptionalAndOutsideDefaultProductionReactor() throws Exception {
Document pom = rootPom();
assertThat(defaultModules(pom))
.doesNotContain("modules/protocols/protocol-jsatl12");
assertThat(profileModules(pom, "optional-attachments"))
.containsExactly("modules/protocols/protocol-jsatl12");
}
@Test
void latestStateServiceIsOptionalAndOutsideDefaultProductionReactor() throws Exception {
Document pom = rootPom();
assertThat(defaultModules(pom))
.doesNotContain("modules/services/vehicle-state-service");
assertThat(profileModules(pom, "optional-latest-state"))
.containsExactly("modules/services/vehicle-state-service");
}
@Test
void rawArchiveStorePrototypeIsRemovedFromBuildSurface() throws Exception {
Document pom = rootPom();
assertThat(defaultModules(pom))
.doesNotContain("modules/sinks/raw-archive-store");
assertThat(profileModules(pom, "optional-raw-archive-store"))
.as("raw bytes are written by sink-archive; raw-archive-store was an unused prototype")
.isEmpty();
assertThat(Files.exists(repositoryRoot().resolve("modules/sinks/raw-archive-store")))
.isFalse();
}
@Test
void eventFileStoreCompatibilityPathIsRemovedFromProductionStack() throws Exception {
Document pom = rootPom();
String readme = Files.readString(repositoryRoot().resolve("README.md"));
assertThat(defaultModules(pom))
.doesNotContain("modules/sinks/event-file-store");
assertThat(profileModules(pom, "optional-event-file-store"))
.as("DuckDB/Parquet event-file-store has been replaced by TDengine raw history")
.isEmpty();
assertThat(Files.exists(repositoryRoot().resolve("modules/sinks/event-file-store")))
.isFalse();
assertThat(rootDependencyManagementArtifacts(pom))
.doesNotContain("event-file-store");
assertThat(hasProjectProperty(pom, "duckdb.version"))
.isFalse();
assertThat(hasDependency(pom, "org.duckdb", "duckdb_jdbc"))
.isFalse();
assertThat(readme)
.doesNotContain("event-file-store")
.doesNotContain("DuckDB")
.doesNotContain("Parquet");
}
@Test
void kafkaSinkModuleUsesKafkaNamingInsteadOfMqNaming() throws Exception {
Document pom = rootPom();
Document kafkaSinkPom = modulePom("modules/sinks/sink-kafka/pom.xml");
String readme = Files.readString(repositoryRoot().resolve("README.md"));
String legacyModule = "modules/sinks/sink-" + "mq";
String legacyArtifact = "sink-" + "mq";
assertThat(defaultModules(pom))
.contains("modules/sinks/sink-kafka")
.doesNotContain(legacyModule);
assertThat(rootDependencyManagementArtifacts(pom))
.contains("sink-kafka")
.doesNotContain(legacyArtifact);
assertThat(firstDirectChild(kafkaSinkPom.getDocumentElement(), "artifactId").getTextContent())
.isEqualTo("sink-kafka");
assertThat(readme)
.contains("sink-kafka/")
.doesNotContain(legacyArtifact);
}
@Test
void xindaModulesAreRemovedFromBuildSurface() throws Exception {
Document pom = rootPom();
String rootPomText = Files.readString(repositoryRoot().resolve("pom.xml"));
String readme = Files.readString(repositoryRoot().resolve("README.md"));
assertThat(defaultModules(pom))
.doesNotContain("modules/inbound/inbound-xinda-push")
.doesNotContain("modules/apps/xinda-push-app");
assertThat(profileModules(pom, "legacy-xinda"))
.containsExactly(
"modules/inbound/inbound-xinda-push",
"modules/apps/xinda-push-app");
.as("xinda is deleted, not hidden behind an opt-in profile")
.isEmpty();
assertThat(Files.exists(repositoryRoot().resolve("modules/inbound/inbound-xinda-push")))
.isFalse();
assertThat(Files.exists(repositoryRoot().resolve("modules/apps/xinda-push-app")))
.isFalse();
assertThat(rootPomText)
.doesNotContain("legacy-xinda")
.doesNotContain("inbound-xinda-push")
.doesNotContain("xinda-push-app");
assertThat(readme)
.contains("信达 Push 已废弃并删除源码")
.doesNotContain("信达 Push 仅保留")
.doesNotContain("废弃兼容模块");
}
@Test
void privateLingniuRepositoryIsRemovedWithXinda() throws Exception {
Document pom = rootPom();
assertThat(repositoryIds(pom.getDocumentElement()))
.doesNotContain("lingniu-nexus");
assertThat(profileRepositoryIds(pom, "legacy-xinda"))
.isEmpty();
}
@Test
void optionalModuleVersionsAreScopedToTheirProfiles() throws Exception {
Document pom = rootPom();
assertThat(rootDependencyManagementArtifacts(pom))
.doesNotContain(
"protocol-jt1078",
"protocol-jsatl12",
"vehicle-state-service",
"raw-archive-store",
"command-gateway",
"inbound-xinda-push",
"xinda-push-app");
assertThat(profileDependencyManagementArtifacts(pom, "optional-command-gateway"))
.containsExactly("protocol-jt1078", "command-gateway");
assertThat(profileDependencyManagementArtifacts(pom, "optional-attachments"))
.containsExactly("protocol-jsatl12");
assertThat(profileDependencyManagementArtifacts(pom, "optional-latest-state"))
.containsExactly("vehicle-state-service");
assertThat(profileDependencyManagementArtifacts(pom, "optional-raw-archive-store"))
.isEmpty();
assertThat(profileDependencyManagementArtifacts(pom, "optional-event-file-store"))
.isEmpty();
assertThat(profileDependencyManagementArtifacts(pom, "legacy-xinda"))
.isEmpty();
}
@Test
void appsDirectoryContainsOnlyRealMavenModules() throws Exception {
try (Stream<Path> apps = Files.list(repositoryRoot().resolve("modules/apps"))) {
assertThat(apps.filter(Files::isDirectory)
.filter(path -> !Files.exists(path.resolve("pom.xml")))
.map(path -> path.getFileName().toString()))
.isEmpty();
}
}
@Test
void vehicleIdentityTestSupportIsCentralizedInsteadOfCopiedIntoProtocolTests() throws Exception {
Document pom = rootPom();
List<String> testSupportFixtures = javaFilesNamed("InMemoryVehicleIdentityService.java");
assertThat(defaultModules(pom))
.contains("modules/testing/vehicle-identity-test-support");
assertThat(rootDependencyManagementArtifacts(pom))
.contains("vehicle-identity-test-support");
assertThat(testSupportFixtures)
.containsExactly("modules/testing/vehicle-identity-test-support/src/main/java/com/lingniu/ingest/identity/InMemoryVehicleIdentityService.java");
for (String modulePom : List.of(
"modules/protocols/protocol-jt808/pom.xml",
"modules/inbound/inbound-mqtt/pom.xml",
"modules/protocols/protocol-jt1078/pom.xml",
"modules/protocols/protocol-jsatl12/pom.xml")) {
assertThat(hasDependency(modulePom(modulePom), "com.lingniu.ingest", "vehicle-identity-test-support"))
.as(modulePom)
.isTrue();
}
}
@Test
void readmeMatchesCurrentRuntimeVersionsAndSplitAppResponsibilities() throws Exception {
Document pom = rootPom();
String readme = Files.readString(repositoryRoot().resolve("README.md"));
assertThat(readme)
.contains("Spring Boot " + projectProperty(pom, "spring-boot.version"))
.contains("Netty " + projectProperty(pom, "netty.version"))
.contains("Eclipse Paho " + projectProperty(pom, "paho-mqtt.version"))
.contains("session-core/ 设备会话 + 鉴权 + Token + Redis SessionStore")
.contains("| 并发 | Disruptor 4 + 虚拟线程 |")
.contains("| 事件流 | **Kafka**唯一生产消息通道vin 分区,保证单车有序) |")
.contains("| 冷存 | 本地文件系统 |")
.contains("| 构建 | Maven |")
.contains("protocol-jt1078/ JT/T 1078optional-command-gateway profile")
.contains("protocol-jsatl12/ 苏标主动安全报警附件optional-attachments profile")
.contains("vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询optional-latest-state profile")
.contains("command-gateway/ 可选 HTTP → 设备下行命令optional-command-gateway profile")
.contains("vehicle-history-app/ TDengine 历史查询 + RAW JSON")
.contains("vehicle-analytics-app/ JT808 每日里程指标消费")
.contains("协议接入应用只负责收、解析、校验、规整和投递 Kafka")
.contains("生产链路只支持 Kafka")
.contains("`vehicle-analytics-app` 将 JT808 `daily_mileage_km` 写入 MySQL `vehicle_stat_metric`")
.contains("mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true")
.contains("生产服务只在 ECS/Portainer 上运行")
.doesNotContain("Spring Boot 3.4")
.doesNotContain("Netty 4.1")
.doesNotContain("HiveMQ MQTT Client")
.doesNotContain("RabbitMQ")
.doesNotContain("RocketMQ")
.doesNotContain("其他 MQ 后端")
.doesNotContain("Spotless")
.doesNotContain("ArchUnit")
.doesNotContain("S3 / OSS / 本地")
.doesNotContain("StructuredTaskScope")
.doesNotContain("mvn clean install -DskipTests")
.doesNotContain("-DskipTests")
.doesNotContain("spring-boot:run")
.doesNotContain("Caffeine 内存降级")
.doesNotContain("Redis/Memory SessionStore")
.doesNotContain("event-file-store")
.doesNotContain("DuckDB")
.doesNotContain("Parquet")
.doesNotContain("Parquet + DuckDB 文件型明细库")
.doesNotContain("车辆状态/日统计消费")
.doesNotContain("本服务不写业务库")
.doesNotContain("里程统计全部由 Kafka 下游消费者实现")
.doesNotContain("| 消息队列 |");
}
@Test
void archiveBuildSurfaceDocumentsLocalFilesystemOnly() throws Exception {
String archivePom = Files.readString(repositoryRoot().resolve("modules/sinks/sink-archive/pom.xml"));
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
assertThat(archivePom)
.contains("原始报文冷存:本地文件系统实现。")
.doesNotContain("S3/OSS")
.doesNotContain("MinIO")
.doesNotContain("software.amazon.awssdk")
.doesNotContain("com.aliyun.oss");
assertThat(moduleDataFlow)
.contains("sink-archive<br/>RawArchive 到本地文件系统冷存")
.contains("归档文件的真实本地路径只属于 ArchiveStore")
.contains("原始报文冷存,当前为本地文件系统实现")
.doesNotContain("本地/S3/OSS")
.doesNotContain("S3/OSS")
.doesNotContain("MinIO");
}
@Test
void yutongMqttBuildSurfaceUsesOnlyPahoClient() throws Exception {
Document pom = rootPom();
assertThat(hasProjectProperty(pom, "paho-mqtt.version"))
.isTrue();
assertThat(hasDependency(pom, "org.eclipse.paho", "org.eclipse.paho.client.mqttv3"))
.isTrue();
assertThat(hasProjectProperty(pom, "fusesource-mqtt.version"))
.isFalse();
assertThat(hasDependency(pom, "org.fusesource.mqtt-client", "mqtt-client"))
.isFalse();
}
@Test
void activeAppBootPackagingConfigurationIsManagedOnceInRootPom() throws Exception {
Document pom = rootPom();
Element managedBootPlugin = managedPlugin(pom, "org.springframework.boot", "spring-boot-maven-plugin");
assertThat(managedBootPlugin).isNotNull();
assertThat(managedBootPlugin.getTextContent())
.contains("${spring-boot.version}")
.contains("--sun-misc-unsafe-memory-access=allow")
.contains("-Dio.netty.transport.noNative=true")
.contains("repackage")
.contains("build-info");
for (String appPom : ACTIVE_APP_POMS) {
Document app = modulePom(appPom);
Element bootPlugin = buildPlugin(app, "org.springframework.boot", "spring-boot-maven-plugin");
assertThat(bootPlugin)
.as(appPom)
.isNotNull();
assertThat(firstDirectChild(bootPlugin, "configuration"))
.as(appPom + " should inherit shared JVM arguments from root pluginManagement")
.isNull();
assertThat(firstDirectChild(bootPlugin, "executions"))
.as(appPom + " should inherit repackage/build-info executions from root pluginManagement")
.isNull();
}
}
@Test
void observabilityBuildSurfaceUsesMicrometerWithoutUnusedOpenTelemetryBom() throws Exception {
Document pom = rootPom();
Document observabilityPom = modulePom("modules/core/observability/pom.xml");
String readme = Files.readString(repositoryRoot().resolve("README.md"));
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
String traceSpan = Files.readString(repositoryRoot()
.resolve("modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/annotation/TraceSpan.java"));
assertThat(hasProjectProperty(pom, "micrometer.version"))
.isTrue();
assertThat(hasDependency(pom, "io.micrometer", "micrometer-bom"))
.isTrue();
assertThat(hasProjectProperty(pom, "otel.version"))
.isFalse();
assertThat(hasDependency(pom, "io.opentelemetry", "opentelemetry-bom"))
.isFalse();
assertThat(hasDependency(observabilityPom, "com.lingniu.ingest", "ingest-core"))
.as("observability is an interceptor API module and should not depend on core implementation")
.isFalse();
assertThat(productionJavaContains("io.opentelemetry"))
.isFalse();
assertThat(readme)
.contains("| 可观测 | Micrometer + Prometheus |")
.doesNotContain("OpenTelemetry");
assertThat(moduleDataFlow)
.contains("observability<br/>metrics / health")
.doesNotContain("observability<br/>metrics / tracing / health");
assertThat(traceSpan)
.doesNotContain("OpenTelemetry")
.doesNotContain("span");
}
@Test
void activeAppsDependOnSharedObservabilityForHealthAndMetrics() throws Exception {
for (String appPom : ACTIVE_APP_POMS) {
assertThat(hasDependency(modulePom(appPom), "com.lingniu.ingest", "observability"))
.as(appPom + " exposes actuator health/readiness and Prometheus metrics")
.isTrue();
}
}
@Test
void springdocUiIsOnlyPackagedWithAppsThatExposeRestApis() throws Exception {
for (String appPom : List.of(
"modules/apps/gb32960-ingest-app/pom.xml",
"modules/apps/jt808-ingest-app/pom.xml",
"modules/apps/yutong-mqtt-app/pom.xml")) {
assertThat(hasDependency(modulePom(appPom), "org.springdoc", "springdoc-openapi-starter-webmvc-ui"))
.as(appPom + " exposes Actuator only; Swagger UI belongs to query/API apps")
.isFalse();
}
for (String appPom : List.of(
"modules/apps/vehicle-history-app/pom.xml",
"modules/apps/vehicle-analytics-app/pom.xml")) {
assertThat(hasDependency(modulePom(appPom), "org.springdoc", "springdoc-openapi-starter-webmvc-ui"))
.as(appPom + " exposes REST APIs")
.isTrue();
}
}
@Test
void buildSurfaceDoesNotManageLombokWhenProductionSourcesDoNotUseIt() throws Exception {
Document pom = rootPom();
assertThat(hasProjectProperty(pom, "lombok.version"))
.isFalse();
assertThat(hasDependency(pom, "org.projectlombok", "lombok"))
.isFalse();
assertThat(productionJavaContains("lombok."))
.isFalse();
}
@Test
void buildSurfaceDoesNotManageArchUnitWhenThereAreNoArchitectureTests() throws Exception {
Document pom = rootPom();
assertThat(hasProjectProperty(pom, "archunit.version"))
.isFalse();
assertThat(hasDependency(pom, "com.tngtech.archunit", "archunit-junit5"))
.isFalse();
assertThat(javaSourcesContain("import com.tngtech." + "archunit"))
.isFalse();
}
@Test
void buildSurfaceDoesNotManageTestcontainersWhenKafkaTestsUseInMemoryFakes() throws Exception {
Document pom = rootPom();
Document kafkaSinkPom = modulePom("modules/sinks/sink-kafka/pom.xml");
assertThat(hasProjectProperty(pom, "testcontainers.version"))
.isFalse();
assertThat(hasDependency(pom, "org.testcontainers", "testcontainers-bom"))
.isFalse();
assertThat(hasDependency(kafkaSinkPom, "org.testcontainers", "kafka"))
.isFalse();
assertThat(javaSourcesContain("import org." + "testcontainers"))
.isFalse();
}
@Test
void productionDocsDoNotAdvertiseMemorySessionStore() throws Exception {
String runbook = Files.readString(repositoryRoot()
.resolve("docs/operations/gb32960-service-split-runbook.md"));
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
String redesignSpec = Files.readString(repositoryRoot()
.resolve("docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md"));
assertThat(runbook)
.contains("Session state requires Redis")
.contains("REDIS_HOST")
.contains("REDIS_DATABASE=50")
.doesNotContain("SESSION_STORE=memory")
.doesNotContain("without Redis");
assertThat(moduleDataFlow)
.contains("SessionStore 仅保留 Redis 后端")
.doesNotContain("SessionStore 支持 <code>memory</code>")
.doesNotContain("无 Redis 降级");
assertThat(redesignSpec)
.contains("Reads Redis latest state")
.doesNotContain("Redis or in-memory")
.doesNotContain("Redis/in-memory");
}
@Test
void sessionCorePomDescriptionDoesNotAdvertiseRedisAsFallback() throws Exception {
Document sessionCorePom = modulePom("modules/core/session-core/pom.xml");
String description = firstDirectChild(sessionCorePom.getDocumentElement(), "description").getTextContent();
assertThat(description)
.contains("Redis SessionStore")
.doesNotContain("兜底")
.doesNotContain("fallback")
.doesNotContain("memory");
}
@Test
void rootPomDescriptionMatchesCurrentRuntimeStack() throws Exception {
Document pom = rootPom();
String description = firstDirectChild(pom.getDocumentElement(), "description").getTextContent();
assertThat(description)
.contains("只支持 Kafka")
.contains("Java 25")
.contains("Spring Boot " + projectProperty(pom, "spring-boot.version"))
.contains("Netty " + projectProperty(pom, "netty.version"))
.doesNotContain("Spring Boot 3.4");
}
@Test
void defaultProductionSurfaceOnlySupportsKafkaAsMessageBackbone() throws Exception {
List<Path> productionSurface = List.of(
repositoryRoot().resolve("README.md"),
repositoryRoot().resolve("pom.xml"),
repositoryRoot().resolve("deploy/portainer/docker-compose.yml"),
repositoryRoot().resolve("docs/operations/gb32960-service-split-runbook.md"),
repositoryRoot().resolve("docs/operations/vehicle-ingest-tdengine-verification.md"),
repositoryRoot().resolve("docs/operations/jt808-daily-mileage-runbook.md"),
repositoryRoot().resolve("docs/target-architecture.md"));
for (Path path : productionSurface) {
String text = Files.readString(path);
assertThat(text)
.as(path.toString())
.doesNotContain("sink-mq")
.doesNotContain("RabbitMQ")
.doesNotContain("RocketMQ")
.doesNotContain("Pulsar")
.doesNotContain("AMQP")
.doesNotContain("其他 MQ 后端");
}
}
@Test
void architectureDecisionsMatchCurrentDefaultProductionSurface() throws Exception {
String decisions = Files.readString(repositoryRoot().resolve("DECISIONS.md"));
assertThat(decisions)
.contains("## ADR-004 信达 PushRemoved")
.contains("## ADR-006 部署形态GB32960 三应用拆分\n- **Status**: Superseded by ADR-011")
.contains("## ADR-011 默认生产面:三协议接入 + 历史 + 统计")
.contains("## ADR-012 JSATL12 附件上传Optional only")
.contains("GB32960、JT808、Yutong MQTT、vehicle-history-app、vehicle-analytics-app")
.contains("Xinda Push 源码、Maven profile、Woodpecker 镜像发布和历史消费绑定全部移除")
.contains("JSATL12 仅保留在 `optional-attachments` profile")
.contains("vehicle-state-service 仅保留在 `optional-latest-state` profile")
.contains("raw-archive-store 原型已删除")
.contains("协议接入应用不直接写业务库")
.contains("`vehicle-analytics-app` 写入 MySQL `vehicle_stat_metric`")
.doesNotContain("event-file-store")
.doesNotContain("optional-event-file-store")
.doesNotContain("optional-raw-archive-store")
.doesNotContain("legacy-xinda")
.doesNotContain("## ADR-004 信达 Push保留模块但彻底重写\n- **Status**: Accepted")
.doesNotContain("CI/CD 分别构建三个镜像")
.doesNotContain("Spring Boot 3.4.x");
}
@Test
void productionDocsMatchCompactHistoryAndJt808OnlyAnalyticsSurface() throws Exception {
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
String targetArchitecture = Files.readString(repositoryRoot().resolve("docs/target-architecture.md"));
String tdengineVerification = Files.readString(repositoryRoot()
.resolve("docs/operations/vehicle-ingest-tdengine-verification.md"));
String runbook = Files.readString(repositoryRoot()
.resolve("docs/operations/gb32960-service-split-runbook.md"));
assertThat(moduleDataFlow)
.contains("TDengine raw_frames + locations")
.contains("TDengine <code>raw_frames</code> 和位置表")
.doesNotContain("telemetry field 行映射")
.doesNotContain("可选 telemetry field 表");
assertThat(tdengineVerification)
.contains("history 写入 raw、location 到 TDengine不写 telemetry_fields")
.doesNotContain("按需开启 telemetry field");
assertThat(runbook)
.contains("| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` |")
.contains("| `vehicle.event.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` |")
.doesNotContain("| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` |");
assertThat(targetArchitecture)
.contains("vehicle-state-service is absent from the default Maven reactor")
.doesNotContain("kafka --> state[\"vehicle-state-service\"]")
.doesNotContain("state --> redisState[\"Redis<br/>latest state\"]");
}
@Test
void supersededDesignDocsPointToCurrentArchitecture() throws Exception {
for (String path : List.of(
"docs/superpowers/specs/2026-06-23-gb32960-production-readiness-design.md",
"docs/superpowers/specs/2026-06-23-gb32960-service-split-design.md")) {
String doc = Files.readString(repositoryRoot().resolve(path));
assertThat(doc)
.as(path)
.startsWith("> **Superseded:**")
.contains("docs/target-architecture.md")
.contains("docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md");
}
}
@Test
void supersededImplementationPlansPointToCurrentArchitecture() throws Exception {
for (String path : List.of(
"docs/superpowers/plans/2026-06-23-gb32960-history-duckdb-hot-store.md",
"docs/superpowers/plans/2026-06-23-gb32960-service-split.md",
"docs/superpowers/plans/2026-06-29-vehicle-ingest-redesign-phase1.md")) {
String doc = Files.readString(repositoryRoot().resolve(path));
assertThat(doc)
.as(path)
.startsWith("> **Superseded:**")
.contains("docs/target-architecture.md")
.contains("docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md");
}
}
@Test
void duckDbAndParquetAreNotPartOfTheCurrentBuildSurface() throws Exception {
Document pom = rootPom();
assertThat(hasProjectProperty(pom, "duckdb.version"))
.isFalse();
assertThat(hasDependency(pom, "org.duckdb", "duckdb_jdbc"))
.isFalse();
assertThat(Files.exists(repositoryRoot().resolve("modules/sinks/event-file-store")))
.isFalse();
}
@Test
void legacyEventFileStoreContractsAreRemovedFromProductionSources() throws Exception {
Path root = repositoryRoot();
assertThat(Files.exists(root.resolve("modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/history")))
.isFalse();
assertThat(Files.exists(root.resolve(
"modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java")))
.isFalse();
assertThat(Files.exists(root.resolve(
"modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java")))
.isFalse();
assertThat(productionJavaContains("EventFileStore"))
.isFalse();
assertThat(productionJavaContains("EventFileRecord"))
.isFalse();
assertThat(productionJavaContains("EventFileQuery"))
.isFalse();
}
private static Document rootPom() throws Exception {
return modulePom("pom.xml");
}
private static Document modulePom(String path) throws Exception {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
return factory.newDocumentBuilder().parse(Files.newInputStream(repositoryRoot().resolve("pom.xml")));
return factory.newDocumentBuilder().parse(Files.newInputStream(repositoryRoot().resolve(path)));
}
private static String projectProperty(Document pom, String name) {
Element properties = firstDirectChild(pom.getDocumentElement(), "properties");
Element property = firstDirectChild(properties, name);
if (property == null) {
throw new IllegalArgumentException("project property not found: " + name);
}
return property.getTextContent().trim();
}
private static boolean hasProjectProperty(Document pom, String name) {
Element properties = firstDirectChild(pom.getDocumentElement(), "properties");
return properties != null && firstDirectChild(properties, name) != null;
}
private static List<String> defaultModules(Document pom) {
@@ -55,9 +647,56 @@ class MavenModuleProfileTest {
}
private static List<String> profileModules(Document pom, String profileId) {
Element profile = profile(pom, profileId);
if (profile == null) {
return List.of();
}
return directModules(firstDirectChild(profile, "modules"));
}
private static List<String> rootDependencyManagementArtifacts(Document pom) {
return dependencyManagementArtifacts(pom.getDocumentElement());
}
private static List<String> profileDependencyManagementArtifacts(Document pom, String profileId) {
Element profile = profile(pom, profileId);
if (profile == null) {
return List.of();
}
return dependencyManagementArtifacts(profile);
}
private static List<String> profileRepositoryIds(Document pom, String profileId) {
Element profile = profile(pom, profileId);
if (profile == null) {
return List.of();
}
return repositoryIds(profile);
}
private static List<String> repositoryIds(Element owner) {
Element repositories = firstDirectChild(owner, "repositories");
if (repositories == null) {
return List.of();
}
List<String> ids = new ArrayList<>();
NodeList children = repositories.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
if (!(children.item(i) instanceof Element repository) || !"repository".equals(repository.getTagName())) {
continue;
}
Element id = firstDirectChild(repository, "id");
if (id != null) {
ids.add(id.getTextContent().trim());
}
}
return List.copyOf(ids);
}
private static Element profile(Document pom, String profileId) {
Element profiles = firstDirectChild(pom.getDocumentElement(), "profiles");
if (profiles == null) {
return List.of();
return null;
}
NodeList children = profiles.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
@@ -66,10 +705,10 @@ class MavenModuleProfileTest {
}
Element id = firstDirectChild(profile, "id");
if (id != null && profileId.equals(id.getTextContent().trim())) {
return directModules(firstDirectChild(profile, "modules"));
return profile;
}
}
return List.of();
return null;
}
private static List<String> directModules(Element modules) {
@@ -86,6 +725,136 @@ class MavenModuleProfileTest {
return List.copyOf(out);
}
private static List<String> dependencyManagementArtifacts(Element owner) {
Element dependencyManagement = firstDirectChild(owner, "dependencyManagement");
if (dependencyManagement == null) {
return List.of();
}
Element dependencies = firstDirectChild(dependencyManagement, "dependencies");
if (dependencies == null) {
return List.of();
}
List<String> artifacts = new ArrayList<>();
NodeList children = dependencies.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
if (!(children.item(i) instanceof Element dependency) || !"dependency".equals(dependency.getTagName())) {
continue;
}
Element groupId = firstDirectChild(dependency, "groupId");
Element artifactId = firstDirectChild(dependency, "artifactId");
if (groupId != null
&& artifactId != null
&& "com.lingniu.ingest".equals(groupId.getTextContent().trim())) {
artifacts.add(artifactId.getTextContent().trim());
}
}
return List.copyOf(artifacts);
}
private static Element managedPlugin(Document pom, String groupId, String artifactId) {
Element build = firstDirectChild(pom.getDocumentElement(), "build");
Element pluginManagement = build == null ? null : firstDirectChild(build, "pluginManagement");
return plugin(pluginManagement, groupId, artifactId);
}
private static Element buildPlugin(Document pom, String groupId, String artifactId) {
Element build = firstDirectChild(pom.getDocumentElement(), "build");
return plugin(build, groupId, artifactId);
}
private static Element plugin(Element owner, String groupId, String artifactId) {
Element plugins = owner == null ? null : firstDirectChild(owner, "plugins");
if (plugins == null) {
return null;
}
NodeList children = plugins.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
if (!(children.item(i) instanceof Element plugin) || !"plugin".equals(plugin.getTagName())) {
continue;
}
Element currentGroupId = firstDirectChild(plugin, "groupId");
Element currentArtifactId = firstDirectChild(plugin, "artifactId");
if (currentGroupId != null
&& currentArtifactId != null
&& groupId.equals(currentGroupId.getTextContent().trim())
&& artifactId.equals(currentArtifactId.getTextContent().trim())) {
return plugin;
}
}
return null;
}
private static boolean hasDependency(Document pom, String groupId, String artifactId) {
Element owner = pom.getDocumentElement();
Element dependencyManagement = firstDirectChild(owner, "dependencyManagement");
Element dependencies = firstDirectChild(owner, "dependencies");
if (hasDependencyIn(dependencies, groupId, artifactId)) {
return true;
}
if (dependencyManagement == null) {
return false;
}
return hasDependencyIn(firstDirectChild(dependencyManagement, "dependencies"), groupId, artifactId);
}
private static boolean hasDependencyIn(Element dependencies, String groupId, String artifactId) {
if (dependencies == null) {
return false;
}
NodeList children = dependencies.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
if (!(children.item(i) instanceof Element dependency) || !"dependency".equals(dependency.getTagName())) {
continue;
}
Element currentGroupId = firstDirectChild(dependency, "groupId");
Element currentArtifactId = firstDirectChild(dependency, "artifactId");
if (currentGroupId != null
&& currentArtifactId != null
&& groupId.equals(currentGroupId.getTextContent().trim())
&& artifactId.equals(currentArtifactId.getTextContent().trim())) {
return true;
}
}
return false;
}
private static boolean productionJavaContains(String token) throws Exception {
return javaSourcesContain(token, false);
}
private static boolean javaSourcesContain(String token) throws Exception {
return javaSourcesContain(token, true);
}
private static boolean javaSourcesContain(String token, boolean includeTests) throws Exception {
try (Stream<Path> files = Files.walk(repositoryRoot().resolve("modules"))) {
return files
.filter(Files::isRegularFile)
.filter(path -> path.toString().endsWith(".java"))
.filter(path -> includeTests || !path.toString().contains("/src/test/"))
.anyMatch(path -> {
try {
return Files.readString(path).contains(token);
} catch (Exception e) {
throw new IllegalStateException("failed to read " + path, e);
}
});
}
}
private static List<String> javaFilesNamed(String fileName) throws Exception {
Path root = repositoryRoot();
try (Stream<Path> files = Files.walk(root.resolve("modules"))) {
return files
.filter(Files::isRegularFile)
.filter(path -> fileName.equals(path.getFileName().toString()))
.map(root::relativize)
.map(Path::toString)
.sorted()
.toList();
}
}
private static Element firstDirectChild(Element parent, String tagName) {
NodeList children = parent.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {

View File

@@ -1,15 +1,34 @@
package com.lingniu.ingest.historyapp;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
import org.springframework.core.io.FileSystemResource;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Properties;
import static org.assertj.core.api.Assertions.assertThat;
class PortainerComposeResourceLimitsTest {
@Test
void portainerStackDocumentsDefaultProductionSurfaceOnly() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
assertThat(compose)
.contains("Default production stack: GB32960, JT808, Yutong MQTT, history, analytics.")
.contains("Legacy and optional services stay outside this compose file.")
.doesNotContain("KAFKA_ENABLED")
.doesNotContain("\n xinda-push-app:\n")
.doesNotContain("\n command-gateway:\n")
.doesNotContain("\n vehicle-state-service:\n")
.doesNotContain("\n raw-archive-store:\n")
.doesNotContain("\n event-file-store:\n");
}
@Test
void productionServicesHaveOverridableMemoryLimits() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
@@ -22,19 +41,243 @@ class PortainerComposeResourceLimitsTest {
}
@Test
void vehicleHistoryUsesApplicationConsumerGroupEnvironmentNames() throws IOException {
void productionImagesShareOverridableRegistryAndNamespace() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String registry = "${LINGNIU_IMAGE_REGISTRY:-crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com}";
String namespace = "${LINGNIU_IMAGE_NAMESPACE:-oneos}";
for (String serviceName : List.of(
"gb32960-ingest-app",
"jt808-ingest-app",
"yutong-mqtt-app",
"vehicle-history-app",
"vehicle-analytics-app")) {
assertThat(serviceBlock(compose, serviceName))
.contains("image: " + registry + "/" + namespace + "/" + serviceName
+ ":${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}");
}
assertThat(countOccurrences(compose, "image: crpi-85r4m0ackrm3qpje"))
.isZero();
assertThat(countOccurrences(compose, "/oneos/")).isZero();
}
@Test
void portainerServicesUseContainerReadinessHealthchecks() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String dockerfile = Files.readString(repositoryRoot().resolve("Dockerfile"));
assertThat(dockerfile).contains("apt-get install -y --no-install-recommends curl");
assertThat(compose)
.contains("x-readiness-healthcheck: &readiness-healthcheck\n"
+ " interval: 30s\n"
+ " timeout: 5s\n"
+ " retries: 3\n"
+ " start_period: 60s");
assertServiceReadinessHealthcheck(compose, "gb32960-ingest-app", "20100");
assertServiceReadinessHealthcheck(compose, "jt808-ingest-app", "20400");
assertServiceReadinessHealthcheck(compose, "yutong-mqtt-app", "20500");
assertServiceReadinessHealthcheck(compose, "vehicle-history-app", "20200");
assertServiceReadinessHealthcheck(compose, "vehicle-analytics-app", "20310");
}
@Test
void containerDefaultsIncludeJava25UnsafeCompatibilityFlag() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String dockerfile = Files.readString(repositoryRoot().resolve("Dockerfile"));
assertThat(dockerfile)
.contains("ENV JAVA_OPTS=\"--sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true\"");
assertThat(commonEnvBlock(compose))
.contains("JAVA_OPTS: ${JAVA_OPTS:---sun-misc-unsafe-memory-access=allow -Dio.netty.transport.noNative=true -XX:+UseZGC -XX:MaxRAMPercentage=75}");
}
@Test
void redisSessionDefaultsUseEcsPrivateEndpoint() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
assertThat(compose)
.contains("KAFKA_GROUP_HISTORY_GB32960_EVENT: ${KAFKA_GROUP_HISTORY_GB32960_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-gb32960-event}")
.contains("KAFKA_GROUP_HISTORY_JT808_EVENT: ${KAFKA_GROUP_HISTORY_JT808_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-jt808-event}")
.contains("KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT:-${KAFKA_GROUP_HISTORY:-vehicle-history}-yutong-mqtt-event}")
.contains("KAFKA_GROUP_HISTORY_GB32960_RAW: ${KAFKA_GROUP_HISTORY_GB32960_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-gb32960-raw}")
.contains("KAFKA_GROUP_HISTORY_JT808_RAW: ${KAFKA_GROUP_HISTORY_JT808_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-jt808-raw}")
.contains("KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW: ${KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW:-${KAFKA_GROUP_HISTORY:-vehicle-history}-yutong-mqtt-raw}")
.doesNotContain("\n KAFKA_GROUP_HISTORY_GB32960:")
.doesNotContain("\n KAFKA_GROUP_HISTORY_JT808:")
.doesNotContain("\n KAFKA_GROUP_HISTORY_MQTT_YUTONG:");
.contains("REDIS_HOST: ${REDIS_HOST:-r-bp1u741kij7e51i481.redis.rds.aliyuncs.com}")
.doesNotContain("r-bp1u741kij7e51i481pd.redis.rds.aliyuncs.com");
}
@Test
void activeAppsExposeHealthProbeGroupsByDefault() {
Path root = repositoryRoot();
List<Path> activeAppConfigs = List.of(
root.resolve("modules/apps/gb32960-ingest-app/src/main/resources/application.yml"),
root.resolve("modules/apps/jt808-ingest-app/src/main/resources/application.yml"),
root.resolve("modules/apps/yutong-mqtt-app/src/main/resources/application.yml"),
root.resolve("modules/apps/vehicle-history-app/src/main/resources/application.yml"),
root.resolve("modules/apps/vehicle-analytics-app/src/main/resources/application.yml"));
for (Path config : activeAppConfigs) {
assertThat(yamlProperties(config))
.as(config.toString())
.containsEntry("management.endpoint.health.probes.enabled",
"${MANAGEMENT_HEALTH_PROBES_ENABLED:true}");
}
}
@Test
void productionRunbookChecksLivenessAndReadinessForActiveApps() throws IOException {
String runbook = Files.readString(repositoryRoot()
.resolve("docs/operations/gb32960-service-split-runbook.md"));
for (String httpPort : List.of("20100", "20400", "20500", "20200", "20310")) {
assertThat(runbook)
.contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/liveness")
.contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/readiness");
}
}
@Test
void defaultProductionSurfaceDoesNotShipLocalLaunchctlRunners() throws IOException {
Path root = repositoryRoot();
String runbook = Files.readString(root.resolve("docs/operations/gb32960-service-split-runbook.md"));
assertThat(root.resolve("deploy/local/launchctl")).doesNotExist();
assertThat(runbook)
.contains("Production services run on ECS through Portainer/Docker.")
.doesNotContain("Local launchctl Deployment")
.doesNotContain("deploy/local/launchctl")
.doesNotContain("launchctl bootstrap")
.doesNotContain("launchctl bootout");
}
@Test
void tdengineVerificationRunbookChecksLivenessAndReadinessForManagedApps() throws IOException {
String runbook = Files.readString(repositoryRoot()
.resolve("docs/operations/vehicle-ingest-tdengine-verification.md"));
for (String httpPort : List.of("20100", "20400", "20500", "20200", "20310")) {
assertThat(runbook)
.contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/liveness")
.contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/readiness");
}
}
@Test
void operationRunbooksUseBuildFlagThatSkipsTestCompilation() throws IOException {
Path operations = repositoryRoot().resolve("docs/operations");
String splitRunbook = Files.readString(operations.resolve("gb32960-service-split-runbook.md"));
String tdengineVerification = Files.readString(operations.resolve("vehicle-ingest-tdengine-verification.md"));
assertThat(splitRunbook)
.contains("-Dmaven.test.skip=true")
.doesNotContain("-DskipTests");
assertThat(tdengineVerification)
.contains("-Dmaven.test.skip=true")
.doesNotContain("-DskipTests");
}
@Test
void dockerEntrypointExecsJavaForSignalHandling() throws IOException {
String dockerfile = Files.readString(repositoryRoot().resolve("Dockerfile"));
assertThat(dockerfile)
.contains("ENTRYPOINT [\"sh\", \"-c\", \"exec java $JAVA_OPTS -jar /app/app.jar\"]")
.doesNotContain("\"java $JAVA_OPTS -jar /app/app.jar\"");
}
@Test
void gb32960ComposeExposesBothPlatformAccountsWithoutSecrets() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String gb32960Service = serviceBlock(compose, "gb32960-ingest-app");
assertThat(gb32960Service)
.contains("GB32960_PLATFORM_USER_HYUNDAI: ${GB32960_PLATFORM_USER_HYUNDAI:-Hyundai}")
.contains("GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}")
.contains("GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}")
.contains("GB32960_PLATFORM_USER_YUEJIN: ${GB32960_PLATFORM_USER_YUEJIN:-YueJin}")
.contains("GB32960_PLATFORM_PWD_YUEJIN: ${GB32960_PLATFORM_PWD_YUEJIN:-}")
.contains("GB32960_PLATFORM_IP_YUEJIN: ${GB32960_PLATFORM_IP_YUEJIN:-}");
}
@Test
void vehicleHistoryOnlyExposesBaseConsumerGroupEnvironmentName() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String historyService = serviceBlock(compose, "vehicle-history-app");
assertThat(historyService)
.contains("KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}")
.doesNotContain("KAFKA_GROUP_HISTORY_GB32960_EVENT")
.doesNotContain("KAFKA_GROUP_HISTORY_JT808_EVENT")
.doesNotContain("KAFKA_GROUP_HISTORY_YUTONG_MQTT_EVENT")
.doesNotContain("KAFKA_GROUP_HISTORY_GB32960_RAW")
.doesNotContain("KAFKA_GROUP_HISTORY_JT808_RAW")
.doesNotContain("KAFKA_GROUP_HISTORY_YUTONG_MQTT_RAW");
}
@Test
void kafkaConsumerEnvironmentIsScopedToConsumerAppsOnly() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String commonEnv = commonEnvBlock(compose);
String ingestServices = serviceBlock(compose, "gb32960-ingest-app")
+ serviceBlock(compose, "jt808-ingest-app")
+ serviceBlock(compose, "yutong-mqtt-app");
String historyService = serviceBlock(compose, "vehicle-history-app");
String analyticsService = serviceBlock(compose, "vehicle-analytics-app");
assertThat(commonEnv)
.doesNotContain("KAFKA_CONSUMER_ENABLED")
.doesNotContain("KAFKA_CONSUMER_MAX_POLL_INTERVAL");
assertThat(ingestServices)
.doesNotContain("KAFKA_CONSUMER_");
assertThat(historyService)
.contains("KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_HISTORY:-true}")
.contains("KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_HISTORY:-1800000}")
.doesNotContain("KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS");
assertThat(analyticsService)
.contains("KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED_ANALYTICS:-true}")
.contains("KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS_ANALYTICS:-900000}");
}
@Test
void kafkaTopicEnvironmentIsScopedToServicesThatUseIt() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String commonEnv = commonEnvBlock(compose);
String gb32960Service = serviceBlock(compose, "gb32960-ingest-app");
String jt808Service = serviceBlock(compose, "jt808-ingest-app");
String mqttService = serviceBlock(compose, "yutong-mqtt-app");
String historyService = serviceBlock(compose, "vehicle-history-app");
String analyticsService = serviceBlock(compose, "vehicle-analytics-app");
assertThat(commonEnv).doesNotContain("KAFKA_TOPIC_");
assertThat(gb32960Service)
.contains("KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}")
.contains("KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}")
.contains("KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}")
.doesNotContain("KAFKA_TOPIC_JT808")
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT");
assertThat(jt808Service)
.contains("KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}")
.contains("KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}")
.contains("KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}")
.doesNotContain("KAFKA_TOPIC_GB32960")
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT");
assertThat(mqttService)
.contains("KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}")
.contains("KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}")
.contains("KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}")
.doesNotContain("KAFKA_TOPIC_GB32960")
.doesNotContain("KAFKA_TOPIC_JT808");
assertThat(historyService)
.contains("KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}")
.contains("KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}")
.contains("KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}")
.contains("KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}")
.contains("KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}")
.contains("KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}")
.contains("KAFKA_TOPIC_HISTORY_DLQ: ${KAFKA_TOPIC_HISTORY_DLQ:-vehicle.dlq.history.v1}")
.doesNotContain("KAFKA_TOPIC_GB32960_DLQ")
.doesNotContain("KAFKA_TOPIC_JT808_DLQ")
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT_DLQ");
assertThat(analyticsService)
.contains("KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}")
.contains("KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}")
.doesNotContain("KAFKA_TOPIC_GB32960")
.doesNotContain("KAFKA_TOPIC_YUTONG_MQTT")
.doesNotContain("KAFKA_TOPIC_JT808_RAW");
}
@Test
@@ -44,6 +287,7 @@ class PortainerComposeResourceLimitsTest {
assertThat(compose)
.contains("TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}")
.contains("TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}")
.doesNotContain("TDENGINE_TELEMETRY_FIELDS_ENABLED")
.doesNotContain("\n TDENGINE_ENABLED:")
.doesNotContain("\n TDENGINE_DATABASE:");
}
@@ -62,17 +306,22 @@ class PortainerComposeResourceLimitsTest {
}
@Test
void localLaunchctlHistoryRuntimeDoesNotRequireSharedArchiveRoot() throws IOException {
Path launchctl = repositoryRoot().resolve("deploy/local/launchctl");
String historyTemplate = Files.readString(launchctl.resolve("com.lingniu.vehicle-history.plist.template"));
String readme = Files.readString(launchctl.resolve("README.md"));
void portainerIngestAppsWriteColdArchiveToMountedDataVolume() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String gb32960Service = serviceBlock(compose, "gb32960-ingest-app");
String jt808Service = serviceBlock(compose, "jt808-ingest-app");
String mqttService = serviceBlock(compose, "yutong-mqtt-app");
assertThat(historyTemplate)
.doesNotContain("SINK_ARCHIVE_PATH")
.doesNotContain("__SHARED_ARCHIVE_PATH__");
assertThat(readme)
.doesNotContain("三个服务必须使用同一个 `SINK_ARCHIVE_PATH`")
.doesNotContain("history 服务按同一个 archive root 读取原始帧");
assertThat(gb32960Service)
.contains("SINK_ARCHIVE_PATH: ${GB32960_ARCHIVE_PATH:-/data/archive}")
.contains("- gb32960-ingest-data:/data");
assertThat(jt808Service)
.contains("SINK_ARCHIVE_PATH: ${JT808_ARCHIVE_PATH:-/data/archive}")
.contains("- jt808-ingest-data:/data");
assertThat(mqttService)
.contains("SINK_ARCHIVE_PATH: ${YUTONG_MQTT_ARCHIVE_PATH:-/data/archive}")
.contains("- yutong-mqtt-data:/data");
assertThat(compose).contains("\n yutong-mqtt-data:\n");
}
@Test
@@ -83,9 +332,15 @@ class PortainerComposeResourceLimitsTest {
assertThat(tdengineVerification)
.contains("TDengine `raw_frames`")
.doesNotContain("20482")
.doesNotContain("三个服务必须共用同一个 `SINK_ARCHIVE_PATH`")
.doesNotContain("再回读共享 archive")
.doesNotContain("读取共享 archive 中的原始 `.bin`");
.doesNotContain("读取共享 archive 中的原始 `.bin`")
.doesNotContain("EVENT_FILE_STORE_ENABLED")
.doesNotContain("EVENT_FILE_STORE_PATH")
.doesNotContain("TDENGINE_TELEMETRY_FIELDS_ENABLED")
.doesNotContain("/api/event-history/telemetry/fields' 会存在")
.doesNotContain("/api/event-history/records");
assertThat(splitRunbook)
.contains("TDengine `raw_frames`")
.doesNotContain("must share the same `SINK_ARCHIVE_PATH`")
@@ -93,6 +348,84 @@ class PortainerComposeResourceLimitsTest {
.doesNotContain("shared volume mounted to all three services");
}
@Test
void splitRunbookDocumentsAllActiveProductionAppsAndTopics() throws IOException {
String splitRunbook = Files.readString(repositoryRoot()
.resolve("docs/operations/gb32960-service-split-runbook.md"));
String activeBuildCommand =
"mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true";
assertThat(splitRunbook)
.contains("| Yutong MQTT ingest | `:yutong-mqtt-app`")
.contains("`vehicle.raw.mqtt-yutong.v1`")
.contains("`vehicle.event.mqtt-yutong.v1`")
.contains("`vehicle.dlq.mqtt-yutong.v1`")
.contains("`vehicle.dlq.history.v1`")
.contains(activeBuildCommand)
.contains("modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar")
.doesNotContain("20482")
.doesNotContain("bootstrap-all")
.doesNotContain("mvn -pl :gb32960-ingest-app,:vehicle-history-app,:vehicle-analytics-app")
.doesNotContain("mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:vehicle-history-app,:vehicle-analytics-app")
.doesNotContain("repackaged all three app jars");
}
@Test
void tdengineVerificationRunbookDocumentsAllActiveProductionApps() throws IOException {
String runbook = Files.readString(repositoryRoot()
.resolve("docs/operations/vehicle-ingest-tdengine-verification.md"));
String activeBuildCommand =
"mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true";
assertThat(runbook)
.contains(activeBuildCommand)
.contains("modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar")
.contains("modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar")
.contains("modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar")
.contains("modules/apps/vehicle-history-app/target/vehicle-history-app.jar")
.contains("modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar")
.contains("vehicle.raw.mqtt-yutong.v1")
.contains("vehicle.event.mqtt-yutong.v1")
.contains("vehicle-history-yutong-mqtt-event")
.contains("vehicle-history-yutong-mqtt-raw")
.contains("vehicle-analytics-app")
.doesNotContain("mvn -pl modules/apps/jt808-ingest-app,modules/apps/gb32960-ingest-app,modules/apps/vehicle-history-app")
.doesNotContain("健康检查已确认三个服务");
}
@Test
void architectureDocsNameKafkaInsteadOfGenericMq() throws IOException {
String decisions = Files.readString(repositoryRoot().resolve("DECISIONS.md"));
String dataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
assertThat(decisions)
.contains("生产链路只支持 Kafka")
.doesNotContain("MQ 选型")
.doesNotContain("RocketMQ");
assertThat(dataFlow)
.contains("participant Kafka as sink-kafka")
.doesNotContain("participant MQ")
.doesNotContain("->>MQ")
.doesNotContain("MQ->>");
}
@Test
void activeSpringBootAppsDoNotScanWholeIngestPackage() throws IOException {
Path root = repositoryRoot();
List<Path> activeApps = List.of(
root.resolve("modules/apps/gb32960-ingest-app/src/main/java/com/lingniu/ingest/gb32960app/Gb32960IngestApplication.java"),
root.resolve("modules/apps/jt808-ingest-app/src/main/java/com/lingniu/ingest/jt808app/Jt808IngestApplication.java"),
root.resolve("modules/apps/yutong-mqtt-app/src/main/java/com/lingniu/ingest/yutongmqttapp/YutongMqttApplication.java"),
root.resolve("modules/apps/vehicle-history-app/src/main/java/com/lingniu/ingest/historyapp/VehicleHistoryApplication.java"),
root.resolve("modules/apps/vehicle-analytics-app/src/main/java/com/lingniu/ingest/analyticsapp/VehicleAnalyticsApplication.java"));
for (Path app : activeApps) {
assertThat(Files.readString(app))
.contains("@SpringBootApplication")
.doesNotContain("scanBasePackages = \"com.lingniu.ingest\"");
}
}
@Test
void splitRunbookKeepsAnalyticsAsJt808MetricRuntimeOnly() throws IOException {
String splitRunbook = Files.readString(repositoryRoot()
@@ -108,7 +441,7 @@ class PortainerComposeResourceLimitsTest {
}
@Test
void splitRunbookUsesTdengineHistoryVerificationInsteadOfEventFileStore() throws IOException {
void splitRunbookUsesTdengineHistoryVerificationInsteadOfLegacyRecordApi() throws IOException {
String splitRunbook = Files.readString(repositoryRoot()
.resolve("docs/operations/gb32960-service-split-runbook.md"));
@@ -135,6 +468,24 @@ class PortainerComposeResourceLimitsTest {
.doesNotContain("VEHICLE_STATE_ENABLED");
}
@Test
void kafkaConsumerAppsDoNotDependOnProtocolIngestContainers() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String historyService = serviceBlock(compose, "vehicle-history-app");
String analyticsService = serviceBlock(compose, "vehicle-analytics-app");
assertThat(historyService)
.doesNotContain(" depends_on:")
.doesNotContain("- gb32960-ingest-app")
.doesNotContain("- jt808-ingest-app")
.doesNotContain("- yutong-mqtt-app");
assertThat(analyticsService)
.doesNotContain(" depends_on:")
.doesNotContain("- gb32960-ingest-app")
.doesNotContain("- jt808-ingest-app")
.doesNotContain("- yutong-mqtt-app");
}
@Test
void vehicleAnalyticsDoesNotInheritRedisStateEnvironment() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
@@ -150,6 +501,80 @@ class PortainerComposeResourceLimitsTest {
.doesNotContain("REDIS_PASSWORD");
}
@Test
void mysqlEnvironmentIsScopedToIdentityAndAnalyticsOnly() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String commonEnv = commonEnvBlock(compose);
String identityEnv = blockBetween(compose, "x-identity-mysql-env: &identity-mysql-env\n",
"\nx-redis-session-env:");
String ingestServices = serviceBlock(compose, "gb32960-ingest-app")
+ serviceBlock(compose, "jt808-ingest-app")
+ serviceBlock(compose, "yutong-mqtt-app");
String analyticsService = serviceBlock(compose, "vehicle-analytics-app");
assertThat(commonEnv)
.doesNotContain("MYSQL_JDBC_URL")
.doesNotContain("MYSQL_USERNAME")
.doesNotContain("MYSQL_PASSWORD");
assertThat(identityEnv)
.contains("VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}")
.contains("VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}")
.contains("VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}");
assertThat(ingestServices)
.doesNotContain("\n MYSQL_JDBC_URL:")
.doesNotContain("\n MYSQL_USERNAME:")
.doesNotContain("\n MYSQL_PASSWORD:");
assertThat(analyticsService)
.contains("MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}")
.contains("MYSQL_USERNAME: ${MYSQL_USERNAME:-}")
.contains("MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}");
}
@Test
void ingestAppsReuseSharedIdentityAndRedisEnvironmentAnchors() throws IOException {
String compose = Files.readString(repositoryRoot().resolve("deploy/portainer/docker-compose.yml"));
String gb32960Service = serviceBlock(compose, "gb32960-ingest-app");
String jt808Service = serviceBlock(compose, "jt808-ingest-app");
String mqttService = serviceBlock(compose, "yutong-mqtt-app");
assertThat(compose)
.contains("x-identity-mysql-env: &identity-mysql-env")
.contains("x-redis-session-env: &redis-session-env")
.doesNotContain("VEHICLE_IDENTITY_STORE");
assertThat(gb32960Service)
.contains("- *identity-mysql-env")
.contains("- *redis-session-env");
assertThat(jt808Service)
.contains("- *identity-mysql-env")
.contains("- *redis-session-env");
assertThat(mqttService)
.contains("- *identity-mysql-env")
.doesNotContain("- *redis-session-env");
assertThat(gb32960Service + jt808Service + mqttService)
.doesNotContain("VEHICLE_IDENTITY_MYSQL_JDBC_URL")
.doesNotContain("VEHICLE_IDENTITY_MYSQL_USERNAME")
.doesNotContain("VEHICLE_IDENTITY_MYSQL_PASSWORD")
.doesNotContain("REDIS_HOST")
.doesNotContain("REDIS_PORT")
.doesNotContain("REDIS_DATABASE")
.doesNotContain("REDIS_USERNAME")
.doesNotContain("REDIS_PASSWORD");
}
@Test
void moduleDataFlowShowsSplitProductionAppsInsteadOfDeletedBootstrapAll() throws IOException {
String moduleDataFlow = Files.readString(repositoryRoot().resolve("docs/module-data-flow.html"));
assertThat(moduleDataFlow)
.contains("gb32960-ingest-app")
.contains("jt808-ingest-app")
.contains("yutong-mqtt-app")
.contains("vehicle-history-app")
.contains("vehicle-analytics-app")
.doesNotContain("bootstrap-all");
}
private static void assertServiceMemoryLimit(String compose,
String serviceName,
String envName,
@@ -160,6 +585,18 @@ class PortainerComposeResourceLimitsTest {
.contains(" mem_limit: ${" + envName + ":-" + defaultLimit + "}");
}
private static void assertServiceReadinessHealthcheck(String compose, String serviceName, String httpPort) {
assertThat(serviceBlock(compose, serviceName))
.contains(" healthcheck:\n")
.contains(" <<: *readiness-healthcheck\n")
.contains(" test: [\"CMD-SHELL\", \"curl -fsS http://127.0.0.1:"
+ httpPort + "/actuator/health/readiness >/dev/null\"]")
.doesNotContain(" interval: 30s\n")
.doesNotContain(" timeout: 5s\n")
.doesNotContain(" retries: 3\n")
.doesNotContain(" start_period: 60s\n");
}
private static String serviceBlock(String compose, String serviceName) {
String marker = "\n " + serviceName + ":\n";
int start = compose.indexOf(marker);
@@ -189,13 +626,46 @@ class PortainerComposeResourceLimitsTest {
if (start < 0) {
throw new IllegalArgumentException("common env block not found");
}
int end = compose.indexOf("\nservices:\n", start);
int end = compose.indexOf("\nx-identity-mysql-env:", start);
if (end < 0) {
end = compose.indexOf("\nservices:\n", start);
}
if (end < 0) {
throw new IllegalArgumentException("services block not found");
}
return compose.substring(start, end);
}
private static String blockBetween(String text, String startMarker, String endMarker) {
int start = text.indexOf(startMarker);
if (start < 0) {
throw new IllegalArgumentException("block start not found: " + startMarker);
}
int end = text.indexOf(endMarker, start + startMarker.length());
if (end < 0) {
throw new IllegalArgumentException("block end not found: " + endMarker);
}
return text.substring(start, end);
}
private static int countOccurrences(String text, String needle) {
int count = 0;
int index = 0;
while ((index = text.indexOf(needle, index)) >= 0) {
count++;
index += needle.length();
}
return count;
}
private static Properties yamlProperties(Path path) {
YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean();
factory.setResources(new FileSystemResource(path));
Properties properties = factory.getObject();
assertThat(properties).isNotNull();
return properties;
}
private static Path repositoryRoot() {
Path current = Path.of(System.getProperty("user.dir")).toAbsolutePath();
while (current != null) {

View File

@@ -3,7 +3,6 @@ package com.lingniu.ingest.historyapp;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.eventhistory.EventHistoryEnvelopeIngestor;
import com.lingniu.ingest.eventhistory.Gb32960DecodedFrameService;
import com.lingniu.ingest.eventhistory.Gb32960FrameController;
@@ -11,22 +10,16 @@ import com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration;
import com.lingniu.ingest.eventhistory.Jt808LocationHistoryController;
import com.lingniu.ingest.eventhistory.LocationHistoryController;
import com.lingniu.ingest.eventhistory.RawFrameHistoryController;
import com.lingniu.ingest.eventhistory.TelemetryFieldHistoryController;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.config.Gb32960AutoConfiguration;
import com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.mq.KafkaEventSink;
import com.lingniu.ingest.sink.mq.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.mq.SinkMqProperties;
import com.lingniu.ingest.sink.mq.proto.ParseStatusProto;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeDeadLetterSink;
import com.lingniu.ingest.sink.kafka.KafkaEventSink;
import com.lingniu.ingest.sink.kafka.KafkaEnvelopeConsumerRunner;
import com.lingniu.ingest.sink.kafka.KafkaSinkProperties;
import com.lingniu.ingest.sink.kafka.KafkaSinkAutoConfiguration;
import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
import com.lingniu.ingest.tdenginehistory.TdengineRawFrameRow;
import com.lingniu.ingest.tdenginehistory.config.TdengineHistoryAutoConfiguration;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.junit.jupiter.api.Test;
@@ -37,16 +30,12 @@ import org.springframework.context.annotation.Bean;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
class VehicleHistoryAppCompositionTest {
@@ -75,8 +64,8 @@ class VehicleHistoryAppCompositionTest {
() -> processor("event-history"))
.withBean("eventHistoryRawEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
() -> processor("event-history-raw"))
.withBean(SinkMqProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
.withPropertyValues("lingniu.ingest.sink.mq.consumer.enabled=true")
.withBean(KafkaSinkProperties.class, VehicleHistoryAppCompositionTest::genericOnlyHistoryConsumerProps)
.withPropertyValues("lingniu.ingest.sink.kafka.consumer.enabled=true")
.run(context -> {
assertThat(context).hasFailed();
assertThat(context.getStartupFailure())
@@ -85,12 +74,11 @@ class VehicleHistoryAppCompositionTest {
}
@Test
void createsTdengineHistoryStorageAndQueryBeansWithoutGb32960TcpServerEventFileStoreOrLocalArchiveSink() {
void createsTdengineHistoryStorageAndQueryBeansWithoutGb32960TcpServerOrLocalArchiveSink() {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(
TdengineHistoryAutoConfiguration.class,
SinkMqAutoConfiguration.class,
Gb32960AutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
EventHistoryAutoConfiguration.class))
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
.withAllowBeanDefinitionOverriding(true)
@@ -99,12 +87,9 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.tdengine-history.enabled=true",
"lingniu.ingest.tdengine-history.database=vehicle_history_test",
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.gb32960.enabled=true",
"lingniu.ingest.gb32960.server.enabled=false",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=false",
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=false",
"lingniu.ingest.vehicle-state.enabled=false",
"lingniu.ingest.vehicle-stat.enabled=false")
.run(context -> {
@@ -121,7 +106,7 @@ class VehicleHistoryAppCompositionTest {
assertThat(context).hasSingleBean(Gb32960DecodedFrameService.class);
assertThat(context).hasSingleBean(LocationHistoryController.class);
assertThat(context).hasSingleBean(RawFrameHistoryController.class);
assertThat(context).hasSingleBean(TelemetryFieldHistoryController.class);
assertThat(context).doesNotHaveBean("telemetryFieldHistoryController");
assertThat(context).doesNotHaveBean(Gb32960FrameController.class);
assertThat(context).doesNotHaveBean(Jt808LocationHistoryController.class);
assertThat(context).doesNotHaveBean(Gb32960NettyServer.class);
@@ -129,7 +114,7 @@ class VehicleHistoryAppCompositionTest {
}
@Test
void createsHistoryIngestorWithoutEventFileStoreForTdengineOnlyRuntime() {
void createsHistoryIngestorForTdengineOnlyRuntime() {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(EventHistoryAutoConfiguration.class))
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
@@ -137,19 +122,38 @@ class VehicleHistoryAppCompositionTest {
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
.withPropertyValues(
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=false")
"lingniu.ingest.sink.kafka.consumer.enabled=false")
.run(context -> {
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context).doesNotHaveBean("eventFileStore");
});
}
@Test
void applicationScanDoesNotExposeTelemetryFieldControllerByDefault() {
new ApplicationContextRunner()
.withUserConfiguration(VehicleHistoryApplication.class)
.withBean(TdengineHistoryReader.class, () -> mock(TdengineHistoryReader.class))
.withPropertyValues(
"spring.cloud.nacos.config.enabled=false",
"spring.config.import=",
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.tdengine-history.enabled=false",
"lingniu.ingest.sink.kafka.enabled=false",
"lingniu.ingest.sink.kafka.consumer.enabled=false")
.run(context -> {
assertThat(context).hasSingleBean(LocationHistoryController.class);
assertThat(context).hasSingleBean(RawFrameHistoryController.class);
assertThat(context).doesNotHaveBean("telemetryFieldHistoryController");
});
}
@Test
void createsKafkaConsumerRunnerWhenTdengineHistoryRuntimeConsumesKafka() {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(
TdengineHistoryAutoConfiguration.class,
SinkMqAutoConfiguration.class,
KafkaSinkAutoConfiguration.class,
EventHistoryAutoConfiguration.class))
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
.withAllowBeanDefinitionOverriding(true)
@@ -158,13 +162,12 @@ class VehicleHistoryAppCompositionTest {
"lingniu.ingest.tdengine-history.enabled=true",
"lingniu.ingest.tdengine-history.database=vehicle_history_test",
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.enabled=true",
"lingniu.ingest.sink.mq.type=kafka",
"lingniu.ingest.sink.mq.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.mq.consumer.enabled=true",
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
"lingniu.ingest.sink.mq.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
"lingniu.ingest.sink.kafka.enabled=true",
"lingniu.ingest.sink.kafka.bootstrap-servers=localhost:9092",
"lingniu.ingest.sink.kafka.consumer.enabled=true",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.enabled=true",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.group-id=history-gb32960",
"lingniu.ingest.sink.kafka.consumer.bindings.eventHistoryGb32960EnvelopeConsumerProcessor.topics[0]=vehicle.event.gb32960.v1")
.run(context -> {
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context.getBeanNamesForType(EnvelopeConsumerProcessor.class))
@@ -173,32 +176,6 @@ class VehicleHistoryAppCompositionTest {
});
}
@Test
void historyIngestorIgnoresEventFileStoreBeanAndWritesTdengineOnly() {
EventFileStore eventFileStore = mock(EventFileStore.class);
CapturingTdengineWriter writer = new CapturingTdengineWriter();
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(EventHistoryAutoConfiguration.class))
.withUserConfiguration(VehicleHistoryKafkaConsumerConfiguration.class)
.withBean(EventFileStore.class, () -> eventFileStore)
.withBean(TdengineHistoryWriter.class, () -> writer)
.withBean(EnvelopeDeadLetterSink.class, () -> mock(EnvelopeDeadLetterSink.class))
.withPropertyValues(
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=false")
.run(context -> {
EventHistoryEnvelopeIngestor ingestor = context.getBean(EventHistoryEnvelopeIngestor.class);
ingestor.tryIngest(rawFrameEnvelope().toByteArray());
verify(eventFileStore, never()).appendAll(any());
assertThat(writer.rawFrames)
.extracting(TdengineRawFrameRow::frameId)
.containsExactly("raw-frame-1");
});
}
@SuppressWarnings("unchecked")
private static KafkaProducer<String, byte[]> kafkaProducer() {
return mock(KafkaProducer.class);
@@ -211,8 +188,8 @@ class VehicleHistoryAppCompositionTest {
ignored -> {});
}
private static SinkMqProperties genericOnlyHistoryConsumerProps() {
SinkMqProperties props = new SinkMqProperties();
private static KafkaSinkProperties genericOnlyHistoryConsumerProps() {
KafkaSinkProperties props = new KafkaSinkProperties();
props.setBootstrapServers("localhost:9092");
props.getConsumer().setEnabled(true);
props.getConsumer().setBindings(Map.of(
@@ -221,50 +198,11 @@ class VehicleHistoryAppCompositionTest {
return props;
}
private static SinkMqProperties.Binding binding(String groupId, String topic) {
SinkMqProperties.Binding binding = new SinkMqProperties.Binding();
private static KafkaSinkProperties.Binding binding(String groupId, String topic) {
KafkaSinkProperties.Binding binding = new KafkaSinkProperties.Binding();
binding.setGroupId(groupId);
binding.setTopics(List.of(topic));
return binding;
}
private static VehicleEnvelope rawFrameEnvelope() {
return VehicleEnvelope.newBuilder()
.setSchemaVersion("1.0")
.setEventId("raw-event-1")
.setVin("VINRAW001")
.setSource("JT808")
.setEventTimeMs(1_782_112_400_000L)
.setIngestTimeMs(1_782_112_401_000L)
.setRawArchive(RawArchiveRef.newBuilder()
.setUri("archive://jt808/2026/06/29/raw-frame-1.bin")
.setSizeBytes(68))
.setRawFrameFact(RawFrameFactPayload.newBuilder()
.setFrameId("raw-frame-1")
.setVehicleKey("jt808:013800000000")
.setVin("VINRAW001")
.setPhone("013800000000")
.setMessageId(0x0200)
.setRawUri("archive://jt808/2026/06/29/raw-frame-1.bin")
.setRawSizeBytes(68)
.setParseStatus(ParseStatusProto.PARSE_STATUS_SUCCEEDED))
.build();
}
private static final class CapturingTdengineWriter implements TdengineHistoryWriter {
private final List<TdengineRawFrameRow> rawFrames = new ArrayList<>();
@Override
public void appendRawFrames(List<TdengineRawFrameRow> rows) {
rawFrames.addAll(rows);
}
@Override
public void appendLocations(List<com.lingniu.ingest.tdenginehistory.TdengineLocationRow> rows) {
}
@Override
public void appendTelemetryFields(List<com.lingniu.ingest.tdenginehistory.TdengineTelemetryFieldRow> rows) {
}
}
}

Some files were not shown because too many files have changed in this diff Show More