chore: keep go branch go-only
This commit is contained in:
lingniu
@@ -1,749 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>lingniu-vehicle-ingest 模块与数据流</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #f7f8fa;
|
||||
--panel: #ffffff;
|
||||
--ink: #172033;
|
||||
--muted: #667085;
|
||||
--line: #d8dee8;
|
||||
--blue: #2563eb;
|
||||
--green: #0f766e;
|
||||
--amber: #b45309;
|
||||
--red: #b42318;
|
||||
--purple: #6d28d9;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--ink);
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC", "Microsoft YaHei", sans-serif;
|
||||
line-height: 1.55;
|
||||
}
|
||||
|
||||
header {
|
||||
padding: 32px 40px 24px;
|
||||
background: #101828;
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
header h1 {
|
||||
margin: 0 0 10px;
|
||||
font-size: 30px;
|
||||
line-height: 1.2;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
header p {
|
||||
margin: 0;
|
||||
max-width: 980px;
|
||||
color: #d0d5dd;
|
||||
font-size: 15px;
|
||||
}
|
||||
|
||||
main {
|
||||
max-width: 1420px;
|
||||
margin: 0 auto;
|
||||
padding: 28px 28px 44px;
|
||||
}
|
||||
|
||||
section {
|
||||
margin-bottom: 24px;
|
||||
padding: 24px;
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 1px 2px rgba(16, 24, 40, 0.04);
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 0 0 14px;
|
||||
font-size: 21px;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
h3 {
|
||||
margin: 22px 0 10px;
|
||||
font-size: 17px;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0 0 12px;
|
||||
}
|
||||
|
||||
.summary-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 12px;
|
||||
margin-top: 18px;
|
||||
}
|
||||
|
||||
.summary-card {
|
||||
padding: 14px 16px;
|
||||
border: 1px solid var(--line);
|
||||
border-left: 4px solid var(--blue);
|
||||
border-radius: 6px;
|
||||
background: #fbfcfe;
|
||||
}
|
||||
|
||||
.summary-card strong {
|
||||
display: block;
|
||||
margin-bottom: 4px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.summary-card span {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.diagram {
|
||||
overflow-x: auto;
|
||||
padding: 12px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 6px;
|
||||
background: #ffffff;
|
||||
}
|
||||
|
||||
.legend {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
margin: 14px 0 0;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.legend span {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.dot {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.dot.entry { background: var(--blue); }
|
||||
.dot.core { background: var(--green); }
|
||||
.dot.sink { background: var(--amber); }
|
||||
.dot.safety { background: var(--red); }
|
||||
.dot.command { background: var(--purple); }
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-top: 12px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
th,
|
||||
td {
|
||||
padding: 10px 12px;
|
||||
border: 1px solid var(--line);
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
th {
|
||||
background: #f2f4f7;
|
||||
color: #344054;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
td:first-child {
|
||||
width: 190px;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
|
||||
font-size: 13px;
|
||||
color: #101828;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.tag {
|
||||
display: inline-block;
|
||||
margin: 0 6px 6px 0;
|
||||
padding: 2px 7px;
|
||||
border-radius: 999px;
|
||||
background: #eef2ff;
|
||||
color: #3730a3;
|
||||
font-size: 12px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.tag.red {
|
||||
background: #fee4e2;
|
||||
color: #912018;
|
||||
}
|
||||
|
||||
.tag.green {
|
||||
background: #ccfbef;
|
||||
color: #134e48;
|
||||
}
|
||||
|
||||
.tag.amber {
|
||||
background: #fef0c7;
|
||||
color: #93370d;
|
||||
}
|
||||
|
||||
.callout {
|
||||
margin-top: 14px;
|
||||
padding: 14px 16px;
|
||||
border-left: 4px solid var(--red);
|
||||
border-radius: 6px;
|
||||
background: #fff7f5;
|
||||
}
|
||||
|
||||
code {
|
||||
padding: 1px 4px;
|
||||
border-radius: 4px;
|
||||
background: #eef2f6;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
|
||||
font-size: 0.95em;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
header {
|
||||
padding: 24px 20px 20px;
|
||||
}
|
||||
|
||||
main {
|
||||
padding: 18px;
|
||||
}
|
||||
|
||||
section {
|
||||
padding: 18px;
|
||||
}
|
||||
|
||||
.summary-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
table {
|
||||
display: block;
|
||||
overflow-x: auto;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 560px) {
|
||||
.summary-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>lingniu-vehicle-ingest 模块与数据流</h1>
|
||||
<p>
|
||||
本图基于当前 Maven 多模块项目整理，用于后续讨论协议接入、内部字段、氢能车辆运营统计、安全告警和下游消费边界。
|
||||
当前架构将接入层、历史明细、Redis 热状态和日统计拆成独立模块，通过统一 Kafka Envelope 解耦。
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<section>
|
||||
<h2>一、系统定位</h2>
|
||||
<p>
|
||||
当前项目不是业务库写入服务。它的核心职责是把 GB/T 32960、JT/T 808、Yutong MQTT
|
||||
等默认生产来源的车辆数据统一转换为 <code>VehicleEvent</code> 和全字段 <code>TelemetrySnapshot</code>；
|
||||
JT/T 1078、JSATL12 作为显式 profile 的可选能力保留，
|
||||
再通过 Kafka、原始报文归档、TDengine 历史索引、Redis 热状态和独立统计模块交给下游业务。
|
||||
</p>
|
||||
<div class="summary-grid">
|
||||
<div class="summary-card">
|
||||
<strong>接入方式</strong>
|
||||
<span>Netty TCP、MQTT、后续可扩展更多 Inbound Adapter；信达 Push 源码已删除。</span>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<strong>统一模型</strong>
|
||||
<span>所有协议归一到 <code>VehicleEvent</code> 和全字段内部 <code>TelemetrySnapshot</code>。</span>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<strong>输出方式</strong>
|
||||
<span>Kafka Protobuf Envelope + 原始 bytes 冷存 + TDengine raw_frames/locations + Redis 热状态。</span>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<strong>业务边界</strong>
|
||||
<span>日统计由 <code>vehicle-stat-service</code> 消费 Kafka 实现，告警工单、资产业务继续由下游消费。</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>二、模块分层图</h2>
|
||||
<div class="diagram">
|
||||
<pre class="mermaid">
|
||||
flowchart TB
|
||||
subgraph external["外部来源"]
|
||||
vehicle["车辆终端<br/>GB/T 32960 / JT808 / JT1078"]
|
||||
mqttSource["车企或平台 MQTT"]
|
||||
commandClient["业务系统 / 运维平台<br/>HTTP 下行命令"]
|
||||
end
|
||||
|
||||
subgraph protocolModules["协议模块 modules/protocols"]
|
||||
gb["protocol-gb32960<br/>32960 Netty 接入、鉴权、ACK、解析"]
|
||||
jt808["protocol-jt808<br/>808 Netty 接入、会话、位置/批量位置、媒体、透传、下行"]
|
||||
jt1078["protocol-jt1078<br/>1078 信令 + TCP/UDP RTP 媒体流归档"]
|
||||
jsatl12["protocol-jsatl12<br/>主动安全附件流接入"]
|
||||
end
|
||||
|
||||
subgraph inbound["入口适配 modules/inbound"]
|
||||
mqtt["inbound-mqtt<br/>MQTT endpoint 生命周期、PEM TLS"]
|
||||
mqttProfile["MqttProfileRegistry<br/>按 endpoint profile 解析/映射"]
|
||||
end
|
||||
|
||||
subgraph protocolBase["核心与公共能力 modules/core"]
|
||||
codec["ingest-codec-common<br/>BCD / CRC / BCC / bit 工具"]
|
||||
session["session-core<br/>设备会话、命令分发接口、会话存储"]
|
||||
identity["vehicle-identity<br/>跨协议身份解析、MySQL 绑定表"]
|
||||
obs["observability<br/>metrics / health"]
|
||||
api["ingest-api<br/>ProtocolId / RawFrame / VehicleEvent / 注解 SPI"]
|
||||
registry["HandlerRegistry<br/>按协议、命令、infoType 路由"]
|
||||
dispatcher["Dispatcher<br/>RawFrame 到 Handler 到 EventBus"]
|
||||
bus["DisruptorEventBus<br/>高吞吐事件发布"]
|
||||
end
|
||||
|
||||
subgraph sink["输出与明细存储 modules/sinks"]
|
||||
kafka["sink-kafka<br/>VehicleEvent 到 Protobuf Envelope 到 Kafka"]
|
||||
archive["sink-archive<br/>RawArchive 到本地文件系统冷存"]
|
||||
tdengineStore["tdengine-history-store<br/>TDengine raw_frames + locations"]
|
||||
end
|
||||
|
||||
subgraph services["消费服务 modules/services"]
|
||||
history["event-history-service<br/>Kafka event/raw 到 TDengine 历史查询"]
|
||||
state["vehicle-state-service<br/>可选 Redis 热状态<br/>optional-latest-state"]
|
||||
stat["vehicle-stat-service<br/>Kafka 全字段事件到指标表"]
|
||||
end
|
||||
|
||||
subgraph apps["应用入口 modules/apps"]
|
||||
gbApp["gb32960-ingest-app<br/>GB32960 TCP 接入"]
|
||||
jtApp["jt808-ingest-app<br/>JT808 TCP 接入"]
|
||||
yutongApp["yutong-mqtt-app<br/>宇通 MQTT 接入"]
|
||||
historyApp["vehicle-history-app<br/>历史查询与 TDengine 写入"]
|
||||
analyticsApp["vehicle-analytics-app<br/>808 日里程指标消费"]
|
||||
gateway["command-gateway<br/>可选 HTTP 设备命令入口"]
|
||||
end
|
||||
|
||||
vehicle --> gbApp
|
||||
vehicle --> jtApp
|
||||
vehicle --> jt1078
|
||||
vehicle --> jsatl12
|
||||
mqttSource --> yutongApp
|
||||
gbApp --> gb
|
||||
jtApp --> jt808
|
||||
yutongApp --> mqtt
|
||||
mqtt --> mqttProfile
|
||||
mqttProfile --> api
|
||||
commandClient --> gateway
|
||||
|
||||
gb --> codec
|
||||
jt808 --> codec
|
||||
jt1078 --> codec
|
||||
jsatl12 --> codec
|
||||
|
||||
gb --> api
|
||||
jt808 --> api
|
||||
jsatl12 --> archive
|
||||
jsatl12 --> jt808
|
||||
|
||||
api --> registry
|
||||
registry --> dispatcher
|
||||
dispatcher --> bus
|
||||
bus --> kafka
|
||||
bus --> archive
|
||||
kafka --> historyApp
|
||||
kafka --> analyticsApp
|
||||
historyApp --> history
|
||||
analyticsApp --> stat
|
||||
kafka -.可选热状态.-> state
|
||||
history --> tdengineStore
|
||||
|
||||
gateway --> session
|
||||
session --> jt808
|
||||
session --> gb
|
||||
identity --> jt808
|
||||
identity --> mqttProfile
|
||||
|
||||
obs -.监控.-> dispatcher
|
||||
obs -.监控.-> bus
|
||||
obs -.监控.-> kafka
|
||||
|
||||
classDef entry fill:#eff6ff,stroke:#2563eb,color:#1e3a8a;
|
||||
classDef core fill:#ecfdf3,stroke:#0f766e,color:#134e48;
|
||||
classDef sink fill:#fffbeb,stroke:#b45309,color:#7c2d12;
|
||||
classDef command fill:#f5f3ff,stroke:#6d28d9,color:#4c1d95;
|
||||
classDef support fill:#f8fafc,stroke:#64748b,color:#334155;
|
||||
|
||||
class gb,jt808,jt1078,jsatl12,mqtt,mqttProfile,gbApp,jtApp,yutongApp,historyApp,analyticsApp entry;
|
||||
class api,registry,dispatcher,bus,identity core;
|
||||
class kafka,archive,tdengineStore sink;
|
||||
class gateway,session command;
|
||||
class codec,obs support;
|
||||
</pre>
|
||||
</div>
|
||||
<div class="legend">
|
||||
<span><i class="dot entry"></i>入口/协议模块</span>
|
||||
<span><i class="dot core"></i>核心管线</span>
|
||||
<span><i class="dot sink"></i>输出层</span>
|
||||
<span><i class="dot command"></i>下行命令/会话</span>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>三、上行数据流转</h2>
|
||||
<div class="diagram">
|
||||
<pre class="mermaid">
|
||||
sequenceDiagram
|
||||
autonumber
|
||||
participant Device as 车辆/平台
|
||||
participant Inbound as Inbound Adapter<br/>Netty/MQTT
|
||||
participant Decoder as FrameDecoder<br/>MessageDecoder
|
||||
participant Dispatcher as Dispatcher
|
||||
participant Handler as Protocol Handler<br/>Mapper
|
||||
participant EventBus as DisruptorEventBus
|
||||
participant Kafka as sink-kafka<br/>Kafka Envelope
|
||||
participant Archive as sink-archive<br/>ArchiveStore
|
||||
participant History as vehicle-history-app
|
||||
participant TDengine as tdengine-history-store<br/>raw_frames + locations
|
||||
participant Consumer as 下游业务消费者
|
||||
|
||||
Device->>Inbound: 原始报文 bytes / MQTT message
|
||||
Inbound->>Decoder: 粘包拆帧、校验、协议体解析
|
||||
Inbound->>Inbound: MQTT 按 endpoint profile 选择厂商解析器
|
||||
Decoder->>Dispatcher: RawFrame(protocol, command, infoType, payload, rawBytes)
|
||||
Dispatcher->>EventBus: 先发布 RawArchive 事件
|
||||
EventBus->>Archive: 保存原始 bytes，生成可回放材料
|
||||
Dispatcher->>Handler: 根据 protocol + command + infoType 路由
|
||||
Handler->>Handler: 协议字段映射为内部字段
|
||||
Dispatcher->>Handler: 绑定 rawArchiveKey / rawArchiveUri
|
||||
Handler->>EventBus: VehicleEvent(Realtime / Location / Alarm / Login ...)
|
||||
EventBus->>Kafka: 投递规整后的业务事件
|
||||
Kafka->>Kafka: EnvelopeMapper 转 Protobuf
|
||||
Kafka->>History: event/raw topic，key=vin，单车有序
|
||||
History->>TDengine: 写 raw_frames、locations 和 raw parsed JSON
|
||||
TDengine->>Consumer: 分页历史查询、原始帧追溯、位置查询
|
||||
Kafka->>Consumer: Redis 热状态、指标统计、告警工单等下游消费
|
||||
</pre>
|
||||
</div>
|
||||
<div class="callout">
|
||||
<strong>关键边界：</strong>
|
||||
协议字段只在协议模块内解释，出了 Mapper 以后都应该使用内部字段。统计每日里程、每日用电量、每日用氢量、储氢安全和氢泄露告警时，应消费全字段 TelemetrySnapshot，不要直接依赖某个协议的原始字段名。
|
||||
</div>
|
||||
<div class="callout">
|
||||
<strong>原始追溯：</strong>
|
||||
Dispatcher 为所有带 rawBytes 的 RawFrame 生成统一的 <code>rawArchiveKey</code>，业务事件 metadata 使用 <code>rawArchiveUri=archive://...</code> 指向同一份原始 bytes。归档文件的真实本地路径只属于 ArchiveStore，查询、导出和 Kafka Envelope 使用逻辑 URI 解耦存储实现。
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>四、32960 细化链路</h2>
|
||||
<div class="diagram">
|
||||
<pre class="mermaid">
|
||||
flowchart LR
|
||||
terminal["氢能车 TBOX<br/>GB/T 32960"] --> netty["Gb32960NettyServer<br/>TCP / TLS / Idle 检测"]
|
||||
netty --> access["Gb32960AccessService<br/>VIN 白名单 / 平台登录鉴权"]
|
||||
netty --> frame["Gb32960FrameDecoder<br/>拆帧、转义、校验"]
|
||||
frame --> decoder["Gb32960MessageDecoder<br/>Header + Body 解析"]
|
||||
decoder --> diag["Gb32960FrameDiagnostics<br/>首帧、Raw 块、异常诊断"]
|
||||
decoder --> handler["Gb32960ChannelHandler<br/>ACK / NACK / dispatch"]
|
||||
handler --> ack["Gb32960AckService<br/>登录应答、失败断开"]
|
||||
handler --> dispatcher["Dispatcher"]
|
||||
dispatcher --> rtHandler["Gb32960RealtimeHandler"]
|
||||
rtHandler --> mapper["Gb32960EventMapper<br/>协议字段到内部字段"]
|
||||
mapper --> realtime["RealtimePayload<br/>速度、里程、电量、氢量、压力、温度"]
|
||||
mapper --> alarm["AlarmPayload<br/>安全分类、氢泄露、告警等级"]
|
||||
mapper --> login["Login / Logout / Heartbeat"]
|
||||
realtime --> bus["DisruptorEventBus"]
|
||||
alarm --> bus
|
||||
login --> bus
|
||||
bus --> kafka["Kafka Envelope"]
|
||||
bus --> archive["RawArchive 冷存"]
|
||||
|
||||
classDef safety fill:#fff1f3,stroke:#b42318,color:#7a271a;
|
||||
classDef core fill:#ecfdf3,stroke:#0f766e,color:#134e48;
|
||||
classDef entry fill:#eff6ff,stroke:#2563eb,color:#1e3a8a;
|
||||
classDef sink fill:#fffbeb,stroke:#b45309,color:#7c2d12;
|
||||
|
||||
class terminal,netty,frame,decoder,handler entry;
|
||||
class access,diag,ack,dispatcher,rtHandler,mapper,bus core;
|
||||
class alarm safety;
|
||||
class kafka,archive sink;
|
||||
</pre>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>五、模块职责表</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>模块</th>
|
||||
<th>职责</th>
|
||||
<th>主要输入</th>
|
||||
<th>主要输出</th>
|
||||
<th>业务开发关注点</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>modules/core/ingest-api</td>
|
||||
<td>定义系统边界：协议 ID、RawFrame、IngestContext、VehicleEvent、Payload、Handler 注解、Sink SPI；ProtocolId 提供 UNKNOWN 隔离桶，供消费侧兼容未来协议或脏数据，不作为正式接入协议使用；consumer 包定义 EnvelopeIngestor、EnvelopeConsumerProcessor 和 EnvelopeDeadLetterSink，统一 Kafka 消费结果与死信边界。</td>
|
||||
<td>无直接外部输入。</td>
|
||||
<td>全项目共享的接口和内部事件模型。</td>
|
||||
<td><span class="tag red">内部字段定义</span><span class="tag">新增事件类型</span><span class="tag">字段兼容性</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/ingest-codec-common</td>
|
||||
<td>公共编解码工具，承载 BCD、CRC、BCC、bit 操作等协议底层能力。</td>
|
||||
<td>协议模块传入的 byte、bit、校验材料。</td>
|
||||
<td>解析辅助结果。</td>
|
||||
<td><span class="tag">协议基础能力</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/ingest-core</td>
|
||||
<td>核心管线。扫描 Handler，按协议和命令路由，执行拦截器，发布 VehicleEvent 到 Disruptor。</td>
|
||||
<td>RawFrame。</td>
|
||||
<td>VehicleEvent、RawArchive。</td>
|
||||
<td><span class="tag green">吞吐</span><span class="tag green">路由</span><span class="tag amber">原始可回放</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/session-core</td>
|
||||
<td>设备会话、命令下发抽象、SessionStore、CommandDispatcher 默认实现。SessionStore 仅保留 Redis 后端；协议进程本地只保存真实 Netty Channel，Redis 保存 <code>sessionId</code>、VIN 和 phone 三索引及 TTL，协议模块只依赖 SPI，不感知存储细节。</td>
|
||||
<td>设备连接、命令请求。</td>
|
||||
<td>会话状态、命令分发结果、Redis 会话索引。</td>
|
||||
<td><span class="tag">在线状态</span><span class="tag">下行控制</span><span class="tag green">多实例会话索引</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/vehicle-identity</td>
|
||||
<td>跨协议车辆身份解析，维护 phone、deviceId、plate 到 VIN 的绑定关系；生产运行使用 MySQL 绑定表，避免重启丢失绑定并保证多实例一致。</td>
|
||||
<td>协议模块传入的外部标识。</td>
|
||||
<td>稳定 VIN、解析来源、是否已命中绑定、MySQL 绑定记录。</td>
|
||||
<td><span class="tag red">统计主键</span><span class="tag">多协议归一</span><span class="tag green">持久化绑定</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/core/observability</td>
|
||||
<td>监控、指标、健康检查等横切能力。</td>
|
||||
<td>核心管线和 Sink 运行状态。</td>
|
||||
<td>Micrometer 指标、健康信息。</td>
|
||||
<td><span class="tag green">可用性</span><span class="tag">延迟监控</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/protocols/protocol-gb32960</td>
|
||||
<td>GB/T 32960 接入、鉴权、ACK、报文解析、实时数据和告警映射。</td>
|
||||
<td>32960 TCP 报文。</td>
|
||||
<td>Realtime、Location、Alarm、Login、Logout、Heartbeat、RawFrame。</td>
|
||||
<td><span class="tag red">氢泄露</span><span class="tag red">储罐安全</span><span class="tag">电量/氢量</span><span class="tag">里程</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/protocols/protocol-jt808</td>
|
||||
<td>JT/T 808 接入、注册/鉴权/心跳/注销、位置/位置附加项、批量位置、参数/属性、多媒体、透传、未知上行兜底透传、分帧边界异常和协议解析异常坏帧兜底、下行命令和超长下行分包能力；0x0200 位置附加项 0x01 总里程按 0.1km 解码为内部字段 <code>total_mileage_km</code>，后续每日里程统计不依赖 808 原始字段；0x0102 鉴权帧解析会保留完整 token，并尽量提取 IMEI 和软件版本；注册和鉴权会话均使用共享身份解析后的内部 VIN，避免 deviceId/IMEI 污染后续会话和下行边界；终端连接断开时同步解绑 Channel 并清理 <code>SessionStore</code> 会话，避免 command-gateway 误判离线车辆仍可下行；正常上行 RawArchive 和业务事件 metadata 均使用共享身份解析后的内部 vin，并保留 phone、identityResolved、identitySource 便于冷存和业务事件按同一车辆查询；无法解析终端身份的坏帧 RawArchive 和 Passthrough 均按 vin=unknown、identityResolved=false、identitySource=UNKNOWN 标记；0x0900 透传事件按原始消息 ID 归类，透传类型写入 passthroughType metadata。</td>
|
||||
<td>808 TCP 报文。</td>
|
||||
<td>Location、Login、Logout、Heartbeat、MediaMeta、Passthrough、未知上行 Raw 兜底、坏帧 Passthrough、会话状态、下行响应。</td>
|
||||
<td><span class="tag">GPS 位置</span><span class="tag">在线状态</span><span class="tag">媒体证据</span><span class="tag">命令链路</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/protocols/protocol-jt1078</td>
|
||||
<td>JT/T 1078 音视频能力。信令在 JT808 mapper 存在时桥接复用 JT808 连接和包头；0x1005 乘客流量保留原始 body，同时结构化 channelId、startTime、endTime、passengerGetOn、passengerGetOff metadata 便于查询/导出；0x1205 文件列表保留原始 body，同时结构化 responseSerialNo、fileCount 摘要；下行信令编码由本模块提供，覆盖 0x1003 音视频属性查询、0x9101 实时预览、0x9102 实时控制、0x9201 历史回放、0x9202 回放控制、0x9205 资源列表查询、0x9206 文件上传和 0x9207 文件上传控制，命令发送仍由 command-gateway 经 CommandDispatcher 复用 JT808 在线通道；TCP/UDP RTP 媒体流独立端口接入，生产默认端口按旧接收服务对齐为 <code>11078</code>，按 VIN/通道/时间分段写 ArchiveStore，Kafka 只发送 MediaMeta 引用，事件 metadata 暴露内部 vin、sim、channelId、dataType、packetType、segment、sequence、segmentSizeBytes、archiveKey 和 archiveRef，MediaMeta.sizeBytes 同步当前片段大小，MediaMeta 按 archiveKey 去重，避免同一 SIM 运行中从 fallback 身份切换到内部 VIN 时漏发新归档引用，便于文件明细库追踪、展示和导出；超长/短包/坏魔数等 RTP 入口异常和媒体归档失败都会兜底产出 Passthrough 错误事件，坏 RTP 统一走 Dispatcher 产出 RawArchive 和 Passthrough，统一标记 parseError 和 parseErrorMessage，并保留内部 vin、原始 bytes、peer、长度、原因、RawArchive 引用、segment 和 archiveKey 便于排障；坏 RTP 头部可读时会提取 SIM 并通过共享身份服务映射 VIN。</td>
|
||||
<td>1078 信令报文、TCP/UDP RTP 媒体流。</td>
|
||||
<td>MediaMeta、乘客流量 Passthrough、文件列表 Passthrough、坏 RTP/归档失败 Passthrough、归档媒体分段。</td>
|
||||
<td><span class="tag">视频扩展</span><span class="tag green">信令事件化</span><span class="tag amber">大流冷存</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/protocols/protocol-jsatl12</td>
|
||||
<td>苏标主动安全报警附件接入（仅 <code>optional-attachments</code> profile 显式构建，不属于默认生产面）。依赖 ArchiveStore 和 JT808 decoder；端口按旧接收服务对齐为 <code>7612</code>；附件 DataPacket 按旧服务真实布局 <code>01cd + 50B文件名 + offset + length + data</code> 分帧解析，不再把文件名前 4 字节误判为帧长；附件数据写 ArchiveStore 后通过 Dispatcher 产出 MediaMeta 引用事件，事件 metadata 保留 fileName、fileOffset、declaredChunkSizeBytes、archiveKey 和 archiveRef，便于附件明细查询、导出和补传诊断；内置附件状态机处理 T1210 文件清单、DataPacket 分块区间和 T1212 上传完成消息，数据分块会按 T1210 文件清单中的文件名继承手机号并通过共享身份解析器得到内部 VIN，入口 RawFrame、ArchivedChunk 和 MediaMeta 均保留 phone、vin、identityResolved、identitySource，只有找不到文件归属时才标记 UNKNOWN；按文件名合并无身份数据分块并返回 0x9212 完成/补传应答，补传应答包含缺失 offset/length 区间；附件归档失败会产出 JSATL12 Passthrough 错误事件并保留原始分块；JT_MESSAGE 信令复用 JT808 decoder 后进入 Dispatcher，桥接 RawFrame 使用共享身份解析后的内部 VIN并保留 phone、identityResolved、identitySource，坏 JT 信令入口 RawFrame 同样标记 UNKNOWN 身份；超长/畸形附件帧、未闭合超长 JT 信令和坏 JT 信令都以 JSATL12 Passthrough 兜底，附件帧异常同时标记 frameError 与统一 parseError/parseErrorMessage，保留原始字节、peer、长度和错误元数据，帧长和 worker 线程支持部署配置。</td>
|
||||
<td>主动安全附件流、JT808 信令帧。</td>
|
||||
<td>归档文件、MediaMeta、JT808 统一事件、坏帧/坏信令/归档失败 Passthrough。</td>
|
||||
<td><span class="tag amber">附件归档</span><span class="tag">报警证据</span><span class="tag">信令复用</span><span class="tag">错误隔离</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/inbound/inbound-mqtt</td>
|
||||
<td>MQTT 多 endpoint 生命周期管理，支持 PEM CA、客户端证书/私钥双向 TLS，按 endpoint profile 路由到厂商解析/映射实现，默认提供宇通 profile；生产默认接入口径参考旧接收服务：<code>ssl://cpxlm.axxc.cn:38883</code>、topic <code>/ytforward/shln/+</code>、QoS 2、cleanSession=false、keepAlive=20s、connectTimeout=10s，用户名和密码仍通过环境变量注入；入站 RawFrame 和事件映射均接入统一车辆身份解析，解析器保留 externalVin、phone、mqttDeviceId、plateNo，优先按设备号/终端 ID/IMEI 绑定解析内部 VIN，未绑定且字段形态为 VIN 时再作为显式 VIN 兜底，冷存和事件 metadata 均保留这些外部标识、identityResolved、identitySource；未知 profile、解析失败、profile 运行时异常、endpoint 初始化/连接/订阅失败均兜底为 Passthrough，解析失败诊断和 profile 异常诊断都会同步写入 RawArchive metadata，并在最终 Passthrough 事件保留 parseErrorMessage 或 operational reason，无法解析设备身份的兜底事件和运行类归档都按 vin=unknown、identityResolved=false、identitySource=UNKNOWN 标记，保留原始 payload 或 operational 错误信息便于排障和归档追溯。</td>
|
||||
<td>MQTT topic/message、endpoint profile、TLS PEM 配置。</td>
|
||||
<td>统一 RawFrame、Realtime、Location、Passthrough。</td>
|
||||
<td><span class="tag">车企平台接入</span><span class="tag green">profile 扩展</span><span class="tag green">身份映射</span><span class="tag green">双向 TLS</span><span class="tag green">错误隔离</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/sink-kafka</td>
|
||||
<td>Kafka Sink。把 VehicleEvent 转成 Protobuf Envelope，按 VIN 分区投递；业务事件携带全字段 TelemetrySnapshot 和 rawArchiveUri，显式 RawArchive Envelope 会填充 archive:// 逻辑 URI 与 size，默认 Kafka Sink 仍不发送原始 bytes 本体；同时提供 KafkaEnvelopeConsumerFactory、KafkaEnvelopeConsumerRunner、KafkaEnvelopeConsumerWorker 和 KafkaEnvelopeDeadLetterSink，统一服务侧 Kafka 消费启动、独立 group 绑定、死信发布和 topic 到 EnvelopeConsumerProcessor 的分发。</td>
|
||||
<td>VehicleEvent。</td>
|
||||
<td>Kafka Protobuf 消息、消费侧 DLQ 记录。</td>
|
||||
<td><span class="tag green">单车有序</span><span class="tag">Schema 演进</span><span class="tag red">安全字段下发给消费者</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/sink-archive</td>
|
||||
<td>原始报文冷存，当前为本地文件系统实现；写入键与业务事件 metadata 中的 <code>rawArchiveKey</code> 保持一致。</td>
|
||||
<td>RawArchive 事件、附件流。</td>
|
||||
<td>可回放原始文件、<code>archive://...</code> 逻辑引用。</td>
|
||||
<td><span class="tag amber">问题排查</span><span class="tag amber">合规留痕</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/sinks/tdengine-history-store</td>
|
||||
<td>生产历史库边界，负责 TDengine schema、raw_frames/location 行映射、批量写入和分页查询语句；raw_frames 保留完整 payloadJson.parsed，位置表保持轻量字段并通过 rawUri 关联原始帧。</td>
|
||||
<td>Kafka Protobuf Envelope、Raw Envelope。</td>
|
||||
<td>TDengine <code>raw_frames</code> 和位置表。</td>
|
||||
<td><span class="tag green">生产历史</span><span class="tag green">分页查询</span><span class="tag amber">raw 追溯</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/services/event-history-service</td>
|
||||
<td>消费 32960、808、宇通 MQTT 的 Kafka event/raw topic，生产默认写入 TDengine raw_frames 和位置表，并提供 raw 帧、位置历史等分页查询边界；Raw 查询返回完整 parsed JSON，位置历史只保留核心字段并通过 rawUri 关联原始帧；生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 将坏 protobuf、缺快照和存储异常收敛为结构化结果，并通过 <code>EnvelopeDeadLetterSink</code> 发布死信，避免阻塞 Kafka 分区。</td>
|
||||
<td>Kafka Protobuf Envelope。</td>
|
||||
<td>TDengine raw_frames、位置分页结果、raw archive 逻辑引用。</td>
|
||||
<td><span class="tag green">历史明细</span><span class="tag green">分页查询</span><span class="tag amber">raw JSON</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/services/vehicle-state-service</td>
|
||||
<td>消费 Kafka 全字段事件，更新车辆最新状态、位置、安全和最后事件到 Redis；仅通过 <code>optional-latest-state</code> profile 显式构建，不属于默认生产 reactor；生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf 和缺快照 Envelope，并把失败记录交给死信出口。</td>
|
||||
<td>Kafka Protobuf Envelope。</td>
|
||||
<td><code>vehicle:state:{vin}</code>、<code>vehicle:location:{vin}</code>、<code>vehicle:safety:{vin}</code>。</td>
|
||||
<td><span class="tag green">毫秒级热查询</span><span class="tag red">氢泄露安全</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/services/vehicle-stat-service</td>
|
||||
<td>消费 Kafka 全字段事件，808 每日里程只使用 0x0200 附加项 0x01 上报的总里程做当日首末差值；生产消费可使用 <code>tryIngest</code> 和自动装配的 <code>EnvelopeConsumerProcessor</code> 隔离坏 protobuf，缺少 telemetry_snapshot 的消息会标记为 SKIPPED 并进入死信出口。</td>
|
||||
<td>Kafka Protobuf Envelope、808 <code>total_mileage_km</code>。</td>
|
||||
<td><code>vehicle_stat_metric</code>：<code>metric_key=daily_mileage_km</code>、<code>calculation_method=JT808_TOTAL_MILEAGE_DIFF</code>。</td>
|
||||
<td><span class="tag green">每日里程</span><span class="tag">指标表</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/command-gateway</td>
|
||||
<td>可选 HTTP 到设备下行命令入口。支持会话查询、808 位置查询、参数查询/设置、终端控制、平台通用应答、终端属性查询 0x8107、区域删除 0x8601、人工报警确认 0x8203，以及 1078 音视频属性查询、实时预览/控制、历史回放/控制、资源列表查询、文件上传/控制；通过 CommandDispatcher 复用 protocol-jt808 的在线通道、流水号和同步应答等待能力。</td>
|
||||
<td>业务系统命令请求。</td>
|
||||
<td>CommandDispatcher 调用、设备应答摘要。</td>
|
||||
<td><span class="tag">远程控制</span><span class="tag">参数设置</span><span class="tag">属性查询</span><span class="tag">区域删除</span><span class="tag">报警确认</span><span class="tag">视频控制</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/gb32960-ingest-app</td>
|
||||
<td>生产 GB32960 TCP 接入应用，只负责接收、鉴权、解析、冷存引用和 Kafka 投递。</td>
|
||||
<td>GB/T 32960 TCP 32960。</td>
|
||||
<td><code>vehicle.raw.gb32960.v1</code>、<code>vehicle.event.gb32960.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">32960</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/jt808-ingest-app</td>
|
||||
<td>生产 JT808 TCP 接入应用，只负责 808 注册/鉴权/位置等上行解析、冷存引用和 Kafka 投递。</td>
|
||||
<td>JT/T 808 TCP 808。</td>
|
||||
<td><code>vehicle.raw.jt808.v1</code>、<code>vehicle.event.jt808.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">808</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/yutong-mqtt-app</td>
|
||||
<td>生产宇通 MQTT 接入应用，按 endpoint profile 解析 MQTT 报文并投递 Kafka。</td>
|
||||
<td>宇通 MQTT topic。</td>
|
||||
<td><code>vehicle.raw.mqtt-yutong.v1</code>、<code>vehicle.event.mqtt-yutong.v1</code>。</td>
|
||||
<td><span class="tag green">生产接入</span><span class="tag">MQTT</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/vehicle-history-app</td>
|
||||
<td>生产历史应用，消费 32960、808、宇通 MQTT 的 raw/event Kafka Envelope，写 TDengine 并提供历史查询。</td>
|
||||
<td>Kafka Protobuf Envelope。</td>
|
||||
<td>TDengine <code>raw_frames</code>、位置表和历史查询 API。</td>
|
||||
<td><span class="tag green">历史查询</span><span class="tag amber">raw JSON</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>modules/apps/vehicle-analytics-app</td>
|
||||
<td>生产指标应用，当前只消费 JT808 事件并用 GPS 总里程首末差值写每日里程指标。</td>
|
||||
<td><code>vehicle.event.jt808.v1</code>。</td>
|
||||
<td>MySQL <code>vehicle_stat_metric</code>。</td>
|
||||
<td><span class="tag green">每日里程</span><span class="tag">指标表</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>六、氢能业务开发落点</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>业务主题</th>
|
||||
<th>当前建议落点</th>
|
||||
<th>说明</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>车辆状态、速度、位置</td>
|
||||
<td><code>RealtimePayload</code>、<code>LocationPayload</code></td>
|
||||
<td>32960 和 808 都可能提供位置。统计侧需要明确优先级，例如优先 32960，缺失时用 808 补点。</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>每日里程</td>
|
||||
<td>Kafka 下游统计服务</td>
|
||||
<td>接入层只投递总里程和实时点位；日增里程建议下游按 VIN、自然日、事件时间聚合，处理回补和乱序。</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>每日用电量</td>
|
||||
<td>内部电池字段 + 下游统计</td>
|
||||
<td>接入层保留 SOC、电压、电流等瞬时字段；若有累计电耗字段，可加入内部字段模型后统一投递。</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>每日用氢量</td>
|
||||
<td><code>hydrogenRemainingKg</code> + 下游统计</td>
|
||||
<td>优先使用累计氢耗字段；没有累计值时用氢余量差值估算，并在统计结果里标记估算口径。</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>储氢安全</td>
|
||||
<td><code>RealtimePayload</code> 压力/温度 + <code>AlarmPayload.safetyCategory</code></td>
|
||||
<td>高压、低压、温度、异常 bit 都应统一归入储罐安全域，下游可做趋势、阈值和连续异常判断。</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>氢气泄露</td>
|
||||
<td><code>AlarmPayload.hydrogenLeakDetected</code></td>
|
||||
<td>当前已作为高优先级安全事件。32960 中出现 <code>HYDROGEN_LEAK</code> 时强制映射为 <code>CRITICAL</code>。</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>原始报文追溯</td>
|
||||
<td><code>RawArchive</code> + <code>rawArchiveUri</code> + <code>sink-archive</code></td>
|
||||
<td>业务统计出现争议时，用事件 metadata 中的 <code>archive://...</code> 逻辑 URI 回查原始 bytes，结合 eventId、traceId、VIN、时间复现解析链路。</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>明细展示和导出</td>
|
||||
<td><code>vehicle-history-app</code> + <code>tdengine-history-store</code></td>
|
||||
<td>生产查询通过 TDengine raw_frames、位置表和 rawUri 关联完成；不再维护文件型事件索引旁路。</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<div class="callout">
|
||||
<strong>后续开发建议：</strong>
|
||||
当前 Kafka Protobuf 已加入全字段 <code>TelemetrySnapshot</code>。后续业务统计服务继续只依赖内部字段，不依赖 32960 原始字段名，这样接入 808、MQTT、车企私有协议时不会重写统计口径。
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<script type="module">
|
||||
import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs";
|
||||
|
||||
mermaid.initialize({
|
||||
startOnLoad: true,
|
||||
theme: "base",
|
||||
securityLevel: "strict",
|
||||
flowchart: {
|
||||
htmlLabels: true,
|
||||
curve: "basis"
|
||||
},
|
||||
themeVariables: {
|
||||
fontFamily: "-apple-system, BlinkMacSystemFont, Segoe UI, PingFang SC, Microsoft YaHei, sans-serif",
|
||||
primaryColor: "#eff6ff",
|
||||
primaryTextColor: "#172033",
|
||||
primaryBorderColor: "#2563eb",
|
||||
lineColor: "#667085",
|
||||
secondaryColor: "#ecfdf3",
|
||||
tertiaryColor: "#fffbeb"
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,227 +0,0 @@
|
||||
# 当前 ECS Go 原生部署说明
|
||||
|
||||
更新时间：2026-07-02 01:30 CST
|
||||
|
||||
本文档记录当前 `lingniu-vehicle-ingest` 的生产运行面。当前 goal 的接入链路已经切到 Go 原生 systemd 部署，不再以 Docker/Portainer 作为生产运行方式。密钥只维护在 ECS 环境文件或受控凭据中，不写入 Git。
|
||||
|
||||
## 部署范围
|
||||
|
||||
| systemd 服务 | 二进制 | 说明 | 对外端口 |
|
||||
| --- | --- | --- | --- |
|
||||
| `lingniu-go-gateway.service` | `gateway` | GB32960 TCP、JT808 TCP、宇通 MQTT 接入，解析后写 Kafka RAW 和统一事件 | TCP `32960`、TCP `808` |
|
||||
| `lingniu-go-history-writer.service` | `history-writer` | 消费 Kafka RAW，写 TDengine RAW、位置点、里程点核心表 | 无 HTTP |
|
||||
| `lingniu-go-stat-writer.service` | `stat-writer` | 消费 Kafka RAW，按总里程差值法写 MySQL 每日指标 | 无 HTTP |
|
||||
| `lingniu-go-realtime-api.service` | `realtime-api` | 消费统一事件写 Redis，并提供实时、历史、统计查询 API | HTTP `20210` |
|
||||
|
||||
信达 Push 已废弃，不参与当前 Go 生产链路。旧 Java/Docker 服务不应占用 `808`、`32960`、`20210`。
|
||||
|
||||
## 应用 ECS
|
||||
|
||||
| 项 | 值 |
|
||||
| --- | --- |
|
||||
| 公网 IP | `115.29.187.205` |
|
||||
| 登录用户 | `root` |
|
||||
| 部署目录 | `/opt/lingniu-go-native` |
|
||||
| 当前 release | `/opt/lingniu-go-native/current` |
|
||||
| 环境文件目录 | `/opt/lingniu-go-native/env` |
|
||||
| systemd unit 目录 | `/etc/systemd/system` |
|
||||
| 对外服务 | `32960`、`808`、`20210` |
|
||||
|
||||
运行目录结构：
|
||||
|
||||
```text
|
||||
/opt/lingniu-go-native/
|
||||
current -> /opt/lingniu-go-native/releases/<git-short-sha>
|
||||
releases/<git-short-sha>/
|
||||
gateway
|
||||
history-writer
|
||||
stat-writer
|
||||
realtime-api
|
||||
env/
|
||||
gateway.env
|
||||
history-writer.env
|
||||
stat-writer.env
|
||||
realtime-api.env
|
||||
spool/gateway/
|
||||
```
|
||||
|
||||
## 中间件
|
||||
|
||||
| 组件 | 内网地址 | 公网地址 | 用途 |
|
||||
| --- | --- | --- | --- |
|
||||
| Kafka | `172.17.111.56:9092` | `114.55.58.251:9092` | RAW 和统一事件消息总线 |
|
||||
| TDengine | `172.17.111.57:6041` | `115.29.185.82:6041` | RAW、位置、里程点时序热存储 |
|
||||
| MySQL RDS | `rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306` | `rm-bp179zbv481rnw3e2no.mysql.rds.aliyuncs.com:3306` | 身份映射、JT808 注册、每日指标 |
|
||||
| Redis RDS | `r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379` | 无 | 准实时车辆状态缓存 |
|
||||
|
||||
生产应用之间访问中间件优先使用内网地址。RDS 白名单需要允许应用 ECS 私网访问。
|
||||
|
||||
## Kafka Topic
|
||||
|
||||
| Topic | 生产者 | 消费者 | 内容 |
|
||||
| --- | --- | --- | --- |
|
||||
| `vehicle.raw.go.gb32960.v1` | `gateway` | `history-writer`、`stat-writer` | GB32960 完整 RAW 记录 |
|
||||
| `vehicle.raw.go.jt808.v1` | `gateway` | `history-writer`、`stat-writer` | JT808 完整 RAW 记录 |
|
||||
| `vehicle.raw.go.yutong-mqtt.v1` | `gateway` | `history-writer` | 宇通 MQTT 完整 RAW 记录 |
|
||||
| `vehicle.event.go.unified.v1` | `gateway` | `realtime-api` | 三类协议统一事件，用于 Redis 实时状态 |
|
||||
|
||||
`gateway` 配置了 Kafka retry；生产 env 中启用 `KAFKA_SPOOL_DIR` 时，本地 spool 可在 Kafka 短暂不可用后回放。
|
||||
|
||||
Kafka 消费组：
|
||||
|
||||
| Consumer Group | Topic | 当前验证 |
|
||||
| --- | --- | --- |
|
||||
| `go-history-writer` | 三个 `vehicle.raw.go.*` topic | 2026-07-02 验证 lag 为 `0` |
|
||||
| `go-stat-writer` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1` | 2026-07-02 验证 lag 为 `0` |
|
||||
| `go-realtime-api` | `vehicle.event.go.unified.v1` | 2026-07-02 验证 lag 为 `0` |
|
||||
|
||||
可重复 smoke：
|
||||
|
||||
```bash
|
||||
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
|
||||
```
|
||||
|
||||
该脚本通过 SSH 登录 Kafka ECS，检查 Go 生产 topic 是否存在，并校验 `go-history-writer`、`go-stat-writer`、`go-realtime-api` 的消费组 lag。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
## TDengine 数据层
|
||||
|
||||
默认数据库：`lingniu_vehicle_ts`
|
||||
|
||||
| 表 | 类型 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `raw_frames` | stable | 完整 RAW 帧，包含 `raw_hex`、`parsed_json`、`fields_json`、`source_endpoint` 和协议/车辆 tags |
|
||||
| `vehicle_locations` | stable | 最小化位置历史，保留 `frame_id` 回链 RAW |
|
||||
| `vehicle_mileage_points` | stable | 最小化总里程点，供历史查询和统计复核 |
|
||||
|
||||
设计原则：
|
||||
|
||||
- 完整结构化 payload 只落在 `raw_frames.parsed_json` / `fields_json`。
|
||||
- `vehicle_locations` 和 `vehicle_mileage_points` 只保留核心查询字段，不重复保存完整 JSON。
|
||||
- `telemetry_fields` 不由当前服务维护，字段配置解析由独立子服务处理。
|
||||
- API 传入东八区时间时，服务会转换为 TDengine 当前存储使用的 UTC 字面量再查询。
|
||||
|
||||
## MySQL 数据层
|
||||
|
||||
默认业务库：`lingniu_vehicle_data`
|
||||
|
||||
| 表 | 维护方 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `vehicle_identity_binding` | 人工/外部主数据 | VIN 映射主表，用 `phone`、`device_id`、`plate` 降级定位 VIN |
|
||||
| `jt808_registration` | `gateway` 自动写入 | 仅 JT808 注册和鉴权记录，以 `phone` 作为主键，记录设备、车牌、厂家、鉴权、来源端点 |
|
||||
| `vehicle_daily_metric` | `stat-writer` 自动写入 | 每日指标表，保存 `daily_mileage_km` 和 `daily_total_mileage_km` |
|
||||
|
||||
每日里程算法：
|
||||
|
||||
```text
|
||||
daily_mileage_km = 当日最大 total_mileage_km - 当日最小 total_mileage_km
|
||||
daily_total_mileage_km = 当日最大 total_mileage_km
|
||||
```
|
||||
|
||||
统计按 `Asia/Shanghai` 自然日归档，当前支持 GB32960、JT808、宇通 MQTT 中能解析出 `total_mileage_km` 的数据。JT808 的总里程来自位置附加信息 `0x01`，单位按协议转换为 km；宇通 MQTT 的 `TOTAL_MILEAGE` 原始单位为米，入库前转换为 km。
|
||||
|
||||
## Redis 实时层
|
||||
|
||||
`realtime-api` 消费 `vehicle.event.go.unified.v1`，按 VIN/vehicle key 维护准实时快照。主要能力：
|
||||
|
||||
| API | 说明 |
|
||||
| --- | --- |
|
||||
| `/api/realtime/vehicles/{vin}` | 查询车辆合并实时快照 |
|
||||
| `/api/realtime/vehicles/{vin}/online` | 查询车辆是否在线 |
|
||||
| `/api/realtime/vehicles/{vin}/protocols/{protocol}` | 查询指定协议的实时快照 |
|
||||
|
||||
Redis 只作为实时缓存；历史和统计以 TDengine/MySQL 为准。
|
||||
|
||||
## API 入口
|
||||
|
||||
公网基础地址：
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210
|
||||
```
|
||||
|
||||
常用查询：
|
||||
|
||||
```bash
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/raw-frames?protocol=YUTONG_MQTT&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/locations?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/history/mileage-points?protocol=JT808&dateFrom=2026-07-02T00:00:00%2B08:00&dateTo=2026-07-03T00:00:00%2B08:00&limit=1'
|
||||
curl -sS 'http://115.29.187.205:20210/api/stats/daily-metrics?dateFrom=2026-07-02&dateTo=2026-07-02&limit=20'
|
||||
```
|
||||
|
||||
分页参数统一使用 `limit` 和 `offset`。
|
||||
|
||||
部署后推荐直接运行 Go 原生生产 smoke：
|
||||
|
||||
```bash
|
||||
python3 tools/go_prod_acceptance.py --date 2026-07-02
|
||||
```
|
||||
|
||||
该聚合脚本会串行执行 systemd/端口检查、Kafka topic/consumer lag 检查、HTTP 数据链路检查，任一子检查失败时整体退出码为非 0。需要 SSH 可登录应用 ECS 和 Kafka ECS。
|
||||
|
||||
也可以单独运行子检查：
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
|
||||
```
|
||||
|
||||
`go_systemd_prod_smoke.py` 通过 SSH 检查四个 Go systemd 服务是否 `active` 且 `enabled`，并确认 `808`、`32960` 由 `gateway` 监听、`20210` 由 `realtime-api` 监听，同时要求 `/opt/lingniu-go-native/current` 指向有效 release、四个生产二进制均可执行、`/opt/lingniu-go-native/spool/gateway` 没有待回放文件，避免旧 Java/Docker 进程占用生产端口、部署目录错乱或 Kafka spool 静默堆积。运行环境需要已配置 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
`go_native_prod_smoke.py` 通过 `20210` HTTP API 验证 GB32960、JT808、宇通 MQTT 的 RAW 查询及结构化 `parsed_json`，三类协议的位置和里程点查询及 `frame_id` RAW 回链，GB32960/JT808 的 `daily_mileage_km`、`daily_total_mileage_km` 统计公式，以及三类协议的 Redis realtime snapshot、online、protocol 查询。实时 snapshot/protocol 查询会校验 `updated_at_ms` 新鲜度；默认检查今天东八区数据时，还会要求三类 RAW 最新样本不超过 15 分钟，避免旧数据误判为接收正常；任一检查不达标时退出码为非 0。
|
||||
|
||||
## 部署命令
|
||||
|
||||
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制，打包上传到应用 ECS，切换 `/opt/lingniu-go-native/current`，逐个重启四个 systemd 服务，等待 gateway Kafka spool 清空，并在发布后运行聚合生产验收：
|
||||
|
||||
```bash
|
||||
python3 tools/go_native_deploy.py --date 2026-07-02
|
||||
```
|
||||
|
||||
脚本不会保存 SSH 或数据库密钥；认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`，但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒，可用 `--spool-drain-timeout` 和 `--spool-poll-interval` 调整。
|
||||
|
||||
脚本执行的发布动作等价于：
|
||||
|
||||
```bash
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/gateway
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/history-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/stat-writer
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' ./cmd/realtime-api
|
||||
scp release.tar.gz root@115.29.187.205:/tmp/
|
||||
ln -sfn /opt/lingniu-go-native/releases/<git-short-sha> /opt/lingniu-go-native/current
|
||||
systemctl daemon-reload
|
||||
systemctl restart lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
```
|
||||
|
||||
只更新查询 API 时，可以仅替换 `/opt/lingniu-go-native/current/realtime-api` 并重启：
|
||||
|
||||
```bash
|
||||
systemctl restart lingniu-go-realtime-api.service
|
||||
```
|
||||
|
||||
## 运行检查
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
journalctl -u lingniu-go-gateway --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-history-writer --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-stat-writer --since '5 minutes ago' --no-pager
|
||||
journalctl -u lingniu-go-realtime-api --since '5 minutes ago' --no-pager
|
||||
```
|
||||
|
||||
2026-07-02 01:30 CST 已验证：
|
||||
|
||||
- 四个 Go systemd 服务均为 `active` 且 `enabled`。
|
||||
- `/opt/lingniu-go-native/current` 指向有效 release，四个生产二进制均存在且可执行。
|
||||
- `gateway` 正在监听 `32960` 和 `808`。
|
||||
- `realtime-api` 正在监听 `20210`。
|
||||
- gateway Kafka spool 当前无待回放文件。
|
||||
- GB32960、JT808、YUTONG_MQTT 的东八区 RAW 查询均可命中生产数据，且最新 RAW 样本包含结构化 `parsed_json`。
|
||||
- GB32960、JT808、YUTONG_MQTT 的最新 RAW 样本均在 15 分钟内。
|
||||
- GB32960、JT808、YUTONG_MQTT 的 `vehicle_locations`、`vehicle_mileage_points` 查询均可命中生产数据，且样本包含 `frame_id` 回链 RAW。
|
||||
- `vehicle_daily_metric` 可查到 `daily_mileage_km` 和 `daily_total_mileage_km`，且公式满足 `daily_mileage_km = latest_total_mileage_km - first_total_mileage_km`、`daily_total_mileage_km = latest_total_mileage_km`。
|
||||
- Redis realtime 可查到 GB32960、JT808、YUTONG_MQTT 的在线状态和协议快照，且 snapshot/protocol 快照最近刷新。
|
||||
@@ -1,273 +0,0 @@
|
||||
# GB32960 Split Service Runbook
|
||||
|
||||
This runbook covers the default production runtimes: GB32960 ingest, JT808 ingest, Yutong MQTT ingest, history, and analytics. Kafka is the durability boundary between protocol ingest services and downstream business services.
|
||||
|
||||
## Service Roles
|
||||
|
||||
| Service | Module | Role | Production ports |
|
||||
| --- | --- | --- | --- |
|
||||
| GB32960 ingest | `:gb32960-ingest-app` | Accept GB32960 TCP connections, decode/authenticate frames, produce raw and normalized records to Kafka, and send protocol ACKs after required Kafka production succeeds. | TCP `32960`, HTTP `20100` |
|
||||
| JT808 ingest | `:jt808-ingest-app` | Accept JT/T 808 TCP connections on the production receive port, parse raw frames, archive frame bytes, and produce raw/event envelopes to Kafka. | TCP `808`, HTTP `20400` |
|
||||
| Yutong MQTT ingest | `:yutong-mqtt-app` | Connect to Yutong MQTT, parse subscribed messages, archive payloads, and produce raw/event envelopes to Kafka. | HTTP `20500` |
|
||||
| Vehicle history | `:vehicle-history-app` | Consume GB32960/JT808/Yutong MQTT raw/event Kafka records, store compact history rows plus TDengine `raw_frames` `parsedJson`/`metadataJson`/`rawUri`, and expose TDengine-backed history APIs. | HTTP `20200` |
|
||||
| Vehicle analytics | `:vehicle-analytics-app` | Consume JT808 event Kafka records, calculate daily mileage from reported GPS total mileage, and write metrics to `vehicle_stat_metric`. | HTTP `20310` |
|
||||
|
||||
## Kafka Contract
|
||||
|
||||
| Topic | Producer | Consumers | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `vehicle.raw.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Raw frame records. History writes the parsed JSON, metadata JSON, and raw URI to TDengine `raw_frames`. |
|
||||
| `vehicle.event.gb32960.v1` | `gb32960-ingest-app` | `vehicle-history-app` | Normalized vehicle events keyed by VIN when available. |
|
||||
| `vehicle.dlq.gb32960.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed production or consumer processing. |
|
||||
| `vehicle.raw.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app` | JT808 raw frame records for TDengine `raw_frames` and raw-frame troubleshooting APIs. |
|
||||
| `vehicle.event.jt808.v1` | `jt808-ingest-app` | `vehicle-history-app`, `vehicle-analytics-app` | JT808 parsed location/session/alarm events keyed by phone or mapped VIN; analytics uses location total mileage for daily mileage metrics. |
|
||||
| `vehicle.dlq.jt808.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed JT808 production or consumer processing. |
|
||||
| `vehicle.raw.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT raw payload records for TDengine `raw_frames` and troubleshooting APIs. |
|
||||
| `vehicle.event.mqtt-yutong.v1` | `yutong-mqtt-app` | `vehicle-history-app` | Yutong MQTT parsed telemetry events keyed by mapped VIN or device identity. |
|
||||
| `vehicle.dlq.mqtt-yutong.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed Yutong MQTT production or consumer processing. |
|
||||
| `vehicle.dlq.history.v1` | History app consumer error paths | Operators/replay tooling | Dead-letter records for failed history storage processing. |
|
||||
|
||||
Default consumer groups:
|
||||
|
||||
- History: `vehicle-history`
|
||||
- Analytics statistics: `vehicle-stat`
|
||||
|
||||
## ACK Semantics
|
||||
|
||||
GB32960 success ACKs are sent only after the ingest service completes the required Kafka production boundary for the accepted frame. If required Kafka production fails, the ingest service must not claim a successful ACK for that frame.
|
||||
|
||||
History and analytics failures do not block GB32960 ACKs. They are downstream Kafka consumer failures and should be handled through retry, DLQ inspection, offset replay, and service-specific recovery.
|
||||
|
||||
Authorization failures are rejected before the Kafka durability boundary and can be ACKed/rejected immediately according to protocol handling.
|
||||
|
||||
## Build Prerequisites
|
||||
|
||||
- Java 25 and Maven available on PATH.
|
||||
- Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer` available on the ECS/operator host.
|
||||
- Session state requires Redis. GB32960/JT808 ingest services keep live Netty
|
||||
channels in-process, but session indexes and TTL metadata use Redis only.
|
||||
- Redis is configured through Portainer environment variables such as `REDIS_HOST` and `REDIS_DATABASE=50`; there is no memory session-store fallback.
|
||||
|
||||
If your shell needs an explicit JDK:
|
||||
|
||||
```bash
|
||||
export JAVA_HOME=/opt/homebrew/opt/openjdk@25/libexec/openjdk.jdk/Contents/Home
|
||||
export PATH="$JAVA_HOME/bin:/opt/homebrew/bin:$PATH"
|
||||
```
|
||||
|
||||
## Build
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
Expected result: `BUILD SUCCESS` and these runnable jars:
|
||||
|
||||
- `modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar`
|
||||
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
|
||||
- `modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar`
|
||||
- `modules/apps/vehicle-history-app/target/vehicle-history-app.jar`
|
||||
- `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar`
|
||||
|
||||
## Create Topics
|
||||
|
||||
Run these only on the ECS/operator host after Kafka is reachable. The default bootstrap address is the ECS private Kafka address used by the production stack:
|
||||
|
||||
```bash
|
||||
KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:-172.17.111.56:9092}"
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.jt808.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.jt808.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.jt808.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.mqtt-yutong.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.mqtt-yutong.v1 --partitions 12 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.mqtt-yutong.v1 --partitions 3 --replication-factor 1
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.history.v1 --partitions 3 --replication-factor 1
|
||||
```
|
||||
|
||||
Optional sanity check:
|
||||
|
||||
```bash
|
||||
kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808|mqtt-yutong|history)\.v1'
|
||||
```
|
||||
|
||||
## Production Runtime
|
||||
|
||||
Production services run on ECS through Portainer/Docker. Do not start GB32960, JT808, Yutong MQTT, history, or analytics services on local developer machines; local machines are for code build, unit tests, and source inspection only.
|
||||
|
||||
Use `deploy/portainer/docker-compose.yml` as the production runtime manifest. The default stack is:
|
||||
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
- `vehicle-history-app`
|
||||
- `vehicle-analytics-app`
|
||||
|
||||
`vehicle-history-app` intentionally has no raw archive volume. The history hot path is Kafka to TDengine: raw envelopes are stored in TDengine `raw_frames` with `parsedJson`, `metadataJson`, and `rawUri`. Ingest services may still write original `.bin` files as cold backup, but history API queries must not depend on a shared local archive mount.
|
||||
|
||||
Keep `TDENGINE_HISTORY_ENABLED=true` in the `vehicle-history-app` environment so history reads and writes use TDengine in production.
|
||||
|
||||
`vehicle-analytics-app` is intentionally a metric runtime only. Latest-state APIs belong to `vehicle-state-service`, which stays outside the default reactor and should be built with `-Poptional-latest-state` and deployed as a separate consumer if that surface is needed.
|
||||
|
||||
## Health Verification
|
||||
|
||||
Run these from the ECS host or inside the matching container:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
Expected: each aggregate, liveness, and readiness endpoint returns `{"status":"UP"}` or an equivalent Spring Boot health JSON with top-level status `UP`.
|
||||
|
||||
## End-to-End Verification
|
||||
|
||||
Use a known-valid GB32960 fixture, for example one of the hex samples under `modules/protocols/protocol-gb32960/src/test/resources/samples/gb32960/`.
|
||||
|
||||
Example send command:
|
||||
|
||||
```bash
|
||||
xxd -r -p modules/protocols/protocol-gb32960/src/test/resources/samples/gb32960/realtime_001.hex | nc 127.0.0.1 32960
|
||||
```
|
||||
|
||||
To capture and inspect the binary ACK, write the response to a file:
|
||||
|
||||
```bash
|
||||
rm -f /tmp/gb32960-ack.bin
|
||||
xxd -r -p modules/protocols/protocol-gb32960/src/test/resources/samples/gb32960/realtime_001.hex \
|
||||
| nc -w 3 127.0.0.1 32960 > /tmp/gb32960-ack.bin
|
||||
xxd -p /tmp/gb32960-ack.bin
|
||||
```
|
||||
|
||||
Expected: the ACK file is non-empty and the decoded bytes begin with the GB32960 frame marker `2323`. Treat the ACK as verified only after checking the response bytes from your run.
|
||||
|
||||
Verify only what was actually run in your environment:
|
||||
|
||||
```bash
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000
|
||||
```
|
||||
|
||||
Use TDengine CLI or a JDBC client to verify history writes:
|
||||
|
||||
```sql
|
||||
USE vehicle_ts;
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'GB32960';
|
||||
SELECT COUNT(*) FROM vehicle_locations WHERE protocol = 'GB32960';
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM jt808_locations WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'MQTT_YUTONG';
|
||||
SELECT COUNT(*) FROM vehicle_locations WHERE protocol = 'MQTT_YUTONG';
|
||||
```
|
||||
|
||||
Useful HTTP query checks after events are consumed. For `realtime_001.hex`, the fixture VIN is `LTEST000000000001`; replace `<date-from>`, `<date-to>`, and `<stat-date>` with values that cover the consumed record's event time:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/locations?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/raw-frames?protocol=GB32960&dateFrom=<date-from>&dateTo=<date-to>&vin=LTEST000000000001&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/dictionary'
|
||||
curl -sS 'http://127.0.0.1:20310/api/vehicle-stat/LTEST000000000001/daily?date=<stat-date>'
|
||||
```
|
||||
|
||||
Expected E2E result when Kafka and all services are running:
|
||||
|
||||
- A GB32960 client receives a binary success ACK only after required Kafka production succeeds, and the captured ACK bytes are inspected.
|
||||
- `vehicle.raw.gb32960.v1` receives a raw frame envelope.
|
||||
- `vehicle.event.gb32960.v1` receives one or more normalized records.
|
||||
- TDengine `raw_frames` receives GB32960/JT808 RAW rows with parsed JSON and metadata.
|
||||
- TDengine `raw_frames` receives Yutong MQTT RAW rows with parsed JSON and metadata when MQTT payloads are consumed.
|
||||
- TDengine location tables receive compact GB32960/JT808/Yutong MQTT location rows when applicable telemetry is present.
|
||||
- JT808 analytics writes daily mileage rows to MySQL `vehicle_stat_metric` after consuming applicable `vehicle.event.jt808.v1` records.
|
||||
|
||||
Do not expect `vehicle-history-app` to create raw `.bin` archive files from Kafka raw records in the current implementation. `RawArchiveEventSink` can write archive files only when it receives `VehicleEvent.RawArchive.rawBytes()` inside the same JVM; the Kafka envelope carries only `RawArchiveRef` metadata.
|
||||
|
||||
Do not mark any of these as verified unless the matching command was run and the output was inspected.
|
||||
|
||||
## Runtime Location Policy
|
||||
|
||||
Production services run on ECS through Portainer/Docker.
|
||||
|
||||
- Do not create local service runners for GB32960, JT808, Yutong MQTT, history, or analytics.
|
||||
- Do not run the service jars directly on developer machines for production verification.
|
||||
- Keep local work limited to source edits, builds, and automated tests.
|
||||
|
||||
For Portainer deployments, keep history without an archive volume: history consumes raw/event Kafka topics and writes TDengine history rows.
|
||||
|
||||
## Rollback Guidance
|
||||
|
||||
If the split deployment is unhealthy:
|
||||
|
||||
1. Stop `gb32960-ingest-app` first so new GB32960 ACKs are not issued against an unhealthy Kafka boundary.
|
||||
2. Keep Kafka topics intact for replay unless storage or privacy policy requires deletion.
|
||||
3. Restart or roll back `vehicle-history-app` and `vehicle-analytics-app` independently; their failures do not require protocol ACK rollback because they consume from Kafka offsets.
|
||||
4. If Kafka itself is unavailable, stop `gb32960-ingest-app` or route GB32960 traffic to a previously verified release that preserves the Kafka durability boundary.
|
||||
5. After rollback, compare Kafka consumer group lag and DLQ contents before resuming the split services.
|
||||
|
||||
Rollback commands depend on the deployment environment. For the default production stack, roll back the Portainer stack image version and let Docker restart the affected containers.
|
||||
|
||||
## Task 8 Verification Notes
|
||||
|
||||
Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`:
|
||||
|
||||
- Package build should use the current active-app command shown in the Build section.
|
||||
- Repository-local Kafka setup inspection found no Kafka script, no Docker Compose file, and no compose YAML within the searched repository paths.
|
||||
- `nc -z -w 2 127.0.0.1 9092` exited `1`, so no local Kafka broker was reachable at `127.0.0.1:9092`.
|
||||
- `kafka-topics`, `kafka-topics.sh`, `docker`, and `docker-compose` were not found on PATH.
|
||||
- Kafka topic creation, service startup, health checks, Kafka record checks, archive checks, TDengine checks, stat output checks, and ACK observation were not run because the local Kafka prerequisite was absent.
|
||||
|
||||
## Latest Build Verification
|
||||
|
||||
Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`:
|
||||
|
||||
- Targeted module tests: `mvn -pl :sink-kafka,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test` ended with `BUILD SUCCESS`; Maven reported 55 protocol/sink/core tests, 30 event-history tests, 14 vehicle-state tests, and 19 vehicle-stat tests with 0 failures/errors/skips.
|
||||
- Split app tests should cover the current active apps listed in the Service Roles section.
|
||||
- Package verification should use the active-app package command from the Build section.
|
||||
- Split app startup: not run in this verification pass because an ECS/Portainer runtime was outside that build-only check.
|
||||
- Kafka raw records: not verified in that build-only check.
|
||||
- Kafka event records: not verified in that build-only check.
|
||||
- TDengine history rows: not verified; downstream services were not started without Kafka.
|
||||
- Analytics output: not verified; downstream services were not started without Kafka.
|
||||
- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above on ECS or staging.
|
||||
|
||||
## Remote Kafka Live Verification
|
||||
|
||||
Observed on 2026-06-23 with Kafka `114.55.58.251:9092`:
|
||||
|
||||
- `gb32960-ingest-app` handled TCP `32960` and HTTP `20100`.
|
||||
- Ingest app health check `curl -sS http://127.0.0.1:20100/actuator/health` returned `{"status":"UP"}`.
|
||||
- Kafka AdminClient confirmed topics `vehicle.raw.gb32960.v1` and `vehicle.event.gb32960.v1` existed, and created `vehicle.dlq.gb32960.v1`.
|
||||
- The app accepted the live Hyundai platform connection from `8.134.95.166`, authenticated platform login, and received real GB32960 realtime/report/login/logout frames.
|
||||
- Probe consumer with a temporary group read records from `vehicle.event.gb32960.v1`, including VIN keys such as `LB9A32A2XR0LS1575` and `LNXNEGRR3SR319485`.
|
||||
- Probe consumer with a temporary group read records from `vehicle.raw.gb32960.v1`, including VIN keys such as `LNXNEGRR0SR319444`, `LNXNEGRR2SR318201`, and `LB9A32A23R0LS1532`.
|
||||
- Runtime logs showed successful GB32960 ACK flushes for platform login, vehicle login, and vehicle logout after the Kafka-backed split ingest service was running.
|
||||
|
||||
This verifies the live path `GB32960 TCP 32960 -> gb32960-ingest-app -> Kafka raw/event topics` against the remote Kafka broker. Downstream `vehicle-history-app` and `vehicle-analytics-app` were not started in this pass.
|
||||
|
||||
## 2026-06-29 Superseded Local Verification Record
|
||||
|
||||
Observed on 2026-06-29 in worktree `.worktrees/vehicle-ingest-redesign-phase1`. This record is retained only as historical evidence of parser/storage behavior; do not repeat it as a deployment procedure because production services now run on ECS through Portainer/Docker.
|
||||
|
||||
- External JT808 traffic on TCP `808` was parsed and written through Kafka into TDengine. Direct TDengine checks showed more than `6000` JT808 `raw_frames` and more than `1900` JT808 `vehicle_locations`; current production history does not maintain a `telemetry_fields` table.
|
||||
- A GB32960 synthetic realtime frame sent to TCP `32960` received a binary ACK beginning with `2323`. The same VIN `LTEST202606290001` was queryable from TDengine-backed history APIs with speed, mileage, longitude, and latitude fields.
|
||||
- History raw and event Kafka consumption used separate bindings so raw-frame writes remained current while location/field event ingestion continued.
|
||||
|
||||
The verified behavior was `GB32960/JT808 TCP -> protocol app -> Kafka raw/event topics -> vehicle-history-app -> TDengine raw_frames/location tables -> Swagger/API query`; the deployment method from that historical run is superseded.
|
||||
@@ -1,888 +0,0 @@
|
||||
# Go Vehicle Gateway 旁路验证记录
|
||||
|
||||
验证时间：2026-07-01 22:00-22:10 CST
|
||||
|
||||
## 部署版本
|
||||
|
||||
- Git commit：`60ca42e`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-60ca42e-20260701220622`
|
||||
- 部署方式：ECS `115.29.187.205` 上 Docker Compose 旁路部署
|
||||
- Compose 目录：`/opt/lingniu-go`
|
||||
|
||||
## 旁路端口和 topic
|
||||
|
||||
为了不影响现有 Java 生产服务，本次没有占用生产端口和生产 topic。
|
||||
|
||||
| 项 | 值 |
|
||||
|---|---|
|
||||
| GB32960 旁路 TCP | `115.29.187.205:23296 -> container :32960` |
|
||||
| JT808 旁路 TCP | `115.29.187.205:18080 -> container :808` |
|
||||
| Realtime API | `115.29.187.205:20210` |
|
||||
| GB32960 RAW topic | `vehicle.raw.go.gb32960.v1` |
|
||||
| JT808 RAW topic | `vehicle.raw.go.jt808.v1` |
|
||||
| Yutong MQTT RAW topic | `vehicle.raw.go.yutong-mqtt.v1` |
|
||||
| Unified topic | `vehicle.event.go.unified.v1` |
|
||||
|
||||
## 容器状态
|
||||
|
||||
ECS 上四个 Go 容器均已启动：
|
||||
|
||||
```text
|
||||
go-vehicle-gateway Up
|
||||
go-history-writer Up
|
||||
go-stat-writer Up
|
||||
go-realtime-api Up
|
||||
```
|
||||
|
||||
## Kafka 验证
|
||||
|
||||
向 `18080` 注入 JT808 样例帧后，`vehicle.raw.go.jt808.v1` 和
|
||||
`vehicle.event.go.unified.v1` 均消费到统一 JSON envelope。
|
||||
|
||||
样例解析结果包含：
|
||||
|
||||
- `protocol=JT808`
|
||||
- `message_id=0x0200`
|
||||
- `phone=013307795425`
|
||||
- `longitude=121.069881`
|
||||
- `latitude=30.590151`
|
||||
- `speed_kmh=23`
|
||||
- `total_mileage_km=10241.2`
|
||||
- 表 27 附加项 `0x01` 原始值 `0001900C`
|
||||
|
||||
## TDengine 验证
|
||||
|
||||
目标库：`lingniu_vehicle_ts`
|
||||
|
||||
```text
|
||||
raw_frames count = 3
|
||||
vehicle_locations count = 2
|
||||
vehicle_mileage_points count = 2
|
||||
```
|
||||
|
||||
最新 RAW 标识：
|
||||
|
||||
```text
|
||||
GB32960 | LNBSCB3D4R1234567 | LNBSCB3D4R1234567 |
|
||||
JT808 | JT808:013307795425 | | 013307795425
|
||||
JT808 | JT808:013307795425 | | 013307795425
|
||||
```
|
||||
|
||||
说明：JT808 样例 phone 当前没有在 `vehicle_identity_binding` 中解析到 VIN，因此
|
||||
TDengine tag 使用 `vehicle_key=JT808:013307795425`。
|
||||
|
||||
## Redis 验证
|
||||
|
||||
向 `23296` 注入带 VIN 的 GB32960 合成帧后，Realtime API 可查询：
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LNBSCB3D4R1234567
|
||||
```
|
||||
|
||||
返回字段包含：
|
||||
|
||||
- `vin=LNBSCB3D4R1234567`
|
||||
- `protocols=["GB32960"]`
|
||||
- `online=true`
|
||||
- `speed_kmh=30`
|
||||
- `total_mileage_km=10000`
|
||||
- `soc_percent=85`
|
||||
- `longitude=121`
|
||||
- `latitude=30.56`
|
||||
|
||||
## MySQL 统计验证
|
||||
|
||||
向 `23296` 注入带 VIN 的 GB32960 合成帧后，`vehicle_daily_metric` 写入：
|
||||
|
||||
```text
|
||||
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_mileage_km | 0.000 | 10000.000 | 10000.000 | 1
|
||||
LNBSCB3D4R1234567 | 2020-07-01 | GB32960 | daily_total_mileage_km | 10000.000 | 10000.000 | 10000.000 | 1
|
||||
```
|
||||
|
||||
说明：本次 GB32960 合成帧的设备时间为测试值，因此统计日期为 `2020-07-01`。
|
||||
|
||||
## 已修复问题
|
||||
|
||||
- TDengine WebSocket 普通 `INSERT` 使用参数占位符时，字符串未按预期加引号，导致语法错误。已在 `60ca42e` 中改为显式 SQL literal 转义。
|
||||
- MySQL Go DSN 中 `charset=utf8mb4%2Cutf8` 会被 driver 传给 MySQL，导致语法错误；ECS 旁路 env 已改为 `charset=utf8mb4`。
|
||||
- TDengine ECS 当前可用 root 密码与早先记录不一致，本次旁路 env 已按实际可认证值修正。
|
||||
|
||||
## 未完成
|
||||
|
||||
- 未切换真实生产端口 `32960/808`。
|
||||
- 未将真实外部 32960/808 流量转发到 Go 旁路端口。
|
||||
- 未开启宇通 MQTT 正式订阅。
|
||||
- JT808 样例未解析到 VIN，需维护 `vehicle_identity_binding` 后再验证 808 每日统计。
|
||||
|
||||
## 2026-07-01 22:22 真实归档帧复验
|
||||
|
||||
部署版本已更新：
|
||||
|
||||
- Git commit：`5f4f4fd`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5f4f4fd-20260701222128`
|
||||
|
||||
本轮补充验证内容：
|
||||
|
||||
- JT808 支持 2019 版 versioned header，真实帧 `0x0200` 可解析出 `phone=14894135060`、`longitude=117.178187`、`latitude=30.570155`、`speed_kmh=33.4`、`total_mileage_km=7868.9`。
|
||||
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 可解析 `0x01` 整车、`0x02` 驱动电机、`0x03` 燃料电池、`0x04` 发动机、`0x05` 位置、`0x06` 极值、`0x07` 报警、`0x08` 电压、`0x09` 温度；后续 `0x30` 广东燃料电池扩展暂以 unknown raw 保留。
|
||||
- GB32960 真实帧字段包含 `fuel_cell_hydrogen_consumption_kg_per_100km=1.8`、`longitude=120.800326`、`latitude=31.634907`、`total_mileage_km=53490.9`。
|
||||
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回上述经纬度和氢耗字段。
|
||||
- TDengine `vehicle_locations` 和 `vehicle_mileage_points` 已写入真实 GB32960 位置和里程点：
|
||||
|
||||
```text
|
||||
2020-07-01 08:09:22 | GB32960 | LB9A32A21R0LS1707 | lon=120.800326 | lat=31.634907 | mileage=53490.9
|
||||
```
|
||||
|
||||
后续仍需补齐：
|
||||
|
||||
- JT808 真实 phone `14894135060` 仍未解析到 VIN，需维护身份绑定后再验证 VIN 维度实时和统计。
|
||||
|
||||
## 2026-07-01 22:28 广东燃料电池扩展复验
|
||||
|
||||
部署版本已更新：
|
||||
|
||||
- Git commit：`a15de91`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-a15de91-20260701222743`
|
||||
|
||||
本轮补充验证内容：
|
||||
|
||||
- GB32960 真实燃料电池帧 `VIN=LB9A32A21R0LS1707` 的 vendor 尾部已从 `unknown_unit` 补为结构化块：
|
||||
`0x30 gd_fc_stack`、`0x31 gd_fc_auxiliary`、`0x32 gd_fc_dcdc`、`0x33 gd_fc_air_conditioner`、
|
||||
`0x34 gd_fc_vehicle_info`、`0x80 gd_fc_demo_extension`、`0x83 gd_fc_vendor_tlv`。
|
||||
- Kafka `vehicle.raw.go.gb32960.v1` 最新消息包含上述 block，并输出关键扁平字段：
|
||||
`gd_fc_stack_hydrogen_inlet_pressure_kpa=97`、`gd_fc_stack_cell_count=108`、
|
||||
`gd_fc_vehicle_hydrogen_mass_kg=4.3`、`gd_fc_demo_stack_temp_c=65`。
|
||||
- Realtime API `GET /api/realtime/vehicles/LB9A32A21R0LS1707` 已返回 vendor 扁平字段：
|
||||
|
||||
```text
|
||||
gd_fc_dcdc_controller_temp_c=47
|
||||
gd_fc_demo_stack_temp_c=65
|
||||
gd_fc_stack_cell_count=108
|
||||
gd_fc_stack_frame_cell_count=108
|
||||
gd_fc_stack_hydrogen_inlet_pressure_kpa=97
|
||||
gd_fc_stack_water_outlet_temp_c=65
|
||||
gd_fc_vehicle_hydrogen_mass_kg=4.3
|
||||
```
|
||||
|
||||
- TDengine `raw_frames` 中 GB32960 行数增加到 `4`，最新 RAW 标识：
|
||||
|
||||
```text
|
||||
2026-07-01 14:28:24.869 | 172.20.0.1:47082 | GB32960 | LB9A32A21R0LS1707 | OK
|
||||
```
|
||||
|
||||
后续仍需补齐：
|
||||
|
||||
- JT808 真实 phone `14894135060` 到 VIN 的绑定数据维护与统计复验。
|
||||
- Go 旁路尚未切换到生产 `32960/808` 端口；仍运行在 `23296/18080`。
|
||||
|
||||
## 2026-07-01 22:43 JT808 前导零 phone 身份解析复验
|
||||
|
||||
部署版本已更新：
|
||||
|
||||
- Git commit：`6b55694`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-6b55694-20260701224300`
|
||||
|
||||
本轮修复内容：
|
||||
|
||||
- JT808 2011 老版包头的终端手机号按 BCD[6] 解析会保留前导 `0`，例如包头为 `013079963379`。
|
||||
- `vehicle_identity_binding.phone` 中实际维护的是去前导零后的手机号 `13079963379`。
|
||||
- 身份解析候选键已调整为先查原始 phone，再查去前导零 phone，之后再按 `device_id`、`plate` 降级。
|
||||
|
||||
真实帧复验样本：
|
||||
|
||||
- 归档帧：`/opt/lingniu/vehicle-ingest/gb32960/archive/2026/07/01/JT808/unknown/1782903942845000.bin`
|
||||
- 消息：`0x0200` 位置信息汇报
|
||||
- 包头 phone：`013079963379`
|
||||
- 绑定表 phone：`13079963379`
|
||||
- 解析 VIN：`LKLG7C4E3NA774736`
|
||||
|
||||
Kafka `vehicle.raw.go.jt808.v1` 最新消息确认：
|
||||
|
||||
```text
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
parsed.header.phone_trim_zero=13079963379
|
||||
parsed.identity.resolved=true
|
||||
parsed.identity.source=phone
|
||||
parsed.identity.value=13079963379
|
||||
```
|
||||
|
||||
Realtime API 确认：
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
speed_kmh=0
|
||||
total_mileage_km=0
|
||||
device_time=2026-07-01T19:05:40+08:00
|
||||
```
|
||||
|
||||
TDengine 确认：
|
||||
|
||||
```text
|
||||
raw_frames:
|
||||
2026-07-01 14:49:10.913 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | message_id=512
|
||||
|
||||
vehicle_locations:
|
||||
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | lon=119.557997 | lat=29.049092 | speed=0 | mileage=0
|
||||
|
||||
vehicle_mileage_points:
|
||||
2026-07-01 11:05:40.000 | JT808 | LKLG7C4E3NA774736 | mileage=0
|
||||
```
|
||||
|
||||
MySQL `vehicle_daily_metric` 确认：
|
||||
|
||||
```text
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 0.000 | sample_count=1 | TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
说明：该真实位置帧携带的表 27 附加项 `0x01` 总里程值为 `0`，因此本次只验证身份解析、链路落地和差值统计写入；要验证非零日里程，需要回放或等待同一 VIN 的非零总里程连续位置点。
|
||||
|
||||
## 2026-07-01 22:55 Kafka 发布重试部署复验
|
||||
|
||||
部署版本已更新：
|
||||
|
||||
- Git commit：`7e7066d`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7e7066d-20260701225503`
|
||||
|
||||
本轮修复内容：
|
||||
|
||||
- gateway 的 Kafka 发布增加同步重试封装。
|
||||
- 默认配置：
|
||||
- `KAFKA_PUBLISH_ATTEMPTS=3`
|
||||
- `KAFKA_PUBLISH_BACKOFF_MS=100`
|
||||
- `KAFKA_PUBLISH_TIMEOUT_MS=3000`
|
||||
- RAW 仍先于 unified event 写入；RAW 写失败不会继续写 unified。
|
||||
- 单元测试覆盖首次失败后成功、重试耗尽返回错误、上下文取消时停止重试。
|
||||
|
||||
部署后容器状态：
|
||||
|
||||
```text
|
||||
go-vehicle-gateway | go-7e7066d-20260701225503 | Up
|
||||
go-history-writer | go-7e7066d-20260701225503 | Up
|
||||
go-stat-writer | go-7e7066d-20260701225503 | Up
|
||||
go-realtime-api | go-7e7066d-20260701225503 | Up
|
||||
```
|
||||
|
||||
真实 808 帧回放复验：
|
||||
|
||||
```text
|
||||
source_endpoint=172.20.0.1:51966
|
||||
protocol=JT808
|
||||
message_id=0x0200
|
||||
phone=013079963379
|
||||
vin=LKLG7C4E3NA774736
|
||||
parsed.identity.value=13079963379
|
||||
```
|
||||
|
||||
Kafka `vehicle.raw.go.jt808.v1` 确认新消息写入，partition 6 offset 从 `1` 增加到 `2`。最新消息包含：
|
||||
|
||||
```text
|
||||
received_at_ms=1782917751903
|
||||
source_endpoint=172.20.0.1:51966
|
||||
vin=LKLG7C4E3NA774736
|
||||
fields.longitude=119.557997
|
||||
fields.latitude=29.049092
|
||||
fields.total_mileage_km=0
|
||||
```
|
||||
|
||||
TDengine `raw_frames` 确认新 RAW 落地：
|
||||
|
||||
```text
|
||||
2026-07-01 14:55:51.903 | JT808 | LKLG7C4E3NA774736 | phone=013079963379 | source=172.20.0.1:51966
|
||||
```
|
||||
|
||||
Realtime API 确认 Redis 已刷新：
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
|
||||
received_at_ms=1782917751903
|
||||
source_endpoint=172.20.0.1:51966
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=0
|
||||
```
|
||||
|
||||
说明：当前重试层只能覆盖 Kafka 短暂抖动；如果 Kafka 长时间不可用，仍需后续增加本地磁盘 spool/WAL 和恢复补发。
|
||||
|
||||
## 2026-07-01 23:02 切换 Go 到生产 32960/808 端口
|
||||
|
||||
本轮操作：
|
||||
|
||||
- 停止 ECS 上原有 Java Docker 容器：
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
- `vehicle-history-app`
|
||||
- `vehicle-analytics-app`
|
||||
- `vehicle-state-app`
|
||||
- `telemetry-field-parser-app`
|
||||
- 保留 Go sidecar 和 Portainer agent。
|
||||
- 修改 `/opt/lingniu-go/.env`：
|
||||
- `GO_GB32960_TCP_PORT=32960`
|
||||
- `GO_JT808_TCP_PORT=808`
|
||||
- Go gateway 使用镜像：
|
||||
- `crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-243de5c-20260701230028`
|
||||
- 开启 Kafka 本地 spool：
|
||||
- host：`/opt/lingniu-go/spool/gateway`
|
||||
- container：`/data/spool/gateway`
|
||||
|
||||
端口验证：
|
||||
|
||||
```text
|
||||
0.0.0.0:32960 -> go-vehicle-gateway:32960
|
||||
0.0.0.0:808 -> go-vehicle-gateway:808
|
||||
18080/23296 -> 已释放
|
||||
```
|
||||
|
||||
启动日志确认：
|
||||
|
||||
```text
|
||||
kafka durable spool enabled dir=/data/spool/gateway replay_interval_ms=1000
|
||||
tcp listener started protocol=GB32960 addr=[::]:32960
|
||||
tcp listener started protocol=JT808 addr=[::]:808
|
||||
```
|
||||
|
||||
真实连接确认：
|
||||
|
||||
```text
|
||||
GB32960 external remote: 8.134.95.166:50084
|
||||
JT808 external remote: 115.231.168.135:13801
|
||||
```
|
||||
|
||||
生产 `808` 回放复验：
|
||||
|
||||
```text
|
||||
source_endpoint=172.20.0.1:54680
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=0
|
||||
```
|
||||
|
||||
TDengine 最新 RAW 复验显示生产 808 已有真实外部数据持续落地：
|
||||
|
||||
```text
|
||||
2026-07-01 15:03:02.960 | JT808 | LKLG7C4E1NA774802 | 013079963291 | 222.66.200.68:28394
|
||||
2026-07-01 15:03:02.114 | JT808 | | 14894135583 | 122.152.221.156:33871
|
||||
2026-07-01 15:03:01.352 | JT808 | | 013307795519 | 115.231.168.135:13801
|
||||
2026-07-01 15:03:01.330 | JT808 | LKLG7C4E3NA774736 | 013079963379 | 222.66.200.68:28393
|
||||
```
|
||||
|
||||
Kafka spool 目录确认：
|
||||
|
||||
```text
|
||||
/opt/lingniu-go/spool/gateway file count = 0
|
||||
```
|
||||
|
||||
说明：生产 808 中仍有部分 phone 未解析出 VIN，后续需要继续维护 `vehicle_identity_binding` 的 phone/device_id/plate 到 VIN 映射，或者增强 808 注册/鉴权帧中的 identity 回写。
|
||||
|
||||
## 2026-07-01 23:09 808 里程统计防 0 污染复验
|
||||
|
||||
部署版本已更新：
|
||||
|
||||
- Git commit：`d347525`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-d347525-20260701230845`
|
||||
|
||||
本轮修复内容：
|
||||
|
||||
- `stat-writer` 不再把 `total_mileage_km <= 0` 的采样写入 MySQL 每日指标。
|
||||
- MySQL upsert 对历史 `first_total_mileage_km=0` / `latest_total_mileage_km=0` 做纠偏；后续首次有效总里程会替换掉历史 0。
|
||||
- `realtime-api` 合并实时快照时，非正 `total_mileage_km` 不再覆盖已有正值；速度、位置、状态等其他字段仍按事件时间更新。
|
||||
|
||||
测试验证：
|
||||
|
||||
```text
|
||||
go test ./...
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
生产回放验证：
|
||||
|
||||
- 第一条 808 帧：`device_time=2026-07-01T23:09:35+08:00`，`total_mileage_km=12345.6`
|
||||
- 第二条 808 帧：`device_time=2026-07-01T23:09:36+08:00`，`total_mileage_km=0`
|
||||
- 两条帧使用同一 VIN：`LKLG7C4E3NA774736`
|
||||
|
||||
Redis merged 快照确认：第二条 0 里程帧没有覆盖已有有效总里程。
|
||||
|
||||
```text
|
||||
event_time_ms=1782918576000
|
||||
source_endpoint=172.20.0.1:40946
|
||||
speed_kmh=0
|
||||
longitude=119.557997
|
||||
latitude=29.049092
|
||||
total_mileage_km=12345.6
|
||||
field_times_ms.total_mileage_km=1782918575000
|
||||
```
|
||||
|
||||
MySQL `vehicle_daily_metric` 确认：0 里程采样未进入统计，sample_count 只因有效总里程采样增加一次。
|
||||
|
||||
```text
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | 0.000 | sample_count=13 | first=12345.600 | latest=12345.600
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | 12345.600 | sample_count=13 | first=12345.600 | latest=12345.600
|
||||
```
|
||||
|
||||
说明：生产 TDengine 中已经存在大量 808 非零 `total_mileage_km` 采样，但许多 phone 尚未映射到 VIN，因此 MySQL 按 VIN 的统计只会覆盖已解析 VIN 的车辆。下一步应继续补齐 `vehicle_identity_binding`，让更多 808 车辆进入统计。
|
||||
|
||||
## 2026-07-01 23:23 808 注册身份自动维护复验
|
||||
|
||||
部署版本已更新：
|
||||
|
||||
- Git commit：`7b44f97`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-7b44f97-20260701232200`
|
||||
|
||||
本轮修复内容：
|
||||
|
||||
- 808 `0x0100` 注册帧解析 `province`、`city`、`manufacturer`、`device_type`、`device_id`、`plate_color`、`plate`。
|
||||
- 808 `0x0102` 鉴权帧解析 `auth_token`。
|
||||
- Go gateway 在 identity resolve 后写入 `jt808_registration`，普通上行也会刷新 `latest_seen_at`。
|
||||
- 注册帧通过车牌解析到 VIN 后，会尝试把 phone/device/plate 反写到 `vehicle_identity_binding`。
|
||||
- 注册帧车牌支持 GBK 解码，避免 `Incorrect string value` 写入错误。
|
||||
|
||||
验证命令：
|
||||
|
||||
```text
|
||||
go test ./...
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
ECS 运行状态：
|
||||
|
||||
```text
|
||||
go-vehicle-gateway | go-7b44f97-20260701232200 | 0.0.0.0:808->808, 0.0.0.0:32960->32960
|
||||
go-history-writer | go-7b44f97-20260701232200 | Up
|
||||
go-stat-writer | go-7b44f97-20260701232200 | Up
|
||||
go-realtime-api | go-7b44f97-20260701232200 | 0.0.0.0:20210->20210
|
||||
```
|
||||
|
||||
生产日志确认：
|
||||
|
||||
- 新版本启动后真实 808、32960 连接持续进入。
|
||||
- GBK 车牌注册帧不再出现 `Incorrect string value`。
|
||||
|
||||
MySQL `jt808_registration` 确认：
|
||||
|
||||
```text
|
||||
registration_rows=214
|
||||
distinct_phone=214
|
||||
vin_rows=74
|
||||
plate_rows=74
|
||||
device_rows=72
|
||||
|
||||
013079963379 | device_id=9963379 | plate=沪A06788F | vin=LKLG7C4E3NA774736 | latest_registered_at=2026-07-01 23:23:01
|
||||
013079963321 | device_id=9963321 | plate=沪A59613F | vin=LKLG7C4EXNA774796 | latest_registered_at=2026-07-01 23:23:04
|
||||
```
|
||||
|
||||
说明：`jt808_registration.phone` 保存 808 包头原始 phone；`vehicle_identity_binding.phone` 中已有部分去前导零 phone。Go resolver 会同时查询原始 phone 和去前导零 phone。
|
||||
|
||||
## 2026-07-01 23:28 生产链路与指标清理复验
|
||||
|
||||
本轮复验目标：
|
||||
|
||||
- 确认 Go gateway 的 Kafka durable spool 是重启窗口残留，而不是持续发布失败。
|
||||
- 确认 stat-writer / history-writer / realtime-api 的 Kafka consumer group 无 lag。
|
||||
- 确认真实 808 车辆 registration、TDengine RAW/里程点、Redis 实时查询均在持续更新。
|
||||
- 清理之前生产合成回放产生的 JT808 当日指标测试数据。
|
||||
|
||||
Kafka topic / consumer group：
|
||||
|
||||
```text
|
||||
vehicle.raw.go.jt808.v1 有持续 offset
|
||||
vehicle.raw.go.gb32960.v1 有持续 offset
|
||||
vehicle.event.go.unified.v1 有持续 offset
|
||||
|
||||
go-stat-writer LAG=0
|
||||
go-history-writer LAG=0
|
||||
go-realtime-api LAG=0
|
||||
```
|
||||
|
||||
TDengine：
|
||||
|
||||
```text
|
||||
raw_frames:
|
||||
JT808 2922
|
||||
GB32960 27
|
||||
|
||||
vehicle_mileage_points:
|
||||
JT808 2050
|
||||
GB32960 2
|
||||
```
|
||||
|
||||
Redis 实时查询样例：
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736
|
||||
vin=LKLG7C4E3NA774736
|
||||
source_endpoint=222.66.200.68:29646
|
||||
plate=沪A06788F
|
||||
longitude=119.556866
|
||||
latitude=29.047911
|
||||
total_mileage_km=12345.6
|
||||
```
|
||||
|
||||
说明：该 Redis merged 快照保留了此前有效 `total_mileage_km=12345.6`，后续真实 808 位置帧上报的 `total_mileage_km=0` 不会覆盖该字段。TDengine 中该 VIN 后续真实里程点的 `total_mileage_km` 均为 0，因此 stat-writer 按“只统计正总里程”的规则不再写 JT808 每日指标。
|
||||
|
||||
MySQL 指标清理：
|
||||
|
||||
```text
|
||||
删除前：
|
||||
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_mileage_km | first=0 | latest=0 | sample_count=7
|
||||
LKLG7C4E2NA774775 | JT808 | 2026-07-01 | daily_total_mileage_km | first=0 | latest=0 | sample_count=7
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
|
||||
LKLG7C4E3NA774736 | JT808 | 2026-07-01 | daily_total_mileage_km | first=12345.6 | latest=12345.6 | sample_count=13
|
||||
|
||||
删除后等待真实流量窗口：
|
||||
JT808 2026-07-01 metric rows = 0
|
||||
```
|
||||
|
||||
清理原因：
|
||||
|
||||
- `LKLG7C4E3NA774736` 的 12345.6 来自此前生产连通性合成回放，不是真实平台上报。
|
||||
- `LKLG7C4E2NA774775` 的 0 指标来自旧逻辑污染，当前 stat-writer 已跳过非正总里程。
|
||||
|
||||
Kafka spool：
|
||||
|
||||
```text
|
||||
23:24 spool_count=904
|
||||
23:25 spool_count=862
|
||||
23:28 spool_count=722
|
||||
new_spool_1min=0
|
||||
```
|
||||
|
||||
说明：spool 正在回放下降，且最近 1 分钟没有新增文件；生产 gateway 日志未见 `Incorrect string value`、Kafka publish error 或 replay error。
|
||||
|
||||
## 2026-07-01 23:34 每日指标查询 API 复验
|
||||
|
||||
部署版本已更新：
|
||||
|
||||
- Git commit：`56f4811`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-56f4811-20260701233311`
|
||||
|
||||
新增接口：
|
||||
|
||||
```text
|
||||
GET /api/stats/daily-metrics
|
||||
```
|
||||
|
||||
查询参数：
|
||||
|
||||
```text
|
||||
vin 可选
|
||||
protocol 可选，GB32960 / JT808 / YUTONG_MQTT
|
||||
metricKey 可选，daily_mileage_km / daily_total_mileage_km
|
||||
dateFrom 可选，YYYY-MM-DD
|
||||
dateTo 可选，YYYY-MM-DD
|
||||
limit 可选，默认 50，最大 1000
|
||||
offset 可选，默认 0
|
||||
```
|
||||
|
||||
生产可访问样例：
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2020-07-01&limit=10
|
||||
```
|
||||
|
||||
返回确认：
|
||||
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"vin": "LB9A32A21R0LS1707",
|
||||
"stat_date": "2020-07-01",
|
||||
"protocol": "GB32960",
|
||||
"metric_key": "daily_mileage_km",
|
||||
"metric_value": 0,
|
||||
"metric_unit": "km",
|
||||
"first_total_mileage_km": 53490.9,
|
||||
"latest_total_mileage_km": 53490.9,
|
||||
"sample_count": 3,
|
||||
"calculation_method": "TOTAL_MILEAGE_DIFF",
|
||||
"created_at": "2026-07-01 22:14:13",
|
||||
"updated_at": "2026-07-01 22:28:25"
|
||||
},
|
||||
{
|
||||
"vin": "LB9A32A21R0LS1707",
|
||||
"stat_date": "2020-07-01",
|
||||
"protocol": "GB32960",
|
||||
"metric_key": "daily_total_mileage_km",
|
||||
"metric_value": 53490.9,
|
||||
"metric_unit": "km",
|
||||
"first_total_mileage_km": 53490.9,
|
||||
"latest_total_mileage_km": 53490.9,
|
||||
"sample_count": 3,
|
||||
"calculation_method": "TOTAL_MILEAGE_DIFF",
|
||||
"created_at": "2026-07-01 22:14:13",
|
||||
"updated_at": "2026-07-01 22:28:25"
|
||||
}
|
||||
],
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
"total": 2
|
||||
}
|
||||
```
|
||||
|
||||
JT808 真实生产查询：
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/stats/daily-metrics?protocol=JT808&dateFrom=2026-07-01&dateTo=2026-07-01&limit=10
|
||||
```
|
||||
|
||||
返回 `total=0`。原因是此前合成回放产生的测试指标已清理，当前真实 808 已绑定 VIN 的车辆仍上报 `total_mileage_km=0`，stat-writer 按规则不写非正总里程指标。
|
||||
|
||||
实时接口回归：
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
```
|
||||
|
||||
返回 `online=true`，说明新增 stats API 没有影响 Redis 实时查询。
|
||||
|
||||
发布后 spool 观察：
|
||||
|
||||
```text
|
||||
发布后立即：spool_count=1101，new_spool_1min=102
|
||||
等待 60 秒：spool_count=1033，new_spool_1min=0
|
||||
```
|
||||
|
||||
说明：本次发布重启窗口产生的 spool 正在回放下降，且最近 1 分钟没有新增；gateway/realtime-api 最近日志未见 error、failed、Kafka publish error。
|
||||
|
||||
## 2026-07-01 23:50 生产端口接管与 RAW 查询 API 复验
|
||||
|
||||
部署版本：
|
||||
|
||||
- Git commit：`5937132`
|
||||
- 镜像：`crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-gateway-go:go-5937132-20260701234220`
|
||||
|
||||
生产端口状态：
|
||||
|
||||
```text
|
||||
go-vehicle-gateway 0.0.0.0:808->808/tcp
|
||||
go-vehicle-gateway 0.0.0.0:32960->32960/tcp
|
||||
```
|
||||
|
||||
旧 Java 容器状态：
|
||||
|
||||
```text
|
||||
gb32960-ingest-app restart=no status=exited
|
||||
jt808-ingest-app restart=no status=exited
|
||||
yutong-mqtt-app restart=no status=exited
|
||||
vehicle-history-app restart=no status=exited
|
||||
vehicle-analytics-app restart=no status=exited
|
||||
vehicle-state-app restart=no status=exited
|
||||
telemetry-field-parser-app restart=no status=exited
|
||||
```
|
||||
|
||||
生产连接确认：
|
||||
|
||||
- JT808 端口 `808` 已收到外部连接：`222.66.200.68`、`115.231.168.135`。
|
||||
- GB32960 端口 `32960` 已收到外部连接：`8.134.95.166`、`117.160.0.65`。
|
||||
- ECS 当前没有 `8089` 监听。
|
||||
|
||||
RAW 查询 API：
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames
|
||||
```
|
||||
|
||||
生产可访问样例：
|
||||
|
||||
```text
|
||||
http://115.29.187.205:20210/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
|
||||
http://115.29.187.205:20210/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
|
||||
```
|
||||
|
||||
接口复验结果：
|
||||
|
||||
```text
|
||||
JT808_RAW:
|
||||
total=1
|
||||
protocol=JT808
|
||||
vin=LKLG7C4E3NA774736
|
||||
phone=013079963379
|
||||
message_id=512
|
||||
message_id_hex=0x0200
|
||||
parse_status=OK
|
||||
raw_hex_len=94
|
||||
parsed_json_len=651
|
||||
fields_json_len=186
|
||||
|
||||
GB32960_RAW:
|
||||
total=1
|
||||
protocol=GB32960
|
||||
vin=LB9A32A21R0LS1707
|
||||
message_id=2
|
||||
message_id_hex=0x0002
|
||||
parse_status=OK
|
||||
raw_hex_len=1548
|
||||
parsed_json_len=3698
|
||||
fields_json_len=563
|
||||
```
|
||||
|
||||
同轮回归验证：
|
||||
|
||||
```text
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
online=true
|
||||
protocols=["JT808"]
|
||||
|
||||
GET /api/stats/daily-metrics?vin=LB9A32A21R0LS1707&protocol=GB32960&dateFrom=2020-07-01&dateTo=2026-07-01&limit=1
|
||||
total=1
|
||||
protocol=GB32960
|
||||
metric_key=daily_mileage_km
|
||||
calculation_method=TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
说明：
|
||||
|
||||
- RAW 查询已返回 `raw_hex`、`parsed_json`、`fields_json`，可以用于查看完整解析结果。
|
||||
- 日统计和实时查询在生产端口接管后仍可用。
|
||||
|
||||
## 2026-07-02 00:00 原生 systemd 部署与宇通 MQTT 复验
|
||||
|
||||
根据本次 goal 的新要求，Go 接入链路后续不再使用 Docker 部署。本轮已中断 Docker 镜像构建路径，改为 ECS 原生 Linux 二进制 + systemd。
|
||||
|
||||
部署版本：
|
||||
|
||||
- Git commit：`594ab2d`
|
||||
- 部署目录：`/opt/lingniu-go-native`
|
||||
- 当前 release：`/opt/lingniu-go-native/releases/594ab2d`
|
||||
- 当前指针：`/opt/lingniu-go-native/current -> /opt/lingniu-go-native/releases/594ab2d`
|
||||
|
||||
systemd 服务：
|
||||
|
||||
```text
|
||||
lingniu-go-gateway.service active
|
||||
lingniu-go-history-writer.service active
|
||||
lingniu-go-stat-writer.service active
|
||||
lingniu-go-realtime-api.service active
|
||||
```
|
||||
|
||||
端口确认：
|
||||
|
||||
```text
|
||||
[::]:32960 gateway
|
||||
[::]:808 gateway
|
||||
[::]:20210 realtime-api
|
||||
```
|
||||
|
||||
Go Docker 容器已停止：
|
||||
|
||||
```text
|
||||
go-vehicle-gateway Exited
|
||||
go-history-writer Exited
|
||||
go-stat-writer Exited
|
||||
go-realtime-api Exited
|
||||
```
|
||||
|
||||
宇通 MQTT：
|
||||
|
||||
```text
|
||||
yutong mqtt client started
|
||||
mqtt subscribed broker=ssl://cpxlm.axxc.cn:38883 topic=/ytforward/shln/+ qos=1
|
||||
```
|
||||
|
||||
说明：MQTT broker 存在短连接 `EOF` 后自动重连现象，但订阅后已收到真实数据并落 RAW。
|
||||
|
||||
RAW 查询复验：
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=3
|
||||
|
||||
total=3
|
||||
protocol=YUTONG_MQTT
|
||||
vin=LMRKH9AC1R1004131
|
||||
vehicle_key=LMRKH9AC1R1004131
|
||||
parse_status=OK
|
||||
parsed_json_len=1104
|
||||
fields_json_len=116
|
||||
```
|
||||
|
||||
同轮回归：
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=JT808&vin=LKLG7C4E3NA774736&messageId=0x0200&limit=1
|
||||
total=1
|
||||
parse_status=OK
|
||||
|
||||
GET /api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=1
|
||||
total=1
|
||||
parse_status=OK
|
||||
|
||||
GET /api/realtime/vehicles/LKLG7C4E3NA774736/online
|
||||
online=true
|
||||
protocols=["JT808"]
|
||||
```
|
||||
|
||||
本轮落地文件：
|
||||
|
||||
- `deploy/systemd/README.md`
|
||||
- `deploy/systemd/lingniu-go-gateway.service`
|
||||
- `deploy/systemd/lingniu-go-history-writer.service`
|
||||
- `deploy/systemd/lingniu-go-stat-writer.service`
|
||||
- `deploy/systemd/lingniu-go-realtime-api.service`
|
||||
|
||||
## 2026-07-02 00:04 宇通 MQTT EOF 诊断
|
||||
|
||||
现象：
|
||||
|
||||
Go 原生部署后，`lingniu-go-gateway` 日志中出现多次：
|
||||
|
||||
```text
|
||||
mqtt connection lost error=EOF
|
||||
mqtt connection lost error=write: broken pipe
|
||||
mqtt subscribed topic=/ytforward/shln/+ qos=1
|
||||
```
|
||||
|
||||
10 分钟窗口统计：
|
||||
|
||||
```text
|
||||
mqtt subscribed = 33
|
||||
mqtt connection lost = 32
|
||||
```
|
||||
|
||||
对照旧 Java `yutong-mqtt-app` 历史日志，停止前也存在同类行为：
|
||||
|
||||
```text
|
||||
mqtt endpoint [yutong] received topic=/ytforward/shln/1 bytes=574
|
||||
mqtt endpoint [yutong] connection lost
|
||||
Caused by: java.io.EOFException
|
||||
mqtt endpoint [yutong] re-subscribed topic=/ytforward/shln/+ qos=1
|
||||
mqtt endpoint [yutong] received topic=/ytforward/shln/3 bytes=586
|
||||
```
|
||||
|
||||
判断：
|
||||
|
||||
- EOF/短连接不是 Go TLS 支持或 systemd 原生部署新引入的问题。
|
||||
- 旧 Java 和新 Go 都表现为：订阅、收到数据、broker 断开、自动重连。
|
||||
- 当前链路可持续收到宇通 MQTT 数据并写入 TDengine RAW。
|
||||
|
||||
复验：
|
||||
|
||||
```text
|
||||
GET /api/history/raw-frames?protocol=YUTONG_MQTT&limit=1
|
||||
|
||||
total=1
|
||||
protocol=YUTONG_MQTT
|
||||
vin=LMRKH9ACXR1004094
|
||||
parse_status=OK
|
||||
parsed_json_len=1080
|
||||
fields_json_len=115
|
||||
```
|
||||
|
||||
运维建议：
|
||||
|
||||
- 单条 EOF 不作为不可用判断。
|
||||
- 以 `YUTONG_MQTT` RAW 入库增长、最近成功 `mqtt subscribed` 时间、服务 `systemctl is-active` 作为可用性判断。
|
||||
- 如果 EOF 频率继续升高且 RAW 不再增长，再联系宇通侧确认 broker 长连接策略、同一 clientId 并发限制或网络出口策略。
|
||||
@@ -1,158 +0,0 @@
|
||||
# Go Vehicle Gateway 验证手册
|
||||
|
||||
本文档用于验证 Go 重构运行面：
|
||||
|
||||
- `go-vehicle-gateway`：GB32960 TCP、JT808 TCP、宇通 MQTT 接入。
|
||||
- `go-history-writer`：Kafka RAW 写 TDengine。
|
||||
- `go-stat-writer`：Kafka RAW 写 MySQL 每日指标。
|
||||
- `go-realtime-api`：Kafka unified event 写 Redis，并提供实时查询 API。
|
||||
|
||||
## 本地构建验证
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./...
|
||||
go build ./cmd/gateway
|
||||
go build ./cmd/history-writer
|
||||
go build ./cmd/stat-writer
|
||||
go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
## 生产原生部署验证
|
||||
|
||||
```bash
|
||||
python3 tools/go_prod_acceptance.py --date 2026-07-02
|
||||
```
|
||||
|
||||
该命令会聚合执行：
|
||||
|
||||
- `go_systemd_prod_smoke.py`：应用 ECS systemd 服务 active/enabled、端口归属和 gateway spool
|
||||
- `go_kafka_prod_smoke.py`：Kafka topic 和消费组 lag
|
||||
- `go_native_prod_smoke.py`：HTTP API、RAW `parsed_json`、历史核心表 `frame_id` 回链、每日指标公式、Redis 实时新鲜度、TDengine/MySQL/Redis 数据链路
|
||||
|
||||
如需单独检查 systemd：
|
||||
|
||||
```bash
|
||||
python3 tools/go_systemd_prod_smoke.py --host 115.29.187.205 --user root
|
||||
```
|
||||
|
||||
脚本通过 SSH 检查：
|
||||
|
||||
- `lingniu-go-gateway.service`
|
||||
- `lingniu-go-history-writer.service`
|
||||
- `lingniu-go-stat-writer.service`
|
||||
- `lingniu-go-realtime-api.service`
|
||||
- 上述服务均为 `active` 且 `enabled`
|
||||
- `808`、`32960` 由 `gateway` 监听
|
||||
- `20210` 由 `realtime-api` 监听
|
||||
- `/opt/lingniu-go-native/spool/gateway` 无待回放文件
|
||||
|
||||
## 生产环境变量
|
||||
|
||||
当前 goal 的生产面使用 ECS 原生 systemd 部署，不再以 Docker/Portainer 作为生产运行方式。以下环境变量维护在 `/opt/lingniu-go-native/env/*.env`：
|
||||
|
||||
```bash
|
||||
KAFKA_BROKERS=172.17.111.56:9092
|
||||
TDENGINE_DSN=root:<password>@ws(172.17.111.57:6041)/lingniu_vehicle_ts
|
||||
MYSQL_DSN=lingniu_vehicle:<password>@tcp(rm-bp179zbv481rnw3e2.mysql.rds.aliyuncs.com:3306)/lingniu_vehicle_data?parseTime=true&charset=utf8mb4,utf8&loc=Asia%2FShanghai
|
||||
IDENTITY_MYSQL_DSN=${MYSQL_DSN}
|
||||
VEHICLE_IDENTITY_TABLE=vehicle_identity_binding
|
||||
REDIS_ADDR=r-bp1u741kij7e51i481.redis.rds.aliyuncs.com:6379
|
||||
REDIS_PASSWORD=<password>
|
||||
REDIS_DB=50
|
||||
```
|
||||
|
||||
宇通 MQTT 开启时再配置：
|
||||
|
||||
```bash
|
||||
YUTONG_MQTT_ENABLED=true
|
||||
YUTONG_MQTT_URI=<mqtt-uri>
|
||||
YUTONG_MQTT_TOPICS=/ytforward/shln/+
|
||||
YUTONG_MQTT_CLIENT_ID=lingniu-go-yutong-mqtt
|
||||
YUTONG_MQTT_USERNAME=<username>
|
||||
YUTONG_MQTT_PASSWORD=<password>
|
||||
```
|
||||
|
||||
## ECS 进程检查
|
||||
|
||||
```bash
|
||||
systemctl is-active lingniu-go-gateway lingniu-go-history-writer lingniu-go-stat-writer lingniu-go-realtime-api
|
||||
|
||||
ss -lntp | egrep ':(808|32960|20210)\b'
|
||||
```
|
||||
|
||||
## Kafka 验证
|
||||
|
||||
优先使用 smoke 脚本做可重复检查：
|
||||
|
||||
```bash
|
||||
python3 tools/go_kafka_prod_smoke.py --host 114.55.58.251 --user root --max-lag 100
|
||||
```
|
||||
|
||||
脚本会检查 `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1`、`vehicle.event.go.unified.v1` 是否存在，并汇总 `go-history-writer`、`go-stat-writer`、`go-realtime-api` 的消费 lag。运行环境需要 SSH 免密或已建立可用的 SSH 认证方式。
|
||||
|
||||
手工核对命令：
|
||||
|
||||
```bash
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.jt808.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.gb32960.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.raw.go.yutong-mqtt.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-console-consumer --bootstrap-server 172.17.111.56:9092 \
|
||||
--topic vehicle.event.go.unified.v1 --max-messages 1 --timeout-ms 10000
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-history-writer
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-stat-writer
|
||||
|
||||
kafka-consumer-groups --bootstrap-server 172.17.111.56:9092 \
|
||||
--describe --group go-realtime-api
|
||||
```
|
||||
|
||||
## TDengine 验证
|
||||
|
||||
```sql
|
||||
USE lingniu_vehicle_ts;
|
||||
SHOW STABLES;
|
||||
SELECT COUNT(*) FROM raw_frames;
|
||||
SELECT COUNT(*) FROM vehicle_locations;
|
||||
SELECT COUNT(*) FROM vehicle_mileage_points;
|
||||
```
|
||||
|
||||
## MySQL 验证
|
||||
|
||||
```sql
|
||||
SELECT protocol, metric_key, COUNT(*)
|
||||
FROM vehicle_daily_metric
|
||||
GROUP BY protocol, metric_key;
|
||||
|
||||
SELECT *
|
||||
FROM vehicle_daily_metric
|
||||
WHERE metric_key IN ('daily_mileage_km', 'daily_total_mileage_km')
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
## Redis 实时查询
|
||||
|
||||
```bash
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>'
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/online'
|
||||
curl -sS 'http://115.29.187.205:20210/api/realtime/vehicles/<vin>/protocols/JT808'
|
||||
```
|
||||
|
||||
## 验收证据
|
||||
|
||||
正式切换前至少记录：
|
||||
|
||||
- 一个真实 GB32960 VIN 的 RAW、location、mileage point、Redis snapshot。
|
||||
- 一个真实 JT808 VIN/phone 的 RAW、location、mileage point、daily metric、Redis snapshot。
|
||||
- 一个真实宇通 MQTT VIN 或 device_id 的 RAW、Redis snapshot。
|
||||
- `go-history-writer` 和 `go-stat-writer` 重启后 Kafka offset 能继续推进。
|
||||
@@ -1,34 +0,0 @@
|
||||
# JT808 Daily Mileage Streaming
|
||||
|
||||
## Scope
|
||||
|
||||
The analytics app can calculate daily mileage from JT808 telemetry only. It consumes `vehicle.event.jt808.v1` and saves the current daily result into the common `vehicle-stat` metric repository.
|
||||
|
||||
Only supported message backbone: Kafka.
|
||||
|
||||
## Storage
|
||||
|
||||
Runtime state: none outside `vehicle_stat_metric`. There is no separate JT808 daily-mileage table. The derived value is stored as one `daily_mileage_km` metric row in the common JDBC/MySQL table `vehicle_stat_metric`; the local-day minimum and maximum GPS total mileage values are kept on that same row as calculation source columns.
|
||||
|
||||
The JT808 daily-mileage value is calculated from the GPS total mileage reported in location additional information:
|
||||
|
||||
```text
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
calculation_method = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
The first valid JT808 location point for a vehicle and local day stores both calculation endpoints on the metric row and writes `daily_mileage_km=0.0`. Later ordered or replayed points update the lower endpoint when the reported GPS total mileage is smaller, update the upper endpoint when it is larger, and rewrite the derived `daily_mileage_km` on the same row.
|
||||
|
||||
## Runtime Settings
|
||||
|
||||
Set these in Portainer or Nacos, without committing secrets:
|
||||
|
||||
```text
|
||||
KAFKA_TOPIC_JT808_EVENT=vehicle.event.jt808.v1
|
||||
VEHICLE_STAT_JT808_MILEAGE_ENABLED=true
|
||||
MYSQL_JDBC_URL=<jdbc-url>
|
||||
MYSQL_USERNAME=<user>
|
||||
MYSQL_PASSWORD=<password>
|
||||
```
|
||||
|
||||
Algorithm defaults use the Asia/Shanghai daily boundary. Restart recovery reads the same `daily_mileage_km` metric row.
|
||||
@@ -1,351 +0,0 @@
|
||||
# Vehicle Ingest TDengine 链路验收手册
|
||||
|
||||
这份手册用于验收当前 GB32960/JT808/Yutong MQTT 接入重构链路：
|
||||
|
||||
1. 协议 App 接收 TCP 报文。
|
||||
2. 协议 App 归档原始帧元数据，并把事件信封发布到 Kafka。
|
||||
3. `vehicle-history-app` 消费 Kafka。
|
||||
4. history 写入 raw、location 到 TDengine；不写 telemetry_fields。
|
||||
5. Swagger/API 通过 TDengine 做历史分页查询。
|
||||
|
||||
每一步都必须执行命令并检查输出后，才能标记为已验证。
|
||||
|
||||
## 必需运行参数
|
||||
|
||||
启动服务前先设置：
|
||||
|
||||
```bash
|
||||
export KAFKA_BROKERS=114.55.58.251:9092
|
||||
export TDENGINE_HISTORY_ENABLED=true
|
||||
export TDENGINE_HISTORY_DATABASE=vehicle_ts
|
||||
export TDENGINE_JDBC_URL='jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts'
|
||||
export TDENGINE_DRIVER_CLASS_NAME=com.taosdata.jdbc.ws.WebSocketDriver
|
||||
export TDENGINE_USERNAME=root
|
||||
export TDENGINE_PASSWORD='<tdengine-password>'
|
||||
export TDENGINE_MIN_IDLE=0
|
||||
export TDENGINE_MAX_POOL_SIZE=32
|
||||
```
|
||||
|
||||
`TDENGINE_MIN_IDLE=0` 用于减少冷启动时 TDengine 连接重试日志。TDengine 地址稳定后，再按生产并发调大连接池。
|
||||
|
||||
高吞吐生产验收只验证 TDengine `raw_frames`、位置表和分页查询闭环。RAW 帧的 `parsedJson`、`metadataJson`、`rawUri` 直接进入 TDengine，GB32960 snapshot/fields 接口基于 `raw_frames` 与当前解析器即时返回结构化结果；接入服务写出的原始 `.bin` 只作为冷备复核材料。逐字段趋势宽表由独立字段解析服务消费 Kafka RAW/事件后维护，不由 history-app 写入。
|
||||
|
||||
## 构建
|
||||
|
||||
```bash
|
||||
mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history-app,:vehicle-analytics-app -am package -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
预期产物：
|
||||
|
||||
- `modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar`
|
||||
- `modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar`
|
||||
- `modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar`
|
||||
- `modules/apps/vehicle-history-app/target/vehicle-history-app.jar`
|
||||
- `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar`
|
||||
|
||||
## ECS 服务状态验证
|
||||
|
||||
生产服务只在 ECS 上运行，通过 Portainer/Docker 管理。不要在本机用 `java -jar`、plist 或其它守护进程方式启动 GB32960、JT808、Yutong MQTT、history、analytics。
|
||||
|
||||
```bash
|
||||
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' \
|
||||
| egrep 'gb32960|jt808|yutong|vehicle-history|vehicle-analytics|NAMES'
|
||||
|
||||
ss -lntp | egrep ':(808|32960|20100|20200|20310|20400|20500)\b'
|
||||
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/v3/api-docs \
|
||||
| grep -E '/api/event-history/locations|/api/event-history/raw-frames'
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
预期：五个容器均为运行状态；健康检查为 `UP`；JT808/GB32960 TCP 端口在 ECS 上监听；OpenAPI 中包含通用位置分页查询和 RAW 帧查询接口。history-app 不暴露 `/api/event-history/telemetry/fields`，也不持续写入逐字段宽表。JT808 位置帧中有 GPS 总里程时，`vehicle-analytics-app` 按差值法写入 MySQL `vehicle_stat_metric` 的 `daily_mileage_km` 指标。
|
||||
|
||||
## 运行位置约束
|
||||
|
||||
生产身份绑定固定使用 MySQL，不再提供 file/memory/sqlite 运行时切换入口。生产服务固定运行在 ECS/Portainer，不提供本机守护进程模板。
|
||||
|
||||
- `jt808-ingest-app` 监听 TCP `808`，HTTP `20400`。
|
||||
- `gb32960-ingest-app` 监听 TCP `32960`，HTTP `20100`。
|
||||
- `yutong-mqtt-app` 监听 HTTP `20500`。
|
||||
- `vehicle-history-app` 监听 HTTP `20200`。
|
||||
- `vehicle-analytics-app` 监听 HTTP `20310`。
|
||||
- history 热查询只依赖 Kafka 和 TDengine `raw_frames`；history 不需要共享 archive volume。
|
||||
- GB32960/JT808/Yutong MQTT 接入服务可以继续使用 `SINK_ARCHIVE_PATH` 保存原始冷备，但这不是 history API 的实时查询前置条件。
|
||||
|
||||
JT808 注册身份绑定：
|
||||
|
||||
- 生产需要把 0x0100 注册信息维护到 MySQL 时，只需要配置 `VEHICLE_IDENTITY_MYSQL_*` 连接参数。
|
||||
- 需要提供 `VEHICLE_IDENTITY_MYSQL_JDBC_URL`、`VEHICLE_IDENTITY_MYSQL_USERNAME`、`VEHICLE_IDENTITY_MYSQL_PASSWORD`。
|
||||
- 服务启动时自动创建 `vehicle_identity_binding` 和 `vehicle_identity_binding_registration` 两张表。`vehicle_identity_binding` 只维护 `plate`、`vin` 两列；`vehicle_identity_binding_registration` 按 `protocol + phone` 保存 JT808 0x0100 注册字段。
|
||||
- 注册帧无真实 VIN 时也会写入 registration 表，`vin` 默认为 `unknown`。外部系统识别车辆后只需要维护车牌和 VIN，例如：`INSERT INTO vehicle_identity_binding (plate, vin) VALUES ('沪A61559F', 'LNVIN000000000001') ON DUPLICATE KEY UPDATE vin = VALUES(vin);`
|
||||
- MySQL 绑定在启动时加载到内存索引，`plate/vin` 反写后会按 `VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL` 周期刷新，默认 `60s`；帧解析热路径只查内存，不按帧访问 MySQL。
|
||||
- 后续同一终端上报 raw/event 时，服务会用 registration 表里的 `phone/device_id/plate` 加绑定表里的 `plate/vin` 解析成真实 VIN。
|
||||
|
||||
健康检查：
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20400/actuator/health
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20400/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20100/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20500/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20200/actuator/health/readiness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/liveness
|
||||
curl -sS http://127.0.0.1:20310/actuator/health/readiness
|
||||
```
|
||||
|
||||
API / Swagger：
|
||||
|
||||
- JT808 ingest health: `http://127.0.0.1:20400/actuator/health`
|
||||
- GB32960 ingest health: `http://127.0.0.1:20100/actuator/health`
|
||||
- Yutong MQTT ingest health: `http://127.0.0.1:20500/actuator/health`
|
||||
- History query: `http://127.0.0.1:20200/swagger-ui/index.html`
|
||||
- Analytics metrics: `http://127.0.0.1:20310/swagger-ui/index.html`
|
||||
|
||||
## Kafka Topic 验证
|
||||
|
||||
历史服务必须消费事件 topic 和 raw topic：
|
||||
|
||||
- `vehicle.event.gb32960.v1`
|
||||
- `vehicle.raw.gb32960.v1`
|
||||
- `vehicle.event.jt808.v1`
|
||||
- `vehicle.raw.jt808.v1`
|
||||
- `vehicle.event.mqtt-yutong.v1`
|
||||
- `vehicle.raw.mqtt-yutong.v1`
|
||||
|
||||
使用 Kafka CLI 或管理工具检查 topic 是否存在，并确认 history consumer group 在测试流量后没有持续堆积。默认 `KAFKA_GROUP_HISTORY=vehicle-history` 时，实际 group 会拆成：
|
||||
|
||||
- `vehicle-history-gb32960-event`
|
||||
- `vehicle-history-gb32960-raw`
|
||||
- `vehicle-history-jt808-event`
|
||||
- `vehicle-history-jt808-raw`
|
||||
- `vehicle-history-yutong-mqtt-event`
|
||||
- `vehicle-history-yutong-mqtt-raw`
|
||||
|
||||
## JT808 实时转发验收
|
||||
|
||||
如果外部平台已经把 JT808 报文转发到 ECS TCP `808`，至少观察 60 秒日志和 Kafka 数据。
|
||||
|
||||
必须拿到以下证据：
|
||||
|
||||
- `jt808-ingest-app` 日志显示接入连接或解析到上游消息 ID。
|
||||
- Kafka 的 `vehicle.event.jt808.v1` 和 `vehicle.raw.jt808.v1` 有新增记录。
|
||||
- `vehicle-history-app` 日志没有持续出现 consumer 或 TDengine 写入失败。
|
||||
|
||||
消费完成后，使用已知终端手机号查询：
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/jt808/locations?phone=<phone>&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
```
|
||||
|
||||
预期：响应包含 `items`，并且每条记录包含 `eventTime`、`phone`、`longitude`、`latitude`、`speedKmh`、`rawUri`、`metadataJson`。
|
||||
|
||||
## GB32960 Snapshot / 字段投影验收
|
||||
|
||||
默认高吞吐模式下，GB32960 全字段查询不依赖 telemetry_fields 宽表。history 先查 TDengine `raw_frames`，直接使用 RAW 行中的 `parsedJson`/`parsedFields` 生成 snapshot 和字段投影：
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots?vin=<vin>&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin=<vin>&fields=VEHICLE.speedKmh,VEHICLE.totalMileageKm,POSITION_V2016.longitude,POSITION_V2016.latitude&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
```
|
||||
|
||||
预期：snapshot 返回 `sourceFrames.rawArchiveUri` 和解析后的 `blocks`；字段投影返回所选字段。新增协议字段后，先通过 replay 或专用解析任务补写 `raw_frames.parsedJson`，再基于历史 RAW 结构化结果查询。
|
||||
|
||||
## TDengine 直接检查
|
||||
|
||||
使用 TDengine CLI 或 JDBC 客户端检查超级表和子表是否创建：
|
||||
|
||||
```sql
|
||||
USE vehicle_ts;
|
||||
SHOW STABLES;
|
||||
SELECT COUNT(*) FROM raw_frames;
|
||||
SELECT COUNT(*) FROM vehicle_locations;
|
||||
SELECT COUNT(*) FROM jt808_locations;
|
||||
```
|
||||
|
||||
预期：
|
||||
|
||||
- `raw_frames`、`vehicle_locations`、`jt808_locations` 存在；有对应协议实时流量后，计数持续增长。
|
||||
- 字段趋势宽表不属于 history-app 验收范围，由独立字段解析服务负责。
|
||||
- 有实时流量后，计数持续增加。
|
||||
|
||||
## 失败语义
|
||||
|
||||
- TDengine 不可达时，TDengine 查询接口应该返回 HTTP `503`，消息类似 `tdengine history query failed`。
|
||||
- TDengine 未启用时，TDengine 专属接口不应该出现在 OpenAPI 中。
|
||||
- 路径不存在时，API 层应该返回 HTTP `404`，而不是泛化的存储失败。
|
||||
|
||||
## 2026-06-29 Superseded Local Live Verification Record
|
||||
|
||||
该记录仅保留为解析和存储行为的历史证据；生产运行位置已经收敛到 ECS/Portainer。
|
||||
|
||||
当时参与验证的服务：
|
||||
|
||||
- `jt808-ingest-app`：TCP `808`，HTTP `20400`
|
||||
- `gb32960-ingest-app`：TCP `32960`，HTTP `20100`
|
||||
- `vehicle-history-app`：HTTP `20200`
|
||||
|
||||
当次健康检查均返回 `{"status":"UP"}`。
|
||||
|
||||
可重复运行 live 验收工具：
|
||||
|
||||
```bash
|
||||
python3 tools/vehicle_ingest_live_verify.py \
|
||||
--tdengine-rest-url 'http://115.29.185.82:6041/rest/sql/vehicle_ts' \
|
||||
--tdengine-username root \
|
||||
--tdengine-password '<tdengine-password>' \
|
||||
--history-base-url 'http://127.0.0.1:20200' \
|
||||
--date-from '2026-06-29 00:00:00' \
|
||||
--date-to '2026-06-29 23:59:59' \
|
||||
--jt808-peer-like '222.66.200.68:%' \
|
||||
--gb32960-peer-like '115.29.187.205:%'
|
||||
```
|
||||
|
||||
该工具输出 `pass`、`warn` 或 `fail`：当前正式 GB32960 只有平台登录、没有车辆 `0x02` 时会输出 `warn`，避免把平台在线误判为车辆数据在线。正式车辆数据验收时增加 `--require-gb32960-vehicle-realtime`，缺少非测试 VIN 的 `0x02` 会直接返回非 0 退出码。
|
||||
|
||||
JT808 真实转发链路已验证：
|
||||
|
||||
- ECS TCP `808` 有外部平台 `222.66.200.68` 持续发送 JT808 报文。
|
||||
- `vehicle.raw.jt808.v1` 和 `vehicle.event.jt808.v1` 被 history 消费。
|
||||
- `raw_frames` 中 `protocol='JT808'` 的真实行数已超过 `36000`。
|
||||
- 外部平台真实 0x0100 注册帧已超过 `347` 条，真实 0x0200 位置帧已超过 `12400` 条。
|
||||
- `jt808_locations` 中位置行数已超过 `8783`。
|
||||
- 当前高吞吐默认链路不维护 `telemetry_fields`；JT808 明细验证以 `raw_frames`、`vehicle_locations` 和 API 分页结果为准。
|
||||
- 分页 API 已用终端号 `13079963301` 验证第一页和 nextCursor 第二页，样例位置包含 `longitude=118.913846`、`latitude=31.927309`、`statusFlag=3` 和 `archive://2026/06/29/JT808/...` 原始帧引用。
|
||||
- 注册帧样例：终端号 `13079963320`、deviceId `9963320`、deviceType `SEG-9888G`、plate `沪A61559F`、maker `70112`、province `31`、city `113`、plateColor `2`。生产 VIN 反写依赖 MySQL identity store，后续 raw/event 的 VIN 解析也从 MySQL 绑定读取。
|
||||
|
||||
## JT808 ECS smoke / 轻压测工具
|
||||
|
||||
仓库提供 `tools/jt808_e2e_smoke.py`，用于重复验证 TCP `808` 到 TDengine history 的闭环：
|
||||
|
||||
```bash
|
||||
export TDENGINE_REST_URL='http://<tdengine-host>:6041/rest/sql/vehicle_ts'
|
||||
export TDENGINE_USERNAME='root'
|
||||
export TDENGINE_PASSWORD='<tdengine-password>'
|
||||
|
||||
python3 tools/jt808_e2e_smoke.py \
|
||||
--connection-mode session \
|
||||
--start-phone 13079962000 \
|
||||
--frames 3 \
|
||||
--expect-history-count 3 \
|
||||
--verify-pagination \
|
||||
--archive-root "$PROJECT_ROOT/data/archive-jt808" \
|
||||
--tdengine-rest-url "$TDENGINE_REST_URL" \
|
||||
--tdengine-username "$TDENGINE_USERNAME" \
|
||||
--tdengine-password "$TDENGINE_PASSWORD"
|
||||
```
|
||||
|
||||
默认 `--connection-mode per-frame`，适合单帧连接 smoke。生产验收建议使用 `--connection-mode session` 覆盖同一连接内连续位置帧。脚本会输出发送数量、history 可见数量、分页验证结果、raw archive 检查结果和 `tdengineRawFrames`。`tdengineRawFrames` 表示本次可见位置记录中的唯一 `rawUri` 有多少个已确认写入 TDengine `raw_frames`。
|
||||
- 合成 0x0100 注册帧终端号 `13079969999` 已验证：TCP `808` 返回 `0x8100` 注册 ACK，`raw_frames.metadata_json` 包含 `jt808.register.province`、`jt808.register.city`、`jt808.register.maker`、`jt808.register.deviceType`、`jt808.register.deviceId`、`jt808.register.plateColor`、`jt808.register.plate`，对应 raw archive 文件存在。
|
||||
|
||||
复查 SQL：
|
||||
|
||||
```sql
|
||||
USE vehicle_ts;
|
||||
SELECT COUNT(*) FROM raw_frames WHERE protocol = 'JT808';
|
||||
SELECT COUNT(*) FROM jt808_locations WHERE protocol = 'JT808';
|
||||
SELECT event_time, vehicle_key, phone, message_id, raw_uri
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'JT808'
|
||||
ORDER BY event_time DESC
|
||||
LIMIT 5;
|
||||
SELECT ts, frame_id, phone, message_id, metadata_json, raw_uri
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'JT808' AND phone = '13079969999' AND message_id = 256
|
||||
ORDER BY ts DESC
|
||||
LIMIT 3;
|
||||
```
|
||||
|
||||
复查 API：
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/jt808/locations?phone=13079963301&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&order=DESC&limit=3'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/jt808/raw-frames?phone=13079963320&messageId=256&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&order=DESC&limit=3'
|
||||
```
|
||||
|
||||
`/api/event-history/jt808/raw-frames` 用于排查注册、鉴权、心跳、位置等原始帧索引。返回项会保留 `messageId`、`messageIdHex`、`rawUri`、`parseStatus`、`peer`、`vin`、`phone` 和完整 `metadataJson`；注册帧的 `metadataJson` 包含 `jt808.register.province`、`jt808.register.city`、`jt808.register.maker`、`jt808.register.deviceType`、`jt808.register.deviceId`、`jt808.register.plateColor`、`jt808.register.plate`、`identitySource`、`identityResolved` 等字段。
|
||||
|
||||
GB32960 链路已用正式平台登录和合成实时帧验证：
|
||||
|
||||
- 正式平台 `115.29.187.205` 已连接 TCP `32960`，服务收到 `0x05 PLATFORM_LOGIN` 并返回 ACK；日志显示账号 `Hyundai`、策略 `ALLOW`。
|
||||
- 2026-06-29 正式平台登录 raw 已有 `3` 条，均落入 TDengine `raw_frames`，可通过 RAW 查询看到 `parsedJson` 中的 `PLATFORM_LOGIN` 解析结果。
|
||||
- 当前正式平台尚未发送车辆实时上报 `0x02`；非测试 VIN 的 `message_id=2` 计数为 `0`，因此不能把平台登录误判为车辆数据在线。
|
||||
- TCP `32960` 返回 GB32960 `2323` ACK。
|
||||
- `raw_frames` 中测试 VIN `LTEST202606290001` 已有 `2` 条 raw 记录。
|
||||
- 当前默认通过 GB32960 snapshot/fields 即时解析 `raw_frames` 中的 RAW JSON，不维护字段宽表增长。
|
||||
- latest raw 包含 `archive://2026/06/29/GB32960/LTEST202606290001/...`。
|
||||
- snapshots API 可查到 `speedKmh=62.4`、`totalMileageKm=223456.7`、`longitude=116.397128`、`latitude=39.916527`。
|
||||
|
||||
复查 SQL：
|
||||
|
||||
```sql
|
||||
USE vehicle_ts;
|
||||
SELECT COUNT(*) FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001';
|
||||
SELECT event_time, vin, raw_uri, frame_id
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND vin = 'LTEST202606290001'
|
||||
ORDER BY event_time DESC
|
||||
LIMIT 5;
|
||||
SELECT ts, vin, message_id, event_time, raw_uri, peer, metadata_json
|
||||
FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND peer LIKE '115.29.187.205:%'
|
||||
ORDER BY ts DESC
|
||||
LIMIT 5;
|
||||
SELECT COUNT(*) FROM raw_frames
|
||||
WHERE protocol = 'GB32960' AND message_id = 2 AND vin <> '' AND vin NOT LIKE 'LTEST%';
|
||||
```
|
||||
|
||||
也可以直接运行仓库 smoke 工具验证 TCP `32960`、GB32960 snapshot/fields、TDengine `raw_frames`。如果需要复核冷备 `.bin`，再传入接入服务 archive 根目录：
|
||||
|
||||
```bash
|
||||
export TDENGINE_REST_URL='http://<tdengine-host>:6041/rest/sql/vehicle_ts'
|
||||
export TDENGINE_USERNAME='root'
|
||||
export TDENGINE_PASSWORD='<tdengine-password>'
|
||||
|
||||
python3 tools/gb32960_e2e_smoke.py \
|
||||
--archive-root "$PROJECT_ROOT/data/archive-jt808" \
|
||||
--tdengine-rest-url "$TDENGINE_REST_URL" \
|
||||
--tdengine-username "$TDENGINE_USERNAME" \
|
||||
--tdengine-password "$TDENGINE_PASSWORD"
|
||||
```
|
||||
|
||||
预期输出包含 `records >= 1`、`fieldCounts` 中关键字段均大于 0、`tdengineRawFrames` 等于本次可见唯一 `rawUri` 数量。传入 `--archive-root` 时，`archiveChecked >= 1` 说明冷备文件也能复核；这里的 `records` 字段表示 snapshot 数量，用于兼容早期脚本输出名。
|
||||
|
||||
复查 API：
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots?vin=LTEST202606290001&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin=LTEST202606290001&fields=VEHICLE.speedKmh,VEHICLE.totalMileageKm,POSITION_V2016.longitude,POSITION_V2016.latitude&dateFrom=2026-06-29T00:00:00%2B08:00&dateTo=2026-06-29T23:59:59%2B08:00&limit=10'
|
||||
```
|
||||
|
||||
## 当前注意事项
|
||||
|
||||
- 当前生产链路使用 TDengine WebSocket JDBC，例如 `jdbc:TAOS-WS://<tdengine-host>:6041/vehicle_ts`。
|
||||
- `KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest` 适合生产接入新流量；如果要回放历史 Kafka 数据，需要切换 consumer group 或重置 offset。
|
||||
- `vehicle-history-app` raw 和 event 使用不同 consumer binding；raw consumer 必须开启，否则 `raw_frames` 不会持续增长。
|
||||
- JT808 设备没有 VIN 映射时，`vehicle_key` 使用 `jt808:<phone>`；启用 MySQL identity store 后，registration 表中已反写 VIN 的 phone/deviceId/plate 会用于后续 raw/event 的 VIN 解析。
|
||||
- GB32960 正式对端目前只验证到平台登录；等待真实车辆 `0x02` 到达后，再用非测试 VIN 复查 `vehicle_locations`、snapshot/fields API 和导出能力。
|
||||
@@ -1,600 +0,0 @@
|
||||
# GB32960 Body Parser — Per-Block Exception Isolation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** `Gb32960BodyParser` 主循环对单信息块的解析异常实现**单块隔离**：一块解析失败不再整帧放弃，改为把失败块兜成 `InfoBlock.Raw` 并继续解析后续信息块。
|
||||
|
||||
**Architecture:** 在主循环每次 `parser.parse(body)` 外面套 try/catch（仅捕获 `DecodeException` / `BufferUnderflowException` / `IndexOutOfBoundsException`，放行 RuntimeException 以免掩盖 parser bug）。失败时：
|
||||
- 若 `fixedLen ≥ 0` 且剩余字节 ≥ fixedLen → 回滚 reader → 按 fixedLen 截取 Raw → position 前进 fixedLen → `continue` 循环
|
||||
- 否则（变长 parser 或剩余不足）→ 剩余字节全部兜成 Raw → `break` 循环（变长块无法安全找下一块边界）
|
||||
|
||||
通过 `Gb32960Properties.Parse.lenientBlockFailure`（默认 true）控制开关，可以回退到严格模式。**不改 `InfoBlock.Raw` record 结构**（避免 downstream 兼容风险——`Gb32960EventMapper` 按 `findBlock(Class)` 类型查找，新增 Raw 不影响其行为）。
|
||||
|
||||
**Tech Stack:** Java 25, JUnit 5, AssertJ, Spring Boot ConfigurationProperties, Maven。仅改动 `protocol-gb32960` 模块。
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- **Modify** `protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/config/Gb32960Properties.java` — 新增 `Parse` 内嵌类 + `parse` 字段/getter/setter
|
||||
- **Modify** `protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/codec/Gb32960BodyParser.java` — 主循环 try/catch + recovery 逻辑 + `lenientBlockFailure` 字段
|
||||
- **Modify** `protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/config/Gb32960AutoConfiguration.java` — 把配置穿给 BodyParser
|
||||
- **Create** `protocol-gb32960/src/test/java/com/lingniu/ingest/protocol/gb32960/codec/Gb32960BodyParserIsolationTest.java` — 3 个隔离场景 + 1 个严格模式回退场景
|
||||
- **Modify** `bootstrap-all/src/main/resources/application.yml` — 添加 `parse:` 注释段示例
|
||||
- **Modify** `CHANGELOG.md` — 追加本次变更条目
|
||||
|
||||
---
|
||||
|
||||
## Task 1: 基线验证 —— 现有测试全绿
|
||||
|
||||
**Files:** 无
|
||||
|
||||
- [ ] **Step 1.1: 跑 protocol-gb32960 模块测试**
|
||||
|
||||
Run: `mvn -pl protocol-gb32960 test -q`
|
||||
Expected: BUILD SUCCESS，所有现有测试通过（`Gb32960DecoderGoldenTest`、`Gb32960FullBlocksTest`、`Gb32960DecoderTest`、`GuangdongFcEndToEndTest`、parser/* 各 Block Parser 单测、profile/* 选择器单测）。
|
||||
|
||||
如果基线红了，**先停下来修复基线**再进入 Task 2。
|
||||
|
||||
---
|
||||
|
||||
## Task 2: 添加 Parse 配置子节点
|
||||
|
||||
**Files:**
|
||||
- Modify: `protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/config/Gb32960Properties.java`
|
||||
|
||||
- [ ] **Step 2.1: 在 `Gb32960Properties` 类体中（在 `Auth` 字段后、`Tls` 字段前）新增 `parse` 字段**
|
||||
|
||||
位置参考：现在 L24 `private Auth auth = new Auth();` 下面一行，在 L27 `private Tls tls = new Tls();` 之前插入：
|
||||
|
||||
```java
|
||||
/**
|
||||
* 报文解析行为配置。
|
||||
*
|
||||
* <p>默认 {@code lenientBlockFailure=true}：单个信息块解析异常时兜成
|
||||
* {@link com.lingniu.ingest.protocol.gb32960.model.InfoBlock.Raw} 继续解析，
|
||||
* 不再整帧放弃。关闭后恢复旧行为（任意异常直接抛 {@code DecodeException}
|
||||
* 放弃整帧），仅在灰度回滚时使用。
|
||||
*/
|
||||
private Parse parse = new Parse();
|
||||
```
|
||||
|
||||
- [ ] **Step 2.2: 在类底部（`Tls` 静态类之后、`VendorExtension` 静态类之前）添加 `Parse` 静态内嵌类**
|
||||
|
||||
```java
|
||||
/**
|
||||
* 报文解析容错策略。
|
||||
*/
|
||||
public static class Parse {
|
||||
/**
|
||||
* 单块解析异常时是否兜底为 {@link com.lingniu.ingest.protocol.gb32960.model.InfoBlock.Raw}
|
||||
* 继续解析。默认开启。
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code true}（默认）：parser 抛 DecodeException / BufferUnderflowException /
|
||||
* IndexOutOfBoundsException 时，固定长度块按 fixedLen 截取 Raw 后 continue；
|
||||
* 变长块或剩余字节不足时，剩余全部兜成 Raw 后 break 循环。
|
||||
* <li>{@code false}：保留旧行为，任意异常直接抛出 DecodeException 放弃整帧。
|
||||
* </ul>
|
||||
*/
|
||||
private boolean lenientBlockFailure = true;
|
||||
|
||||
public boolean isLenientBlockFailure() { return lenientBlockFailure; }
|
||||
public void setLenientBlockFailure(boolean lenientBlockFailure) {
|
||||
this.lenientBlockFailure = lenientBlockFailure;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2.3: 在属性 getter/setter 区域（现有 `getTls` 之后）补充 `getParse` / `setParse`**
|
||||
|
||||
```java
|
||||
public Parse getParse() { return parse; }
|
||||
public void setParse(Parse parse) { this.parse = parse; }
|
||||
```
|
||||
|
||||
- [ ] **Step 2.4: 编译验证**
|
||||
|
||||
Run: `mvn -pl protocol-gb32960 compile -q`
|
||||
Expected: BUILD SUCCESS。
|
||||
|
||||
- [ ] **Step 2.5: 提交**
|
||||
|
||||
```bash
|
||||
git add protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/config/Gb32960Properties.java
|
||||
git commit -m "$(cat <<'EOF'
|
||||
config(gb32960): add parse.lenientBlockFailure toggle
|
||||
|
||||
新增 lingniu.ingest.gb32960.parse.lenientBlockFailure 配置开关（默认 true），
|
||||
为 Gb32960BodyParser 的单块异常隔离行为做回退开关。实现在后续 commit。
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: RED —— 写三个失败测试覆盖隔离场景
|
||||
|
||||
**Files:**
|
||||
- Create: `protocol-gb32960/src/test/java/com/lingniu/ingest/protocol/gb32960/codec/Gb32960BodyParserIsolationTest.java`
|
||||
|
||||
- [ ] **Step 3.1: 创建测试文件**
|
||||
|
||||
完整内容：
|
||||
|
||||
```java
|
||||
package com.lingniu.ingest.protocol.gb32960.codec;
|
||||
|
||||
import com.lingniu.ingest.api.spi.DecodeException;
|
||||
import com.lingniu.ingest.protocol.gb32960.codec.parser.v2016.AlarmV2016BlockParser;
|
||||
import com.lingniu.ingest.protocol.gb32960.codec.parser.v2016.PositionV2016BlockParser;
|
||||
import com.lingniu.ingest.protocol.gb32960.codec.parser.v2016.VehicleV2016BlockParser;
|
||||
import com.lingniu.ingest.protocol.gb32960.model.InfoBlock;
|
||||
import com.lingniu.ingest.protocol.gb32960.model.InfoBlockType;
|
||||
import com.lingniu.ingest.protocol.gb32960.model.ProtocolVersion;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/**
|
||||
* 验证 {@link Gb32960BodyParser} 单块异常隔离行为。
|
||||
*
|
||||
* <p>三个隔离场景：
|
||||
* <ol>
|
||||
* <li>固定长度块 parser 抛异常 → 失败块兜 Raw（按 fixedLen 截取）+ 后续块继续解析
|
||||
* <li>固定长度块剩余字节不足（截断帧） → 兜 Raw(剩余) + break 循环
|
||||
* <li>变长块 parser 抛异常 → 兜 Raw(从失败块起剩余全部) + break 循环
|
||||
* </ol>
|
||||
*
|
||||
* <p>外加一个严格模式回退测试：{@code lenientBlockFailure=false} 时任意异常应抛 DecodeException。
|
||||
*/
|
||||
class Gb32960BodyParserIsolationTest {
|
||||
|
||||
/** 构造一个"看起来合法但中途 parser 失败"用来注入失败的 Position parser。 */
|
||||
private static final InfoBlockParser EXPLODING_FIXED_LEN_POSITION = new InfoBlockParser() {
|
||||
@Override public ProtocolVersion version() { return ProtocolVersion.V2016; }
|
||||
@Override public int typeCode() { return 0x05; }
|
||||
@Override public int fixedLength() { return 9; }
|
||||
@Override public InfoBlock parse(ByteBuffer buffer) {
|
||||
// 模拟 parser 读了几字节后发现不合法就抛
|
||||
buffer.get(); buffer.get();
|
||||
throw new DecodeException("simulated parser failure in Position");
|
||||
}
|
||||
};
|
||||
|
||||
/** 变长块（fixedLength=-1），parse 时消费若干字节后抛。模拟 Alarm/Voltage 类列表读越界。 */
|
||||
private static final InfoBlockParser EXPLODING_VAR_LEN_ALARM = new InfoBlockParser() {
|
||||
@Override public ProtocolVersion version() { return ProtocolVersion.V2016; }
|
||||
@Override public int typeCode() { return 0x07; }
|
||||
@Override public int fixedLength() { return -1; }
|
||||
@Override public InfoBlock parse(ByteBuffer buffer) {
|
||||
// 假装读了个长度字段就爆
|
||||
buffer.get();
|
||||
throw new DecodeException("simulated parser failure in Alarm list-length read");
|
||||
}
|
||||
};
|
||||
|
||||
@Test
|
||||
void fixedLengthBlockFailure_isIsolated_subsequentBlocksStillParsed() {
|
||||
// 帧布局：Vehicle(0x01, 20B) + Position(0x05, 9B 但 parser 爆) + Engine(0x04) 不构造，
|
||||
// 简化为 Vehicle + FailingPosition + Vehicle 再次，验证"Position 之后还能继续"。
|
||||
InfoBlockParserRegistry registry = new InfoBlockParserRegistry(List.of(
|
||||
new VehicleV2016BlockParser(),
|
||||
EXPLODING_FIXED_LEN_POSITION));
|
||||
|
||||
Gb32960BodyParser parser = new Gb32960BodyParser(registry);
|
||||
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
writeValidVehicle(os); // 1+20 = 21B
|
||||
writePositionTypeAnd9ByteBody(os); // 1+9 = 10B (parser 会爆)
|
||||
writeValidVehicle(os); // 1+20 = 21B
|
||||
ByteBuffer body = ByteBuffer.wrap(os.toByteArray());
|
||||
|
||||
var result = parser.parse(ProtocolVersion.V2016, body);
|
||||
|
||||
assertThat(result.blocks()).hasSize(3);
|
||||
assertThat(result.blocks().get(0)).isInstanceOf(InfoBlock.Gb32960V2016.Vehicle.class);
|
||||
assertThat(result.blocks().get(1)).isInstanceOfSatisfying(InfoBlock.Raw.class, raw -> {
|
||||
assertThat(raw.typeCode()).isEqualTo(0x05);
|
||||
assertThat(raw.type()).isEqualTo(InfoBlockType.RAW);
|
||||
assertThat(raw.bytes()).hasSize(9); // 按 fixedLen 截取
|
||||
});
|
||||
assertThat(result.blocks().get(2)).isInstanceOf(InfoBlock.Gb32960V2016.Vehicle.class);
|
||||
assertThat(body.hasRemaining()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void truncatedFixedLengthBlock_isWrappedAsRaw_loopTerminates() {
|
||||
// Vehicle(21B) + Position(type 0x05)但 body 只给 3B —— 不足 9B
|
||||
InfoBlockParserRegistry registry = new InfoBlockParserRegistry(List.of(
|
||||
new VehicleV2016BlockParser(),
|
||||
new PositionV2016BlockParser()));
|
||||
|
||||
Gb32960BodyParser parser = new Gb32960BodyParser(registry);
|
||||
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
writeValidVehicle(os);
|
||||
os.write(0x05); // Position typeCode
|
||||
os.write(0); os.write(0); os.write(0); // 只 3B，远远不够 9B
|
||||
ByteBuffer body = ByteBuffer.wrap(os.toByteArray());
|
||||
|
||||
var result = parser.parse(ProtocolVersion.V2016, body);
|
||||
|
||||
assertThat(result.blocks()).hasSize(2);
|
||||
assertThat(result.blocks().get(0)).isInstanceOf(InfoBlock.Gb32960V2016.Vehicle.class);
|
||||
assertThat(result.blocks().get(1)).isInstanceOfSatisfying(InfoBlock.Raw.class, raw -> {
|
||||
assertThat(raw.typeCode()).isEqualTo(0x05);
|
||||
assertThat(raw.bytes()).hasSize(3); // 剩余字节全兜成 Raw
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void variableLengthBlockFailure_swallowsRemainderAsRaw_loopBreaks() {
|
||||
// Vehicle(21B) + FailingAlarm(0x07)后面还有一个 Vehicle，但因 Alarm 变长无法安全跳过，
|
||||
// 整段 Alarm 起的剩余字节都被兜成 Raw 后 break。
|
||||
InfoBlockParserRegistry registry = new InfoBlockParserRegistry(List.of(
|
||||
new VehicleV2016BlockParser(),
|
||||
EXPLODING_VAR_LEN_ALARM));
|
||||
|
||||
Gb32960BodyParser parser = new Gb32960BodyParser(registry);
|
||||
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
writeValidVehicle(os); // 21B
|
||||
os.write(0x07); // Alarm typeCode
|
||||
for (int i = 0; i < 10; i++) os.write(0xAA); // 10B 的 alarm body（parser 爆）
|
||||
writeValidVehicle(os); // 21B（应当不再被解析）
|
||||
ByteBuffer body = ByteBuffer.wrap(os.toByteArray());
|
||||
|
||||
var result = parser.parse(ProtocolVersion.V2016, body);
|
||||
|
||||
assertThat(result.blocks()).hasSize(2);
|
||||
assertThat(result.blocks().get(0)).isInstanceOf(InfoBlock.Gb32960V2016.Vehicle.class);
|
||||
assertThat(result.blocks().get(1)).isInstanceOfSatisfying(InfoBlock.Raw.class, raw -> {
|
||||
assertThat(raw.typeCode()).isEqualTo(0x07);
|
||||
// 剩余 10B alarm body + 1+20=21B 后续 Vehicle = 31B
|
||||
assertThat(raw.bytes()).hasSize(31);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void strictMode_throwsOnAnyBlockFailure() {
|
||||
InfoBlockParserRegistry registry = new InfoBlockParserRegistry(List.of(
|
||||
new VehicleV2016BlockParser(),
|
||||
EXPLODING_FIXED_LEN_POSITION));
|
||||
|
||||
Gb32960BodyParser parser = new Gb32960BodyParser(registry);
|
||||
parser.setLenientBlockFailure(false);
|
||||
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
writeValidVehicle(os);
|
||||
writePositionTypeAnd9ByteBody(os);
|
||||
ByteBuffer body = ByteBuffer.wrap(os.toByteArray());
|
||||
|
||||
assertThatThrownBy(() -> parser.parse(ProtocolVersion.V2016, body))
|
||||
.isInstanceOf(DecodeException.class);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// helpers
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
/** 写一个完整的合法 V2016 Vehicle 块（typeCode + 20B body）。 */
|
||||
private static void writeValidVehicle(ByteArrayOutputStream os) {
|
||||
os.write(0x01); // typeCode Vehicle
|
||||
os.write(0x01); // vehicleState=1
|
||||
os.write(0x01); // chargingState=1
|
||||
os.write(0x01); // runningMode=1
|
||||
os.write(0); os.write(0); // speed=0
|
||||
os.write(0); os.write(0); os.write(0); os.write(0); // mileage=0
|
||||
os.write(0); os.write(0); // totalVoltage=0
|
||||
os.write(0); os.write(0); // totalCurrent=0（offset 1000，原值0）
|
||||
os.write(50); // soc=50%
|
||||
os.write(0x01); // dcdc
|
||||
os.write(0); // gear
|
||||
os.write(0); os.write(0); // insulation
|
||||
os.write(0); // accelerator
|
||||
os.write(0); // brake
|
||||
}
|
||||
|
||||
/** 写 Position typeCode(0x05) + 9 字节 body（内容不重要，parser 替换成 EXPLODING 的）。 */
|
||||
private static void writePositionTypeAnd9ByteBody(ByteArrayOutputStream os) {
|
||||
os.write(0x05);
|
||||
for (int i = 0; i < 9; i++) os.write(0);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3.2: 跑测试验证 RED**
|
||||
|
||||
Run: `mvn -pl protocol-gb32960 test -Dtest=Gb32960BodyParserIsolationTest -q`
|
||||
Expected:
|
||||
- `fixedLengthBlockFailure_isIsolated_subsequentBlocksStillParsed` FAIL（当前会抛 DecodeException）
|
||||
- `truncatedFixedLengthBlock_isWrappedAsRaw_loopTerminates` FAIL（当前 L132 会抛 DecodeException）
|
||||
- `variableLengthBlockFailure_swallowsRemainderAsRaw_loopBreaks` FAIL（异常冒出整帧失败）
|
||||
- `strictMode_throwsOnAnyBlockFailure` FAIL（没有 `setLenientBlockFailure` 方法，编译就红）
|
||||
|
||||
**验证点**：编译失败（strictMode test 里 `setLenientBlockFailure` 尚未存在）是**预期** RED 信号——下一步实现。
|
||||
|
||||
---
|
||||
|
||||
## Task 4: GREEN —— 实现单块异常隔离
|
||||
|
||||
**Files:**
|
||||
- Modify: `protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/codec/Gb32960BodyParser.java`
|
||||
|
||||
- [ ] **Step 4.1: 在类字段区新增 `lenientBlockFailure` 字段 + setter**
|
||||
|
||||
在 `private final VendorExtensionSelector selector;`（现 L54）下面插入：
|
||||
|
||||
```java
|
||||
/**
|
||||
* 单块解析异常时是否兜底为 Raw 后继续。由
|
||||
* {@link com.lingniu.ingest.protocol.gb32960.config.Gb32960Properties.Parse#isLenientBlockFailure()}
|
||||
* 注入；默认 true。
|
||||
*/
|
||||
private boolean lenientBlockFailure = true;
|
||||
|
||||
public void setLenientBlockFailure(boolean lenientBlockFailure) {
|
||||
this.lenientBlockFailure = lenientBlockFailure;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4.2: 改造主循环：替换 L130~L150 的整个 "parse + 长度校验" 段**
|
||||
|
||||
**完整替换块**：以下代码替换原来从 `int fixedLen = parser.fixedLength();`（L130）开始到 `blocks.add(block);`（L150）结束的整块。
|
||||
|
||||
```java
|
||||
int fixedLen = parser.fixedLength();
|
||||
int posBefore = body.position();
|
||||
|
||||
// 旧行为：fixedLen 预检不足 → 直接抛。新行为（lenient 模式）：走统一恢复路径。
|
||||
if (fixedLen >= 0 && body.remaining() < fixedLen) {
|
||||
if (!lenientBlockFailure) {
|
||||
throw new DecodeException(
|
||||
"info block 0x" + Integer.toHexString(typeCode)
|
||||
+ " needs " + fixedLen + " bytes but got " + body.remaining());
|
||||
}
|
||||
// 剩余字节数不足 fixedLen —— 无法按块截取，整尾兜 Raw + break
|
||||
int remaining = body.remaining();
|
||||
byte[] tail = new byte[remaining];
|
||||
body.get(tail);
|
||||
log.warn("[gb32960] truncated block typeCode=0x{} declaredFixedLen={} remaining={} — wrapping tail as Raw",
|
||||
Integer.toHexString(typeCode), fixedLen, remaining);
|
||||
blocks.add(new InfoBlock.Raw(typeCode, InfoBlockType.RAW, tail));
|
||||
break;
|
||||
}
|
||||
|
||||
InfoBlock block;
|
||||
try {
|
||||
block = parser.parse(body);
|
||||
} catch (DecodeException | java.nio.BufferUnderflowException | IndexOutOfBoundsException e) {
|
||||
if (!lenientBlockFailure) {
|
||||
if (e instanceof DecodeException de) throw de;
|
||||
throw new DecodeException(
|
||||
"parser " + parser.getClass().getSimpleName() + " failed: " + e.getMessage(), e);
|
||||
}
|
||||
// 恢复路径：先回滚 position 到 parse 入口
|
||||
body.position(posBefore);
|
||||
if (fixedLen >= 0) {
|
||||
// 固定长度块：按 fixedLen 截取 → 兜 Raw → continue 循环
|
||||
byte[] corrupt = new byte[fixedLen];
|
||||
body.get(corrupt);
|
||||
log.warn("[gb32960] block parse failed typeCode=0x{} parser={} fixedLen={} — isolated as Raw, continuing",
|
||||
Integer.toHexString(typeCode), parser.getClass().getSimpleName(), fixedLen, e);
|
||||
blocks.add(new InfoBlock.Raw(typeCode, InfoBlockType.RAW, corrupt));
|
||||
continue;
|
||||
} else {
|
||||
// 变长块：无法安全找下一块边界 → 剩余整段兜 Raw + break
|
||||
int remaining = body.remaining();
|
||||
byte[] tail = new byte[remaining];
|
||||
body.get(tail);
|
||||
log.warn("[gb32960] variable-length block parse failed typeCode=0x{} parser={} remaining={} — wrapping remainder as Raw, stopping loop",
|
||||
Integer.toHexString(typeCode), parser.getClass().getSimpleName(), remaining, e);
|
||||
blocks.add(new InfoBlock.Raw(typeCode, InfoBlockType.RAW, tail));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int consumed = body.position() - posBefore;
|
||||
if (fixedLen >= 0 && consumed != fixedLen) {
|
||||
// 这是 parser 契约违反（声明 fixedLen=X 但实际读了 Y），保持抛异常以暴露 parser bug。
|
||||
throw new DecodeException(
|
||||
"parser " + parser.getClass().getSimpleName()
|
||||
+ " consumed " + consumed
|
||||
+ " bytes but declared " + fixedLen);
|
||||
}
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("[gb32960] block typeCode=0x{} parser={} pos {}→{} consumed={} declaredFixed={}",
|
||||
Integer.toHexString(typeCode), parser.getClass().getSimpleName(),
|
||||
typeStartPos, body.position(), consumed, fixedLen);
|
||||
}
|
||||
blocks.add(block);
|
||||
```
|
||||
|
||||
- [ ] **Step 4.3: 运行隔离测试验证 GREEN**
|
||||
|
||||
Run: `mvn -pl protocol-gb32960 test -Dtest=Gb32960BodyParserIsolationTest -q`
|
||||
Expected: 4 个测试全部 PASS。
|
||||
|
||||
- [ ] **Step 4.4: 运行全模块测试检查回归**
|
||||
|
||||
Run: `mvn -pl protocol-gb32960 test -q`
|
||||
Expected: BUILD SUCCESS，所有原有测试（含 Golden、FullBlocks、GuangdongFcEndToEnd）依旧通过。
|
||||
|
||||
如果有原有测试红了：**停下来诊断**。最可能的原因是某个现有测试之前靠"抛异常"行为验证错误帧的，需要手动判断这是测试问题还是实现问题。
|
||||
|
||||
- [ ] **Step 4.5: 提交**
|
||||
|
||||
```bash
|
||||
git add protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/codec/Gb32960BodyParser.java \
|
||||
protocol-gb32960/src/test/java/com/lingniu/ingest/protocol/gb32960/codec/Gb32960BodyParserIsolationTest.java
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(gb32960): isolate per-block parse failures in body parser
|
||||
|
||||
主循环改造：单信息块 parser 抛 DecodeException / BufferUnderflowException /
|
||||
IndexOutOfBoundsException 时不再放弃整帧。
|
||||
|
||||
- 固定长度块（fixedLen≥0）：回滚 reader，按 fixedLen 截取字节兜成 InfoBlock.Raw，
|
||||
position 前进 fixedLen，continue 循环继续解析后续块
|
||||
- 变长块（fixedLen=-1）或剩余不足 fixedLen：剩余字节全部兜成 Raw 后 break
|
||||
- parser 契约违反（consumed != declared fixedLen）仍抛异常，以暴露 parser bug
|
||||
- 通过 lingniu.ingest.gb32960.parse.lenientBlockFailure=false 可回退严格模式
|
||||
- 新增 Gb32960BodyParserIsolationTest 覆盖 3 种失败场景 + 1 严格模式
|
||||
|
||||
不改 InfoBlock.Raw record 结构；下游 Gb32960EventMapper 按 findBlock(Class) 类型
|
||||
查找，新增 Raw 不影响其行为。
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: 把配置注入 BodyParser（AutoConfiguration 连线）
|
||||
|
||||
**Files:**
|
||||
- Modify: `protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/config/Gb32960AutoConfiguration.java`
|
||||
|
||||
- [ ] **Step 5.1: 定位 `Gb32960BodyParser` bean 定义**
|
||||
|
||||
先查看：`grep -n "Gb32960BodyParser" protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/config/Gb32960AutoConfiguration.java`
|
||||
|
||||
根据返回的行号，在构造 `Gb32960BodyParser` 的 @Bean 方法里，构造完成后立即调用：
|
||||
|
||||
```java
|
||||
Gb32960BodyParser parser = new Gb32960BodyParser(profileRegistry, selector);
|
||||
parser.setLenientBlockFailure(properties.getParse().isLenientBlockFailure());
|
||||
return parser;
|
||||
```
|
||||
|
||||
（如果目前的构造返回一行表达式，改成先赋给 local 变量再设 flag 再 return。）
|
||||
|
||||
- [ ] **Step 5.2: 编译并运行全模块测试**
|
||||
|
||||
Run: `mvn -pl protocol-gb32960 test -q`
|
||||
Expected: BUILD SUCCESS。
|
||||
|
||||
- [ ] **Step 5.3: 提交**
|
||||
|
||||
```bash
|
||||
git add protocol-gb32960/src/main/java/com/lingniu/ingest/protocol/gb32960/config/Gb32960AutoConfiguration.java
|
||||
git commit -m "$(cat <<'EOF'
|
||||
config(gb32960): wire parse.lenientBlockFailure into BodyParser bean
|
||||
|
||||
Gb32960AutoConfiguration 在创建 Gb32960BodyParser bean 后，将
|
||||
Gb32960Properties.Parse.lenientBlockFailure 通过 setter 注入，让运行期配置生效。
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: bootstrap-all 配置文档化
|
||||
|
||||
**Files:**
|
||||
- Modify: `bootstrap-all/src/main/resources/application.yml`
|
||||
|
||||
- [ ] **Step 6.1: 在 `lingniu.ingest.gb32960` 节点的 `vendor-extensions` 段之后、`jt808` 之前，插入 parse 配置示例**
|
||||
|
||||
参考位置：现有 application.yml L57 末尾（`vendor-extensions` list 结束）。在 L58 `jt808:` 之前加入：
|
||||
|
||||
```yaml
|
||||
# 报文解析容错。默认启用单块异常隔离：某个信息块解析失败时兜成 Raw 后继续，
|
||||
# 不再整帧丢弃。仅在需要严格失败语义（灰度回滚或抓虫）时设为 false。
|
||||
parse:
|
||||
lenient-block-failure: true
|
||||
```
|
||||
|
||||
- [ ] **Step 6.2: 启动一次 bootstrap-all 的 dry-run 编译**
|
||||
|
||||
Run: `mvn -pl bootstrap-all compile -q`
|
||||
Expected: BUILD SUCCESS（只是配置段注释变化，不会影响编译）。
|
||||
|
||||
- [ ] **Step 6.3: 提交**
|
||||
|
||||
```bash
|
||||
git add bootstrap-all/src/main/resources/application.yml
|
||||
git commit -m "$(cat <<'EOF'
|
||||
config(gb32960): document parse.lenient-block-failure in application.yml
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 7: 全仓库回归测试
|
||||
|
||||
**Files:** 无（只是验证）
|
||||
|
||||
- [ ] **Step 7.1: 跑完整项目测试**
|
||||
|
||||
Run: `mvn test -q`
|
||||
Expected: BUILD SUCCESS。重点关注：
|
||||
- `protocol-gb32960` 模块全绿（Golden、FullBlocks、Isolation、GuangdongFcEndToEnd、Decoder、Mapper、所有 parser 单测、profile）
|
||||
- 其它模块（`ingest-core`、`sink-kafka` 等）不受影响（无改动应自动通过）
|
||||
|
||||
若有红，停下来诊断。
|
||||
|
||||
---
|
||||
|
||||
## Task 8: 更新 CHANGELOG
|
||||
|
||||
**Files:**
|
||||
- Modify: `CHANGELOG.md`
|
||||
|
||||
- [ ] **Step 8.1: 在 CHANGELOG.md 文件顶部（在 `## [0.1.0] — 2026-04-15` 标题之前）新增一个 `## [Unreleased]` 段落；如已存在则追加**
|
||||
|
||||
新增段落内容（如已存在 Unreleased 段则在其 `### Added` / `### Changed` 内部追加）：
|
||||
|
||||
```markdown
|
||||
## [Unreleased]
|
||||
|
||||
### Changed —— GB/T 32960 Body Parser 单块异常隔离
|
||||
- `Gb32960BodyParser` 主循环对单信息块的 parser 异常不再放弃整帧：
|
||||
- 固定长度块（`fixedLength ≥ 0`）：回滚 reader → 按 fixedLen 截取字节兜成
|
||||
`InfoBlock.Raw` → 继续解析后续块；
|
||||
- 变长块或剩余字节不足：剩余字节全部兜成 `Raw` 后终止循环；
|
||||
- `parser` 契约违反（声明 fixedLen=X 但实际读了 Y）仍抛 `DecodeException`，
|
||||
以暴露 parser bug，不被静默吞掉；
|
||||
- 只捕获 `DecodeException` / `BufferUnderflowException` / `IndexOutOfBoundsException`，
|
||||
`RuntimeException` 继续向上抛，保留 bug 可见性。
|
||||
- 新增配置 `lingniu.ingest.gb32960.parse.lenient-block-failure`（默认 `true`）；
|
||||
回退严格模式用 `false`。
|
||||
- `InfoBlock.Raw` record 结构**未变**，下游 `Gb32960EventMapper` 按 `findBlock(Class)`
|
||||
类型匹配，新增 Raw 不影响业务事件映射。
|
||||
- 覆盖测试：`Gb32960BodyParserIsolationTest` —— 3 种隔离场景 + 严格模式回退。
|
||||
```
|
||||
|
||||
- [ ] **Step 8.2: 提交**
|
||||
|
||||
```bash
|
||||
git add CHANGELOG.md
|
||||
git commit -m "$(cat <<'EOF'
|
||||
docs(changelog): record gb32960 body parser block isolation
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review Checklist
|
||||
|
||||
- [x] **Spec coverage**: 原讨论的 A1~A7 全部覆盖——A1 状态机（Task 4 Step 4.2）、A2 三类错误策略（同）、A3 收窄异常种类（同，只 catch 三种）、A4 日志分类（warn 带 parser 名/typeCode/原因）、A5 配置开关（Task 2）、A6 合成帧测试（Task 3 三场景 + 严格回退）、A7 下游兼容（不改 Raw record，`findBlock` 类型匹配不受影响，plan 中已说明）。
|
||||
- [x] **Placeholder scan**: 无 TBD / TODO / "实现上面的" 等占位；每个 Step 含实际代码 or 命令。
|
||||
- [x] **Type consistency**: `lenientBlockFailure` 在 Properties / BodyParser 字段 / setter / 测试中拼写一致。`InfoBlockType.RAW` 枚举值依赖当前存在（已核对 InfoBlock.Raw record）。
|
||||
- [x] **测试命名一致**：`Gb32960BodyParserIsolationTest` 在 Task 3 创建、Task 4/7 引用一致。
|
||||
- [x] **未引入新枚举/proto 变更**：InfoBlock.Raw 沿用现有 `(int typeCode, InfoBlockType type, byte[] bytes)` 构造。
|
||||
@@ -1,817 +0,0 @@
|
||||
> **Superseded:** This 2026-06-23 DuckDB hot-store plan is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine stores hot history, `sink-archive`
|
||||
> owns raw bytes, and `event-file-store` / DuckDB are no longer part of the
|
||||
> current build surface.
|
||||
|
||||
# GB32960 History DuckDB Hot Store Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace the current high-write-risk Parquet rewrite history path with a DuckDB hot history store that supports fast `vin + time` queries and full RAW frame replay.
|
||||
|
||||
**Architecture:** Add a new `DuckDbHotEventFileStore` implementation behind the existing `EventFileStore` interface, backed by append/upsert DuckDB tables instead of rewriting per-vehicle Parquet files. Keep the current HTTP history API and RAW archive reader intact, then switch `vehicle-history-app` to the hot DuckDB store by configuration.
|
||||
|
||||
**Tech Stack:** Java 26, Spring Boot, DuckDB JDBC, JUnit 5, AssertJ, existing Kafka `VehicleEnvelope`, existing `ArchiveStore`.
|
||||
|
||||
---
|
||||
|
||||
## Scope
|
||||
|
||||
This plan implements Phase 1 from `docs/superpowers/specs/2026-06-23-gb32960-production-readiness-design.md`.
|
||||
|
||||
Included:
|
||||
|
||||
- DuckDB hot event table for `EventFileRecord`.
|
||||
- Idempotent writes by `event_id`.
|
||||
- Query by protocol/date/VIN/eventType/eventTime/order/limit.
|
||||
- Lookup by `rawArchiveUri`.
|
||||
- Spring configuration to select the hot store.
|
||||
- Tests that prove history endpoints still replay RAW frames.
|
||||
- Local verification using the existing `vehicle-history-app`.
|
||||
|
||||
Not included in this plan:
|
||||
|
||||
- MySQL daily statistics.
|
||||
- VIN-to-platform local mapping.
|
||||
- Full telemetry point columnar table.
|
||||
- Alarm timeline MySQL tables.
|
||||
- Cold Parquet export.
|
||||
|
||||
Those will be separate plans after this foundation lands.
|
||||
|
||||
## Files
|
||||
|
||||
- Create: `modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbHotEventFileStore.java`
|
||||
- Create: `modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbHotEventFileStoreTest.java`
|
||||
- Modify: `modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreProperties.java`
|
||||
- Modify: `modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfiguration.java`
|
||||
- Modify: `modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfigurationTest.java`
|
||||
- Modify: `modules/apps/vehicle-history-app/src/main/resources/application.yml`
|
||||
- Modify: `modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java`
|
||||
- Modify: `modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Gb32960DecodedFrameServiceTest.java`
|
||||
|
||||
## Task 1: Add Hot Store Configuration
|
||||
|
||||
**Files:**
|
||||
- Modify: `modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreProperties.java`
|
||||
- Modify: `modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfiguration.java`
|
||||
- Test: `modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfigurationTest.java`
|
||||
|
||||
- [ ] **Step 1: Write the failing auto-configuration test**
|
||||
|
||||
Add a test that sets `lingniu.ingest.event-file-store.storage=duckdb-hot` and expects the bean class to be `DuckDbHotEventFileStore`.
|
||||
|
||||
```java
|
||||
@Test
|
||||
void createsDuckDbHotStoreWhenStorageIsDuckDbHot() {
|
||||
contextRunner
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.event-file-store.enabled=true",
|
||||
"lingniu.ingest.event-file-store.storage=duckdb-hot",
|
||||
"lingniu.ingest.event-file-store.path=" + tempDir.resolve("history"))
|
||||
.run(context -> assertThat(context)
|
||||
.hasSingleBean(EventFileStore.class)
|
||||
.getBean(EventFileStore.class)
|
||||
.isInstanceOf(DuckDbHotEventFileStore.class));
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the new test and verify it fails**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-file-store -Dtest=EventFileStoreAutoConfigurationTest#createsDuckDbHotStoreWhenStorageIsDuckDbHot test
|
||||
```
|
||||
|
||||
Expected: compilation failure because `DuckDbHotEventFileStore` and `storage` property do not exist.
|
||||
|
||||
- [ ] **Step 3: Add the `storage` property**
|
||||
|
||||
Add to `EventFileStoreProperties`:
|
||||
|
||||
```java
|
||||
/**
|
||||
* Storage backend. `duckdb-hot` is the production backend; `parquet-sidecar`
|
||||
* keeps the previous implementation available for compatibility tests.
|
||||
*/
|
||||
private String storage = "duckdb-hot";
|
||||
|
||||
public String getStorage() {
|
||||
return storage;
|
||||
}
|
||||
|
||||
public void setStorage(String storage) {
|
||||
this.storage = storage;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Switch auto-configuration by storage mode**
|
||||
|
||||
Change `eventFileStore(...)` to:
|
||||
|
||||
```java
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public EventFileStore eventFileStore(EventFileStoreProperties properties,
|
||||
ObjectProvider<ObjectMapper> objectMapper) {
|
||||
ObjectMapper mapper = mapper(objectMapper);
|
||||
Path root = Path.of(properties.getPath());
|
||||
ZoneId zoneId = ZoneId.of(properties.getZoneId());
|
||||
String storage = properties.getStorage() == null ? "" : properties.getStorage().trim();
|
||||
return switch (storage) {
|
||||
case "", "duckdb-hot" -> new DuckDbHotEventFileStore(root, zoneId, mapper);
|
||||
case "parquet-sidecar" -> new DuckDbParquetEventFileStore(root, zoneId, mapper);
|
||||
default -> throw new IllegalStateException(
|
||||
"unsupported event-file-store storage: " + properties.getStorage());
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
Import `DuckDbHotEventFileStore`.
|
||||
|
||||
- [ ] **Step 5: Run configuration tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-file-store -Dtest=EventFileStoreAutoConfigurationTest test
|
||||
```
|
||||
|
||||
Expected: tests pass after the store class exists in Task 2.
|
||||
|
||||
## Task 2: Implement DuckDB Hot Store Schema and Writes
|
||||
|
||||
**Files:**
|
||||
- Create: `modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbHotEventFileStore.java`
|
||||
- Create: `modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbHotEventFileStoreTest.java`
|
||||
|
||||
- [ ] **Step 1: Write failing append/query/idempotency tests**
|
||||
|
||||
Create `DuckDbHotEventFileStoreTest`:
|
||||
|
||||
```java
|
||||
class DuckDbHotEventFileStoreTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@Test
|
||||
void appendsRecordsAndQueriesByVinTypeAndTime() throws Exception {
|
||||
EventFileStore store = store();
|
||||
store.append(rawRecord("raw-1", "VIN001", "2026-06-23T01:00:00Z"));
|
||||
store.append(rawRecord("raw-2", "VIN002", "2026-06-23T01:00:01Z"));
|
||||
store.append(rawRecord("raw-3", "VIN001", "2026-06-23T01:00:02Z"));
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-23"),
|
||||
LocalDate.parse("2026-06-23"),
|
||||
Instant.parse("2026-06-23T01:00:00Z"),
|
||||
Instant.parse("2026-06-23T01:00:03Z"),
|
||||
EventFileQuery.Order.DESC,
|
||||
2,
|
||||
"VIN001",
|
||||
"RAW_ARCHIVE");
|
||||
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("raw-3", "raw-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void appendAllIsIdempotentByEventId() throws Exception {
|
||||
EventFileStore store = store();
|
||||
EventFileRecord original = rawRecord("same-id", "VIN001", "2026-06-23T01:00:00Z");
|
||||
EventFileRecord replacement = new EventFileRecord(
|
||||
"same-id",
|
||||
ProtocolId.GB32960,
|
||||
"RAW_ARCHIVE",
|
||||
"VIN001",
|
||||
Instant.parse("2026-06-23T01:00:05Z"),
|
||||
Instant.parse("2026-06-23T01:00:06Z"),
|
||||
"archive://replacement.bin",
|
||||
Map.of("source", "replacement"),
|
||||
"{\"replacement\":true}");
|
||||
|
||||
store.appendAll(List.of(original, replacement));
|
||||
|
||||
assertThat(store.query(new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-23"),
|
||||
LocalDate.parse("2026-06-23"),
|
||||
EventFileQuery.Order.ASC,
|
||||
10,
|
||||
"VIN001",
|
||||
"RAW_ARCHIVE")))
|
||||
.singleElement()
|
||||
.satisfies(record -> {
|
||||
assertThat(record.eventId()).isEqualTo("same-id");
|
||||
assertThat(record.rawArchiveUri()).isEqualTo("archive://replacement.bin");
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void findsRecordByRawArchiveUri() throws Exception {
|
||||
EventFileStore store = store();
|
||||
EventFileRecord record = rawRecord("raw-uri", "VIN001", "2026-06-23T01:00:00Z");
|
||||
store.append(record);
|
||||
|
||||
EventFileRecord found = store.findByRawArchiveUri(record.rawArchiveUri());
|
||||
|
||||
assertThat(found).isNotNull();
|
||||
assertThat(found.eventId()).isEqualTo("raw-uri");
|
||||
}
|
||||
|
||||
private EventFileStore store() {
|
||||
return new DuckDbHotEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"), new ObjectMapper());
|
||||
}
|
||||
|
||||
private static EventFileRecord rawRecord(String id, String vin, String eventTime) {
|
||||
return new EventFileRecord(
|
||||
id,
|
||||
ProtocolId.GB32960,
|
||||
"RAW_ARCHIVE",
|
||||
vin,
|
||||
Instant.parse(eventTime),
|
||||
Instant.parse(eventTime).plusMillis(100),
|
||||
"archive://" + id + ".bin",
|
||||
Map.of("platformAccount", "Hyundai", "command", "REALTIME_REPORT"),
|
||||
"{\"eventId\":\"" + id + "\"}");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the test and verify it fails**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-file-store -Dtest=DuckDbHotEventFileStoreTest test
|
||||
```
|
||||
|
||||
Expected: compilation failure because `DuckDbHotEventFileStore` does not exist.
|
||||
|
||||
- [ ] **Step 3: Create the hot store class**
|
||||
|
||||
Create `DuckDbHotEventFileStore` with this structure:
|
||||
|
||||
```java
|
||||
public final class DuckDbHotEventFileStore implements EventFileStore {
|
||||
|
||||
private static final TypeReference<Map<String, String>> STRING_MAP = new TypeReference<>() {};
|
||||
|
||||
private final Path root;
|
||||
private final Path dbPath;
|
||||
private final ZoneId partitionZone;
|
||||
private final ObjectMapper objectMapper;
|
||||
private volatile boolean initialized;
|
||||
|
||||
public DuckDbHotEventFileStore(Path root, ZoneId partitionZone) {
|
||||
this(root, partitionZone, new ObjectMapper());
|
||||
}
|
||||
|
||||
public DuckDbHotEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) {
|
||||
if (root == null) {
|
||||
throw new IllegalArgumentException("root must not be null");
|
||||
}
|
||||
this.root = root.toAbsolutePath();
|
||||
this.dbPath = this.root.resolve("events.duckdb");
|
||||
this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone;
|
||||
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void appendAll(List<EventFileRecord> records) throws IOException {
|
||||
if (records == null || records.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
ensureInitialized();
|
||||
try (Connection connection = DriverManager.getConnection(jdbcUrl())) {
|
||||
connection.setAutoCommit(false);
|
||||
try {
|
||||
upsertRecords(connection, records);
|
||||
connection.commit();
|
||||
} catch (SQLException | IOException e) {
|
||||
connection.rollback();
|
||||
throw e;
|
||||
} finally {
|
||||
connection.setAutoCommit(true);
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("write duckdb hot event store failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EventFileRecord> query(EventFileQuery query) throws IOException {
|
||||
ensureInitialized();
|
||||
// Implement in Task 3.
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EventFileRecord findByRawArchiveUri(String rawArchiveUri) throws IOException {
|
||||
ensureInitialized();
|
||||
// Implement in Task 3.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add schema initialization**
|
||||
|
||||
Add:
|
||||
|
||||
```java
|
||||
private void ensureInitialized() throws IOException {
|
||||
if (initialized) {
|
||||
return;
|
||||
}
|
||||
synchronized (this) {
|
||||
if (initialized) {
|
||||
return;
|
||||
}
|
||||
Files.createDirectories(root);
|
||||
try (Connection connection = DriverManager.getConnection(jdbcUrl());
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute("""
|
||||
CREATE TABLE IF NOT EXISTS event_records (
|
||||
event_id VARCHAR PRIMARY KEY,
|
||||
protocol VARCHAR NOT NULL,
|
||||
event_type VARCHAR NOT NULL,
|
||||
vin VARCHAR NOT NULL,
|
||||
event_time_ms BIGINT NOT NULL,
|
||||
ingest_time_ms BIGINT NOT NULL,
|
||||
partition_date DATE NOT NULL,
|
||||
raw_archive_uri VARCHAR NOT NULL,
|
||||
metadata_json VARCHAR NOT NULL,
|
||||
payload_json VARCHAR NOT NULL
|
||||
)
|
||||
""");
|
||||
statement.execute("""
|
||||
CREATE INDEX IF NOT EXISTS event_records_protocol_date_time_idx
|
||||
ON event_records(protocol, partition_date, event_time_ms)
|
||||
""");
|
||||
statement.execute("""
|
||||
CREATE INDEX IF NOT EXISTS event_records_vin_date_time_idx
|
||||
ON event_records(protocol, vin, partition_date, event_time_ms)
|
||||
""");
|
||||
statement.execute("""
|
||||
CREATE INDEX IF NOT EXISTS event_records_vin_type_date_time_idx
|
||||
ON event_records(protocol, vin, event_type, partition_date, event_time_ms)
|
||||
""");
|
||||
statement.execute("""
|
||||
CREATE INDEX IF NOT EXISTS event_records_raw_archive_uri_idx
|
||||
ON event_records(raw_archive_uri)
|
||||
""");
|
||||
initialized = true;
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("initialize duckdb hot event store failed", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Add idempotent batch upsert**
|
||||
|
||||
Use DuckDB `INSERT OR REPLACE` inside one transaction:
|
||||
|
||||
```java
|
||||
private void upsertRecords(Connection connection, List<EventFileRecord> records)
|
||||
throws SQLException, IOException {
|
||||
try (PreparedStatement ps = connection.prepareStatement("""
|
||||
INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""")) {
|
||||
for (EventFileRecord record : records) {
|
||||
ps.setString(1, record.eventId());
|
||||
ps.setString(2, record.protocol().name());
|
||||
ps.setString(3, record.eventType());
|
||||
ps.setString(4, record.vin());
|
||||
ps.setLong(5, record.eventTime().toEpochMilli());
|
||||
ps.setLong(6, record.ingestTime().toEpochMilli());
|
||||
ps.setString(7, LocalDate.ofInstant(record.eventTime(), partitionZone).toString());
|
||||
ps.setString(8, record.rawArchiveUri());
|
||||
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
|
||||
ps.setString(10, record.payloadJson());
|
||||
ps.addBatch();
|
||||
}
|
||||
ps.executeBatch();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Run the hot store tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-file-store -Dtest=DuckDbHotEventFileStoreTest test
|
||||
```
|
||||
|
||||
Expected: query tests still fail until Task 3 implements reads; append initialization should compile.
|
||||
|
||||
## Task 3: Implement Hot Store Queries and Raw URI Lookup
|
||||
|
||||
**Files:**
|
||||
- Modify: `modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbHotEventFileStore.java`
|
||||
- Test: `modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbHotEventFileStoreTest.java`
|
||||
|
||||
- [ ] **Step 1: Implement `query(EventFileQuery)`**
|
||||
|
||||
Use prepared statements for every external value:
|
||||
|
||||
```java
|
||||
@Override
|
||||
public List<EventFileRecord> query(EventFileQuery query) throws IOException {
|
||||
ensureInitialized();
|
||||
String order = query.order() == EventFileQuery.Order.DESC ? "DESC" : "ASC";
|
||||
StringBuilder where = new StringBuilder("""
|
||||
WHERE protocol = ?
|
||||
AND partition_date BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)
|
||||
""");
|
||||
if (query.vin() != null) {
|
||||
where.append(" AND vin = ?\n");
|
||||
}
|
||||
if (query.eventType() != null) {
|
||||
where.append(" AND event_type = ?\n");
|
||||
}
|
||||
if (query.eventTimeFrom() != null) {
|
||||
where.append(" AND event_time_ms >= ?\n");
|
||||
}
|
||||
if (query.eventTimeTo() != null) {
|
||||
where.append(" AND event_time_ms <= ?\n");
|
||||
}
|
||||
String sql = """
|
||||
SELECT event_id, protocol, event_type, vin, event_time_ms, ingest_time_ms,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM event_records
|
||||
%s
|
||||
ORDER BY event_time_ms %s, ingest_time_ms %s, event_id %s
|
||||
LIMIT ?
|
||||
""".formatted(where, order, order, order);
|
||||
try (Connection connection = DriverManager.getConnection(jdbcUrl());
|
||||
PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
bindQuery(ps, query);
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
List<EventFileRecord> out = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
out.add(record(rs));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("query duckdb hot event store failed", e);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add query binding helper**
|
||||
|
||||
```java
|
||||
private static void bindQuery(PreparedStatement ps, EventFileQuery query) throws SQLException {
|
||||
int index = 1;
|
||||
ps.setString(index++, query.protocol().name());
|
||||
ps.setString(index++, query.dateFrom().toString());
|
||||
ps.setString(index++, query.dateTo().toString());
|
||||
if (query.vin() != null) {
|
||||
ps.setString(index++, query.vin());
|
||||
}
|
||||
if (query.eventType() != null) {
|
||||
ps.setString(index++, query.eventType());
|
||||
}
|
||||
if (query.eventTimeFrom() != null) {
|
||||
ps.setLong(index++, query.eventTimeFrom().toEpochMilli());
|
||||
}
|
||||
if (query.eventTimeTo() != null) {
|
||||
ps.setLong(index++, query.eventTimeTo().toEpochMilli());
|
||||
}
|
||||
ps.setInt(index, query.limit());
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Implement `findByRawArchiveUri`**
|
||||
|
||||
```java
|
||||
@Override
|
||||
public EventFileRecord findByRawArchiveUri(String rawArchiveUri) throws IOException {
|
||||
if (rawArchiveUri == null || rawArchiveUri.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
ensureInitialized();
|
||||
try (Connection connection = DriverManager.getConnection(jdbcUrl());
|
||||
PreparedStatement ps = connection.prepareStatement("""
|
||||
SELECT event_id, protocol, event_type, vin, event_time_ms, ingest_time_ms,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM event_records
|
||||
WHERE raw_archive_uri = ?
|
||||
ORDER BY event_type = 'RAW_ARCHIVE' DESC, ingest_time_ms DESC, event_id DESC
|
||||
LIMIT 1
|
||||
""")) {
|
||||
ps.setString(1, rawArchiveUri);
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
return rs.next() ? record(rs) : null;
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("query duckdb hot store by raw archive uri failed", e);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add record mapper and helpers**
|
||||
|
||||
```java
|
||||
private EventFileRecord record(ResultSet rs) throws SQLException, IOException {
|
||||
return new EventFileRecord(
|
||||
rs.getString("event_id"),
|
||||
protocol(rs.getString("protocol")),
|
||||
rs.getString("event_type"),
|
||||
rs.getString("vin"),
|
||||
Instant.ofEpochMilli(rs.getLong("event_time_ms")),
|
||||
Instant.ofEpochMilli(rs.getLong("ingest_time_ms")),
|
||||
rs.getString("raw_archive_uri"),
|
||||
readMetadata(rs.getString("metadata_json")),
|
||||
rs.getString("payload_json"));
|
||||
}
|
||||
|
||||
private Map<String, String> readMetadata(String json) throws IOException {
|
||||
if (json == null || json.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
return objectMapper.readValue(json, STRING_MAP);
|
||||
}
|
||||
|
||||
private static ProtocolId protocol(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return ProtocolId.UNKNOWN;
|
||||
}
|
||||
try {
|
||||
return ProtocolId.valueOf(value);
|
||||
} catch (IllegalArgumentException ex) {
|
||||
return ProtocolId.UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
private String jdbcUrl() {
|
||||
return "jdbc:duckdb:" + dbPath;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run hot store tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-file-store -Dtest=DuckDbHotEventFileStoreTest test
|
||||
```
|
||||
|
||||
Expected: all `DuckDbHotEventFileStoreTest` tests pass.
|
||||
|
||||
## Task 4: Preserve Legacy Store Tests and Update Defaults
|
||||
|
||||
**Files:**
|
||||
- Modify: `modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStoreTest.java`
|
||||
- Modify: `modules/apps/vehicle-history-app/src/main/resources/application.yml`
|
||||
- Modify: `modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java`
|
||||
|
||||
- [ ] **Step 1: Keep Parquet tests explicitly legacy**
|
||||
|
||||
No behavior change is needed in `DuckDbParquetEventFileStoreTest`; leave it instantiating `DuckDbParquetEventFileStore` directly. Add a class comment:
|
||||
|
||||
```java
|
||||
/**
|
||||
* Compatibility coverage for the legacy Parquet sidecar backend.
|
||||
* Production history uses DuckDbHotEventFileStore through auto-configuration.
|
||||
*/
|
||||
class DuckDbParquetEventFileStoreTest {
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Set vehicle-history-app storage default**
|
||||
|
||||
Add to `modules/apps/vehicle-history-app/src/main/resources/application.yml`:
|
||||
|
||||
```yaml
|
||||
event-file-store:
|
||||
enabled: ${EVENT_FILE_STORE_ENABLED:true}
|
||||
storage: ${EVENT_FILE_STORE_STORAGE:duckdb-hot}
|
||||
path: ${EVENT_FILE_STORE_PATH:./target/event-store/}
|
||||
zone-id: ${EVENT_FILE_STORE_ZONE_ID:Asia/Shanghai}
|
||||
batch-size: ${EVENT_FILE_STORE_BATCH_SIZE:1000}
|
||||
flush-interval-millis: ${EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS:1000}
|
||||
```
|
||||
|
||||
Keep existing indentation and only add `storage`; if `batch-size` already exists, update it to `1000`.
|
||||
|
||||
- [ ] **Step 3: Update app default test**
|
||||
|
||||
In `VehicleHistoryAppDefaultsTest`, assert:
|
||||
|
||||
```java
|
||||
assertThat(context.getEnvironment()
|
||||
.getProperty("lingniu.ingest.event-file-store.storage"))
|
||||
.isEqualTo("duckdb-hot");
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run app default tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :vehicle-history-app -Dtest=VehicleHistoryAppDefaultsTest test
|
||||
```
|
||||
|
||||
Expected: default configuration test passes.
|
||||
|
||||
## Task 5: Verify History Ingest Still Archives RAW Bytes
|
||||
|
||||
**Files:**
|
||||
- Modify: `modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestorTest.java`
|
||||
- Test: `modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Gb32960DecodedFrameServiceTest.java`
|
||||
|
||||
- [ ] **Step 1: Add an integration-style test using the hot store**
|
||||
|
||||
Add a test that writes a RAW envelope through `EventHistoryEnvelopeIngestor` into `DuckDbHotEventFileStore`, then finds it by URI.
|
||||
|
||||
```java
|
||||
@Test
|
||||
void rawArchiveEnvelopeCanBeFoundFromDuckDbHotStoreByUri(@TempDir Path tempDir) throws Exception {
|
||||
EventFileStore store = new DuckDbHotEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"), OBJECT_MAPPER);
|
||||
CapturingArchiveStore archive = new CapturingArchiveStore();
|
||||
EventHistoryEnvelopeIngestor ingestor =
|
||||
new EventHistoryEnvelopeIngestor(store, new TelemetryEnvelopeRecordMapper(), archive);
|
||||
String rawArchiveKey = "2026/06/23/GB32960/VINRAW001/raw-event-hot.bin";
|
||||
String rawArchiveUri = "archive://" + rawArchiveKey;
|
||||
byte[] rawBytes = new byte[]{0x23, 0x23, 0x02, 0x01};
|
||||
|
||||
VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
|
||||
.setSchemaVersion("1.0")
|
||||
.setEventId("raw-event-hot")
|
||||
.setVin("VINRAW001")
|
||||
.setSource("GB32960")
|
||||
.setProtocolVersion("V2016")
|
||||
.setEventTimeMs(1_782_112_400_000L)
|
||||
.setIngestTimeMs(1_782_112_401_000L)
|
||||
.putMetadata(RawArchiveKeys.META_KEY, rawArchiveKey)
|
||||
.putMetadata(RawArchiveKeys.META_URI, rawArchiveUri)
|
||||
.setRawArchive(RawArchiveRef.newBuilder()
|
||||
.setUri(rawArchiveUri)
|
||||
.setSizeBytes(rawBytes.length)
|
||||
.setData(ByteString.copyFrom(rawBytes))
|
||||
.build())
|
||||
.build();
|
||||
|
||||
EnvelopeIngestResult result = ingestor.tryIngest(envelope.toByteArray());
|
||||
|
||||
assertThat(result.status()).isEqualTo(EnvelopeIngestResult.Status.STORED);
|
||||
assertThat(store.findByRawArchiveUri(rawArchiveUri))
|
||||
.isNotNull()
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.isEqualTo("raw-event-hot");
|
||||
assertThat(archive.bytesByKey).containsEntry(rawArchiveKey, rawBytes);
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the event history ingestor test**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-history-service -Dtest=EventHistoryEnvelopeIngestorTest test
|
||||
```
|
||||
|
||||
Expected: test passes.
|
||||
|
||||
- [ ] **Step 3: Run decoded frame service tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-history-service -Dtest=Gb32960DecodedFrameServiceTest test
|
||||
```
|
||||
|
||||
Expected: existing replay/snapshot tests pass. If they use a fake store, no change is needed.
|
||||
|
||||
## Task 6: Full Module Verification
|
||||
|
||||
**Files:**
|
||||
- No source changes unless failures expose missing imports or config assertions.
|
||||
|
||||
- [ ] **Step 1: Run sink module tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-file-store test
|
||||
```
|
||||
|
||||
Expected: all event-file-store tests pass, including both hot and legacy stores.
|
||||
|
||||
- [ ] **Step 2: Run event-history-service tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :event-history-service test
|
||||
```
|
||||
|
||||
Expected: all event-history-service tests pass.
|
||||
|
||||
- [ ] **Step 3: Package vehicle-history-app**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mvn -pl :vehicle-history-app -am package -DskipTests
|
||||
```
|
||||
|
||||
Expected: package succeeds.
|
||||
|
||||
## Task 7: Local Runtime Verification
|
||||
|
||||
**Files:**
|
||||
- No committed source changes.
|
||||
|
||||
- [ ] **Step 1: Stop any old history service on port 20200**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
lsof -tiTCP:20200 -sTCP:LISTEN | xargs -r kill
|
||||
```
|
||||
|
||||
Expected: no command output, or the old process exits.
|
||||
|
||||
- [ ] **Step 2: Start vehicle-history-app with hot DuckDB store**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
EVENT_FILE_STORE_STORAGE=duckdb-hot \
|
||||
EVENT_FILE_STORE_PATH=./target/live-history-event-store \
|
||||
SINK_ARCHIVE_PATH=./target/live-history-archive \
|
||||
KAFKA_BROKERS=114.55.58.251:9092 \
|
||||
KAFKA_CONSUMER_ENABLED=true \
|
||||
KAFKA_GROUP_HISTORY=vehicle-history-hot-$(date +%Y%m%d%H%M%S) \
|
||||
java --sun-misc-unsafe-memory-access=allow \
|
||||
-jar modules/apps/vehicle-history-app/target/vehicle-history-app.jar
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
- app starts on `http://127.0.0.1:20200`;
|
||||
- logs show Kafka consumer subscribed;
|
||||
- `target/live-history-event-store/events.duckdb` is created.
|
||||
|
||||
- [ ] **Step 3: Verify health**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:20200/actuator/health
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
```json
|
||||
{"status":"UP"}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify a recent VIN query**
|
||||
|
||||
Run with a VIN observed in live archive:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/telemetry-snapshots?vin=LNXNEGRR9SR318194&platformAccount=Hyundai&dateFrom=2026-06-23&dateTo=2026-06-24&order=DESC&limit=3'
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
- response is a JSON array;
|
||||
- if live traffic for that VIN exists after service start, at least one snapshot appears;
|
||||
- `rawArchiveUris` point to existing files under `target/live-history-archive`.
|
||||
|
||||
- [ ] **Step 5: Verify raw frame replay**
|
||||
|
||||
Use one `rawArchiveUri` from Step 4:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/frame?rawArchiveUri=archive://REPLACE_ME&platformAccount=Hyundai'
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
- response contains `vin`, `command`, `eventTime`, and parsed `blocks`;
|
||||
- no `raw archive is missing` warning for the selected URI.
|
||||
|
||||
## Self-Review Checklist
|
||||
|
||||
- [ ] The new hot store does not rewrite Parquet files on every append.
|
||||
- [ ] `appendAll` writes one batch in one transaction.
|
||||
- [ ] Duplicate `event_id` replay is idempotent.
|
||||
- [ ] Existing history HTTP APIs still use `EventFileStore`, so controller contracts are unchanged.
|
||||
- [ ] RAW bytes still land in `ArchiveStore`.
|
||||
- [ ] `findByRawArchiveUri` is backed by DuckDB index.
|
||||
- [ ] No MySQL credential, RDS host, username, or password appears in the plan or source files.
|
||||
- [ ] This plan does not claim the full production goal is complete; it only lands the history foundation.
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,46 +0,0 @@
|
||||
# JT808 Kafka Streaming Mileage Plan
|
||||
|
||||
## Current Goal
|
||||
|
||||
Consume JT808 Kafka location events and write the derived daily mileage into the common vehicle-stat metric repository.
|
||||
|
||||
## Current Design
|
||||
|
||||
- Only supported message backbone: Kafka.
|
||||
- Source topic: `vehicle.event.jt808.v1`
|
||||
- Runtime app: `vehicle-analytics-app`
|
||||
- Runtime state: none outside `vehicle_stat_metric`
|
||||
- Metric output: `VehicleStatRepository.recordDailyMileageSample(...)`
|
||||
- Production metric storage: JDBC/MySQL `vehicle_stat_metric`
|
||||
- Date boundary: `Asia/Shanghai`
|
||||
- Calculation method: `JT808_TOTAL_MILEAGE_DIFF`
|
||||
|
||||
JT808 daily mileage is calculated only from the GPS total mileage reported by JT808 location additional information:
|
||||
|
||||
```text
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
```
|
||||
|
||||
The first valid local-day sample writes `daily_mileage_km=0.0`. Later ordered or replayed samples update the same metric row with:
|
||||
|
||||
```text
|
||||
metric_value = max_total_mileage_km - min_total_mileage_km
|
||||
metric_key = daily_mileage_km
|
||||
```
|
||||
|
||||
The local-day minimum and maximum GPS total mileage values stay on that same metric row as calculation source columns so restarts recover from MySQL without Redis or another state table.
|
||||
|
||||
## Runtime Properties
|
||||
|
||||
```text
|
||||
KAFKA_TOPIC_JT808_EVENT=vehicle.event.jt808.v1
|
||||
VEHICLE_STAT_ENABLED=true
|
||||
VEHICLE_STAT_JT808_MILEAGE_ENABLED=true
|
||||
MYSQL_JDBC_URL=<jdbc-url>
|
||||
MYSQL_USERNAME=<user>
|
||||
MYSQL_PASSWORD=<password>
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
Do not create or write a protocol-specific JT808 daily-mileage table. Do not add another message backbone, distance accumulation, integral calculation, Redis state, or memory state back into this path unless the mileage definition changes again.
|
||||
@@ -1,720 +0,0 @@
|
||||
# Go Vehicle Ingest Redesign Phase 1 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build the first production-capable Go runtime for GB32960, JT808, and Yutong MQTT ingestion with unified Kafka, TDengine, MySQL statistics, and Redis realtime state boundaries.
|
||||
|
||||
**Architecture:** Create a new single Go module at `go/vehicle-gateway` and migrate only the useful pieces from the existing `go/ingest-edge` and `go/vehicle-state` prototypes. The gateway produces unified envelopes to Kafka; independent consumers write TDengine history, MySQL daily metrics, and Redis realtime state.
|
||||
|
||||
**Tech Stack:** Go 1.26+, Kafka (`segmentio/kafka-go`), Redis (`redis/go-redis`), MySQL (`go-sql-driver/mysql`), TDengine official Go connector, MQTT (`eclipse/paho.mqtt.golang`), standard library TCP.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- Create: `go/vehicle-gateway/go.mod`
|
||||
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/*`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/*`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/yutongmqtt/*`
|
||||
- Create: `go/vehicle-gateway/internal/gateway/*`
|
||||
- Create: `go/vehicle-gateway/internal/identity/*`
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/*`
|
||||
- Create: `go/vehicle-gateway/internal/history/*`
|
||||
- Create: `go/vehicle-gateway/internal/stats/*`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/*`
|
||||
- Create: `go/vehicle-gateway/internal/observability/*`
|
||||
- Modify: `README.md`
|
||||
- Modify: `docs/target-architecture.md`
|
||||
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||
|
||||
The existing `go/ingest-edge` and `go/vehicle-state` directories are migration sources only. After phase 1 is verified, remove or mark them superseded.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Create Single Go Module
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/go.mod`
|
||||
- Create: `go/vehicle-gateway/internal/observability/logger.go`
|
||||
- Create: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
|
||||
- [ ] **Step 1: Create module manifest**
|
||||
|
||||
Add `go/vehicle-gateway/go.mod`:
|
||||
|
||||
```go
|
||||
module lingniu-vehicle-ingest/go/vehicle-gateway
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.1
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/taosdata/driver-go/v3 v3.8.1
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add logger helper**
|
||||
|
||||
Add `go/vehicle-gateway/internal/observability/logger.go`:
|
||||
|
||||
```go
|
||||
package observability
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
func NewLogger(service string) *slog.Logger {
|
||||
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
|
||||
return slog.New(handler).With("service", service)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add temporary gateway entrypoint**
|
||||
|
||||
Add `go/vehicle-gateway/cmd/gateway/main.go`:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("vehicle-gateway")
|
||||
logger.Info("vehicle gateway scaffold started")
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify scaffold builds**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go mod tidy
|
||||
go test ./...
|
||||
go build ./cmd/gateway
|
||||
```
|
||||
|
||||
Expected: all commands exit `0`.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Define Unified Envelope
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope.go`
|
||||
- Create: `go/vehicle-gateway/internal/envelope/envelope_test.go`
|
||||
|
||||
- [ ] **Step 1: Write envelope tests**
|
||||
|
||||
Add `go/vehicle-gateway/internal/envelope/envelope_test.go`:
|
||||
|
||||
```go
|
||||
package envelope
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "LNBVIN00000000001" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
|
||||
e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
|
||||
if got := e.VehicleKey(); got != "JT808:013307795425" {
|
||||
t.Fatalf("VehicleKey() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameEnvelopeEventIDStable(t *testing.T) {
|
||||
e := FrameEnvelope{
|
||||
Protocol: ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "013307795425",
|
||||
Sequence: 1,
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7e02000000ff7e",
|
||||
}
|
||||
a := e.StableEventID()
|
||||
b := e.StableEventID()
|
||||
if a == "" || a != b {
|
||||
t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests and confirm failure**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/envelope
|
||||
```
|
||||
|
||||
Expected: fail because `FrameEnvelope` is not defined.
|
||||
|
||||
- [ ] **Step 3: Implement envelope**
|
||||
|
||||
Add `go/vehicle-gateway/internal/envelope/envelope.go`:
|
||||
|
||||
```go
|
||||
package envelope
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Protocol string
|
||||
|
||||
const (
|
||||
ProtocolGB32960 Protocol = "GB32960"
|
||||
ProtocolJT808 Protocol = "JT808"
|
||||
ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
|
||||
)
|
||||
|
||||
type ParseStatus string
|
||||
|
||||
const (
|
||||
ParseOK ParseStatus = "OK"
|
||||
ParsePartial ParseStatus = "PARTIAL"
|
||||
ParseBadFrame ParseStatus = "BAD_FRAME"
|
||||
)
|
||||
|
||||
type FrameEnvelope struct {
|
||||
EventID string `json:"event_id"`
|
||||
TraceID string `json:"trace_id"`
|
||||
Protocol Protocol `json:"protocol"`
|
||||
MessageID string `json:"message_id"`
|
||||
Sequence uint16 `json:"sequence"`
|
||||
VIN string `json:"vin,omitempty"`
|
||||
VehicleKeyHint string `json:"vehicle_key,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
Parsed map[string]any `json:"parsed,omitempty"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
ParseStatus ParseStatus `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) VehicleKey() string {
|
||||
if key := strings.TrimSpace(e.VIN); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
|
||||
return key
|
||||
}
|
||||
if key := strings.TrimSpace(e.Phone); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
if key := strings.TrimSpace(e.DeviceID); key != "" {
|
||||
return string(e.Protocol) + ":" + key
|
||||
}
|
||||
return string(e.Protocol) + ":unknown"
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) StableEventID() string {
|
||||
if strings.TrimSpace(e.EventID) != "" {
|
||||
return e.EventID
|
||||
}
|
||||
input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
|
||||
e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
|
||||
sum := sha256.Sum256([]byte(input))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
|
||||
if e.EventID == "" {
|
||||
e.EventID = e.StableEventID()
|
||||
}
|
||||
if e.ParseStatus == "" {
|
||||
e.ParseStatus = ParseOK
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify tests pass**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/envelope
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Implement JT808 Frame and 0200 Parser
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/jt808/parser_test.go`
|
||||
|
||||
- [ ] **Step 1: Add sample frame tests**
|
||||
|
||||
Use the production sample provided in the thread:
|
||||
|
||||
```text
|
||||
7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E
|
||||
```
|
||||
|
||||
Expected assertions:
|
||||
|
||||
- message id is `0x0200`
|
||||
- phone keeps normalized BCD string `013307795425`
|
||||
- sequence is `1`
|
||||
- `fields.total_mileage_km` is parsed from additional item `0x01`
|
||||
- latitude, longitude, speed, direction, alarm, status, and device time are present
|
||||
|
||||
- [ ] **Step 2: Implement parser**
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- strip `0x7e` start/end delimiters
|
||||
- unescape `0x7d 0x02 -> 0x7e`
|
||||
- unescape `0x7d 0x01 -> 0x7d`
|
||||
- verify XOR checksum
|
||||
- parse 2011/2013 common header
|
||||
- parse BCD phone from 6 bytes and keep both raw and normalized values in `parsed.header`
|
||||
- parse 0200 fixed body
|
||||
- parse additional item list as `id,length,value_hex`
|
||||
- parse additional `0x01` as `total_mileage_km = uint32 / 10`
|
||||
|
||||
- [ ] **Step 3: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/protocol/jt808
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Implement GB32960 Frame and Data Unit Parser
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser.go`
|
||||
- Create: `go/vehicle-gateway/internal/protocol/gb32960/parser_test.go`
|
||||
|
||||
- [ ] **Step 1: Add parser tests**
|
||||
|
||||
Tests must cover:
|
||||
|
||||
- `##` frame boundary
|
||||
- command id
|
||||
- response flag
|
||||
- 17-byte VIN
|
||||
- encryption flag
|
||||
- payload length
|
||||
- BCC verification
|
||||
- realtime data command `0x02`
|
||||
- reissue data command `0x03`
|
||||
- data unit `0x01` vehicle status fields
|
||||
- data unit `0x05` position fields
|
||||
|
||||
- [ ] **Step 2: Implement parser**
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- parse header without allocating large temporary buffers
|
||||
- keep RAW as hex in envelope
|
||||
- write all recognized data units to `Parsed`
|
||||
- write core fields to `Fields`
|
||||
- unsupported data unit stays in `Parsed["unknown_units"]`
|
||||
|
||||
- [ ] **Step 3: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/protocol/gb32960
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Implement Kafka Sink
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink.go`
|
||||
- Create: `go/vehicle-gateway/internal/eventbus/kafka_sink_test.go`
|
||||
|
||||
- [ ] **Step 1: Add sink interface**
|
||||
|
||||
Create a sink interface:
|
||||
|
||||
```go
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Sink interface {
|
||||
PublishRaw(context.Context, envelope.FrameEnvelope) error
|
||||
PublishUnified(context.Context, envelope.FrameEnvelope) error
|
||||
Close() error
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Implement Kafka topic routing**
|
||||
|
||||
Rules:
|
||||
|
||||
- `GB32960 -> vehicle.raw.gb32960.v1`
|
||||
- `JT808 -> vehicle.raw.jt808.v1`
|
||||
- `YUTONG_MQTT -> vehicle.raw.yutong-mqtt.v1`
|
||||
- unified topic is `vehicle.event.unified.v1`
|
||||
- message key is `env.VehicleKey()`
|
||||
|
||||
- [ ] **Step 3: Verify routing with unit tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/eventbus
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Implement TDengine History Writer
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/history/schema.go`
|
||||
- Create: `go/vehicle-gateway/internal/history/writer.go`
|
||||
- Create: `go/vehicle-gateway/internal/history/writer_test.go`
|
||||
- Create: `go/vehicle-gateway/cmd/history-writer/main.go`
|
||||
|
||||
- [ ] **Step 1: Add schema bootstrap SQL**
|
||||
|
||||
Implement schema strings for:
|
||||
|
||||
- database `lingniu_vehicle_ts`
|
||||
- stable `raw_frames`
|
||||
- stable `vehicle_locations`
|
||||
- stable `vehicle_mileage_points`
|
||||
|
||||
- [ ] **Step 2: Implement writer**
|
||||
|
||||
Rules:
|
||||
|
||||
- `AppendRawFrame` always writes one raw row.
|
||||
- `AppendLocation` writes only when longitude and latitude exist.
|
||||
- `AppendMileagePoint` writes only when `total_mileage_km` exists.
|
||||
- child table name is deterministic hash of `protocol + vehicle_key`.
|
||||
- escape tag values.
|
||||
|
||||
- [ ] **Step 3: Use TDengine official driver**
|
||||
|
||||
Import WebSocket driver in command:
|
||||
|
||||
```go
|
||||
import _ "github.com/taosdata/driver-go/v3/taosWS"
|
||||
```
|
||||
|
||||
Default driver name:
|
||||
|
||||
```text
|
||||
TDENGINE_DRIVER=taosWS
|
||||
```
|
||||
|
||||
Short-term compatibility:
|
||||
|
||||
```text
|
||||
TDENGINE_DRIVER=taosSql
|
||||
```
|
||||
|
||||
Only use `taosSql` when ECS TDengine WebSocket is not available.
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/history
|
||||
go build ./cmd/history-writer
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Implement MySQL Daily Metric Writer
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/stats/schema.go`
|
||||
- Create: `go/vehicle-gateway/internal/stats/daily_metric.go`
|
||||
- Create: `go/vehicle-gateway/internal/stats/daily_metric_test.go`
|
||||
- Create: `go/vehicle-gateway/cmd/stat-writer/main.go`
|
||||
|
||||
- [ ] **Step 1: Add schema bootstrap**
|
||||
|
||||
Implement `vehicle_daily_metric` schema from the design spec.
|
||||
|
||||
- [ ] **Step 2: Add metric derivation tests**
|
||||
|
||||
Test cases:
|
||||
|
||||
- no `total_mileage_km` produces no metric
|
||||
- one sample produces:
|
||||
- `daily_mileage_km = 0`
|
||||
- `daily_total_mileage_km = sample`
|
||||
- later larger sample updates:
|
||||
- `latest_total_mileage_km`
|
||||
- `daily_mileage_km`
|
||||
- `daily_total_mileage_km`
|
||||
- out-of-order smaller sample updates:
|
||||
- `first_total_mileage_km`
|
||||
- `daily_mileage_km`
|
||||
|
||||
- [ ] **Step 3: Implement MySQL upsert**
|
||||
|
||||
Use one table and one idempotent upsert:
|
||||
|
||||
```sql
|
||||
INSERT INTO vehicle_daily_metric
|
||||
(vin, stat_date, protocol, metric_key, metric_value, metric_unit,
|
||||
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method)
|
||||
VALUES (?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
first_total_mileage_km = LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)),
|
||||
latest_total_mileage_km = GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)),
|
||||
metric_value = CASE
|
||||
WHEN metric_key = 'daily_mileage_km'
|
||||
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
- LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
|
||||
WHEN metric_key = 'daily_total_mileage_km'
|
||||
THEN GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
|
||||
ELSE VALUES(metric_value)
|
||||
END,
|
||||
sample_count = sample_count + 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/stats
|
||||
go build ./cmd/stat-writer
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Implement Redis Realtime API
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/realtime/repository.go`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/repository_test.go`
|
||||
- Create: `go/vehicle-gateway/internal/realtime/http.go`
|
||||
- Create: `go/vehicle-gateway/cmd/realtime-api/main.go`
|
||||
|
||||
- [ ] **Step 1: Add repository contract**
|
||||
|
||||
Methods:
|
||||
|
||||
- `Update(ctx, envelope.FrameEnvelope) error`
|
||||
- `GetMerged(ctx, vin string) (Snapshot, error)`
|
||||
- `GetProtocol(ctx, vin string, protocol envelope.Protocol) (Snapshot, error)`
|
||||
- `IsOnline(ctx, vin string) (OnlineStatus, error)`
|
||||
|
||||
- [ ] **Step 2: Implement merge logic**
|
||||
|
||||
Rules:
|
||||
|
||||
- update `vehicle:latest:{vin}:{protocol}`
|
||||
- update `vehicle:latest:{vin}` with newest fields
|
||||
- update `vehicle:online:{vin}`
|
||||
- update sorted set `vehicle:last_seen`
|
||||
|
||||
- [ ] **Step 3: Implement HTTP API**
|
||||
|
||||
Routes:
|
||||
|
||||
- `GET /api/realtime/vehicles/{vin}`
|
||||
- `GET /api/realtime/vehicles/{vin}/online`
|
||||
- `GET /api/realtime/vehicles/{vin}/protocols/{protocol}`
|
||||
|
||||
- [ ] **Step 4: Verify**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
go test ./internal/realtime
|
||||
go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Wire Gateway Runtime
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/internal/gateway/tcp_server.go`
|
||||
- Create: `go/vehicle-gateway/internal/gateway/mqtt_client.go`
|
||||
- Modify: `go/vehicle-gateway/cmd/gateway/main.go`
|
||||
|
||||
- [ ] **Step 1: Implement TCP server**
|
||||
|
||||
Rules:
|
||||
|
||||
- one goroutine per accepted connection
|
||||
- bounded max connections
|
||||
- read timeout and idle timeout
|
||||
- protocol-specific frame extractor
|
||||
- structured peer endpoint
|
||||
- graceful shutdown on SIGTERM
|
||||
|
||||
- [ ] **Step 2: Implement MQTT client**
|
||||
|
||||
Rules:
|
||||
|
||||
- connect with official production config from environment
|
||||
- subscribe configured topic list
|
||||
- convert each message into envelope
|
||||
- publish raw and unified events
|
||||
- reconnect with backoff
|
||||
|
||||
- [ ] **Step 3: Verify local JSON mode**
|
||||
|
||||
Run without Kafka:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
GB32960_TCP_ADDR=:132960 JT808_TCP_ADDR=:18080 go run ./cmd/gateway
|
||||
```
|
||||
|
||||
Expected: service starts and logs configured listeners.
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Docker and ECS Deployment
|
||||
|
||||
**Files:**
|
||||
- Create: `go/vehicle-gateway/Dockerfile`
|
||||
- Create: `deploy/portainer/docker-compose-go.yml`
|
||||
- Modify: `docs/operations/current-ecs-deployment.md`
|
||||
|
||||
- [ ] **Step 1: Add multi-stage Dockerfile**
|
||||
|
||||
Build all commands:
|
||||
|
||||
- `gateway`
|
||||
- `history-writer`
|
||||
- `stat-writer`
|
||||
- `realtime-api`
|
||||
|
||||
- [ ] **Step 2: Add Portainer compose**
|
||||
|
||||
Services:
|
||||
|
||||
- `go-vehicle-gateway`
|
||||
- `go-history-writer`
|
||||
- `go-stat-writer`
|
||||
- `go-realtime-api`
|
||||
|
||||
Each service must include:
|
||||
|
||||
- restart policy
|
||||
- memory limit
|
||||
- Kafka env
|
||||
- MySQL/TDengine/Redis env as needed
|
||||
- logging options
|
||||
|
||||
- [ ] **Step 3: Verify Linux build**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd go/vehicle-gateway
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/gateway
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/history-writer
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/stat-writer
|
||||
GOOS=linux GOARCH=amd64 go build ./cmd/realtime-api
|
||||
```
|
||||
|
||||
Expected: all commands exit `0`.
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Production Verification
|
||||
|
||||
**Files:**
|
||||
- Create: `docs/operations/go-vehicle-gateway-verification.md`
|
||||
|
||||
- [ ] **Step 1: Record test commands**
|
||||
|
||||
Document commands to verify:
|
||||
|
||||
- gateway process health
|
||||
- Kafka topic consumption
|
||||
- TDengine row counts
|
||||
- MySQL daily metric rows
|
||||
- Redis realtime lookup
|
||||
|
||||
- [ ] **Step 2: Validate real traffic**
|
||||
|
||||
Evidence required:
|
||||
|
||||
- one real 32960 VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||
- one real JT808 phone/VIN with RAW, location, mileage point, daily metric, Redis snapshot
|
||||
- one real Yutong MQTT VIN with RAW and Redis snapshot
|
||||
|
||||
- [ ] **Step 3: Keep old Java services until evidence is captured**
|
||||
|
||||
Only disable Java equivalents after the evidence file contains successful command outputs and timestamps.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review Checklist
|
||||
|
||||
- The plan creates one new Go module instead of extending scattered prototypes.
|
||||
- The plan covers GB32960, JT808, and Yutong MQTT ingress.
|
||||
- The plan covers Kafka, TDengine, MySQL, and Redis.
|
||||
- The plan includes 32960 and 808 daily mileage and daily total mileage.
|
||||
- The plan includes local and ECS verification.
|
||||
- The plan does not restore Xinda Push.
|
||||
- The plan keeps Java services untouched until Go evidence exists.
|
||||
@@ -1,481 +0,0 @@
|
||||
> **Superseded:** This 2026-06-23 DuckDB-based design is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine is the default hot history store,
|
||||
> `event-file-store` and Xinda Push have been removed from the production codebase.
|
||||
|
||||
# GB32960 Production Readiness Design
|
||||
|
||||
## Goal
|
||||
|
||||
Make the GB32960 ingestion, history, and analytics services production-ready:
|
||||
|
||||
- receive and parse GB32960 traffic quickly and accurately;
|
||||
- maximize VIN-to-platform/vendor-profile association through platform login and local mappings;
|
||||
- store complete RAW frames and queryable telemetry history with DuckDB;
|
||||
- support full frame replay from persisted RAW bytes;
|
||||
- compute real-time daily vehicle metrics and alarm timelines into MySQL;
|
||||
- reduce service coupling and improve cohesion inside each module.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Do not put database credentials in code, YAML files, tests, docs, or commits.
|
||||
- Do not make MySQL the source for full telemetry history. MySQL is for derived daily metrics and alarm timelines.
|
||||
- Do not require every private GB32960 extension to be perfect before storing data. RAW bytes must always be kept when a valid frame is received.
|
||||
|
||||
## Production Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
tcp["GB32960 TCP:32960"] --> ingest["gb32960-ingest-app"]
|
||||
mapping["local vin-platform-profile.jsonl"] --> ingest
|
||||
ingest --> kafkaEvent["Kafka vehicle.event.gb32960.v1"]
|
||||
ingest --> kafkaRaw["Kafka vehicle.raw.gb32960.v1"]
|
||||
kafkaEvent --> history["vehicle-history-app"]
|
||||
kafkaRaw --> history
|
||||
history --> duckdb["DuckDB hot history"]
|
||||
history --> raw["RAW frame archive"]
|
||||
kafkaEvent --> analytics["vehicle-analytics-app"]
|
||||
analytics --> mysql["MySQL daily stats + alarms"]
|
||||
```
|
||||
|
||||
## Service Boundaries
|
||||
|
||||
### 1. `gb32960-ingest-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Netty TCP accept, frame splitting, GB32960 decode, auth, ACK, and Kafka production.
|
||||
- Platform login handling and connection-local `platformAccount`.
|
||||
- VIN-to-platform/vendor-profile resolution.
|
||||
- Produce structured telemetry, session, alarm, diagnostics, and RAW envelope messages.
|
||||
|
||||
It must not:
|
||||
|
||||
- write DuckDB, MySQL, or local archive files;
|
||||
- run historical queries;
|
||||
- run daily statistics.
|
||||
|
||||
ACK policy:
|
||||
|
||||
- login/report-like GB32960 commands use durable ACK: ACK only after Kafka dispatch succeeds;
|
||||
- malformed bytes that cannot form a valid frame are dropped with diagnostics;
|
||||
- valid frames with partial private-block parse failures still go to Kafka with `parseStatus=PARTIAL`.
|
||||
|
||||
VIN/profile resolution priority:
|
||||
|
||||
1. active platform login on the current TCP channel;
|
||||
2. exact VIN entry from local mapping file;
|
||||
3. VIN prefix entry from local mapping file;
|
||||
4. configured platform extension rule;
|
||||
5. standard GB32960 profile.
|
||||
|
||||
Local mapping file format:
|
||||
|
||||
```jsonl
|
||||
{"vin":"LNXNEGRR2SR321390","platformAccount":"Hyundai","vendorProfile":"guangdong-fc","enabled":true}
|
||||
{"vinPrefix":"LNXNEGRR","platformAccount":"Hyundai","vendorProfile":"guangdong-fc","enabled":true}
|
||||
```
|
||||
|
||||
The mapping file is loaded at startup and can be reloaded by an actuator endpoint or file watcher in a later phase.
|
||||
|
||||
### 2. `vehicle-history-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume Kafka event and RAW topics.
|
||||
- Persist RAW bytes to an append-only archive.
|
||||
- Persist query indexes and compact telemetry points to DuckDB.
|
||||
- Serve high-performance query APIs:
|
||||
- full frame list by `vin + eventTime`;
|
||||
- telemetry fields by `vin + eventTime`;
|
||||
- one-frame replay by `rawArchiveUri`;
|
||||
- parse diagnostics and missing-profile troubleshooting.
|
||||
|
||||
It must not:
|
||||
|
||||
- listen on GB32960 TCP;
|
||||
- run MySQL statistics;
|
||||
- mutate daily business metrics.
|
||||
|
||||
### 3. `vehicle-analytics-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume Kafka event topic.
|
||||
- Maintain real-time daily stats in MySQL.
|
||||
- Maintain full alarm timelines in MySQL.
|
||||
- Expose metrics query APIs and operational health.
|
||||
|
||||
It must not:
|
||||
|
||||
- parse RAW frames for normal operation;
|
||||
- write DuckDB history;
|
||||
- own GB32960 TCP connectivity.
|
||||
|
||||
## Kafka Contract
|
||||
|
||||
Keep Protobuf `VehicleEnvelope`, but make the producer contract stricter:
|
||||
|
||||
- `eventId`: stable idempotency key, deterministic from protocol, VIN, command, event time, raw checksum, and sequence when available.
|
||||
- `vin`: required for vehicle frames; platform-only commands use `_platform` or empty plus `platformAccount`.
|
||||
- `eventTimeMs`: GB32960 data collection time when available.
|
||||
- `ingestTimeMs`: service receive time.
|
||||
- `metadata.platformAccount`: resolved platform account.
|
||||
- `metadata.vendorProfile`: selected parser profile.
|
||||
- `metadata.parseStatus`: `OK`, `PARTIAL`, or `FAILED`.
|
||||
- `metadata.parseErrorCode`: stable code for failures.
|
||||
- `raw_archive.data`: present on RAW topic only; event topic keeps `rawArchiveUri` pointer.
|
||||
- `telemetry_snapshot`: standard field projection for downstream services.
|
||||
|
||||
Consumer rule:
|
||||
|
||||
- History consumes both event and RAW topics.
|
||||
- Analytics consumes only event topic.
|
||||
- Consumer offset is committed only after the target store commit succeeds.
|
||||
- Reprocessing must be safe through deterministic ids and database upsert/ignore semantics.
|
||||
|
||||
## DuckDB History Design
|
||||
|
||||
Use DuckDB as the hot historical database, not a Parquet-rewrite sidecar.
|
||||
|
||||
DuckDB official guidance says row-by-row insert loops are inefficient for bulk loading, and Appender/batched writes are the intended high-throughput path. Therefore history writes use a single writer worker with bounded batches and one DuckDB connection.
|
||||
|
||||
### Tables
|
||||
|
||||
`gb32960_frame_index`
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS gb32960_frame_index (
|
||||
event_id VARCHAR PRIMARY KEY,
|
||||
vin VARCHAR NOT NULL,
|
||||
platform_account VARCHAR,
|
||||
vendor_profile VARCHAR,
|
||||
command VARCHAR NOT NULL,
|
||||
command_code INTEGER NOT NULL,
|
||||
event_time TIMESTAMP NOT NULL,
|
||||
event_time_ms BIGINT NOT NULL,
|
||||
ingest_time TIMESTAMP NOT NULL,
|
||||
ingest_time_ms BIGINT NOT NULL,
|
||||
partition_date DATE NOT NULL,
|
||||
raw_archive_uri VARCHAR NOT NULL,
|
||||
raw_checksum VARCHAR,
|
||||
raw_size_bytes BIGINT NOT NULL,
|
||||
parse_status VARCHAR NOT NULL,
|
||||
parse_error_code VARCHAR,
|
||||
block_count INTEGER NOT NULL,
|
||||
metadata_json VARCHAR NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS gb32960_frame_vin_time_idx
|
||||
ON gb32960_frame_index(vin, event_time_ms);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS gb32960_frame_raw_uri_idx
|
||||
ON gb32960_frame_index(raw_archive_uri);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS gb32960_frame_partition_time_idx
|
||||
ON gb32960_frame_index(partition_date, event_time_ms);
|
||||
```
|
||||
|
||||
`gb32960_telemetry_point`
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS gb32960_telemetry_point (
|
||||
event_id VARCHAR NOT NULL,
|
||||
vin VARCHAR NOT NULL,
|
||||
event_time TIMESTAMP NOT NULL,
|
||||
event_time_ms BIGINT NOT NULL,
|
||||
ingest_time TIMESTAMP NOT NULL,
|
||||
platform_account VARCHAR,
|
||||
vendor_profile VARCHAR,
|
||||
field_key VARCHAR NOT NULL,
|
||||
value_type VARCHAR NOT NULL,
|
||||
value_text VARCHAR NOT NULL,
|
||||
value_num DOUBLE,
|
||||
unit VARCHAR,
|
||||
quality VARCHAR NOT NULL,
|
||||
raw_archive_uri VARCHAR NOT NULL,
|
||||
PRIMARY KEY(event_id, field_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS gb32960_point_vin_field_time_idx
|
||||
ON gb32960_telemetry_point(vin, field_key, event_time_ms);
|
||||
```
|
||||
|
||||
`gb32960_alarm_frame`
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS gb32960_alarm_frame (
|
||||
event_id VARCHAR PRIMARY KEY,
|
||||
vin VARCHAR NOT NULL,
|
||||
event_time TIMESTAMP NOT NULL,
|
||||
event_time_ms BIGINT NOT NULL,
|
||||
level VARCHAR NOT NULL,
|
||||
active_bits_json VARCHAR NOT NULL,
|
||||
fault_codes_json VARCHAR NOT NULL,
|
||||
raw_archive_uri VARCHAR NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS gb32960_alarm_vin_time_idx
|
||||
ON gb32960_alarm_frame(vin, event_time_ms);
|
||||
```
|
||||
|
||||
### RAW Archive
|
||||
|
||||
RAW archive remains file/object based:
|
||||
|
||||
```text
|
||||
archive://yyyy/MM/dd/GB32960/{vin}/{event_id}.bin
|
||||
```
|
||||
|
||||
The archive writer verifies:
|
||||
|
||||
- stored byte length equals envelope `raw_archive.size_bytes`;
|
||||
- checksum matches;
|
||||
- URI is unique for the deterministic event id.
|
||||
|
||||
Frame replay path:
|
||||
|
||||
1. query `gb32960_frame_index` by `rawArchiveUri`;
|
||||
2. read RAW bytes from archive;
|
||||
3. resolve `platformAccount/vendorProfile` from index metadata;
|
||||
4. decode using current parser;
|
||||
5. return decoded frame plus original indexed parse diagnostics.
|
||||
|
||||
## Query APIs
|
||||
|
||||
Keep existing endpoints but back them by the new DuckDB tables:
|
||||
|
||||
- `GET /api/event-history/gb32960/frames`
|
||||
- full frame query by VIN/time.
|
||||
- `GET /api/event-history/gb32960/frame`
|
||||
- one RAW frame replay by `rawArchiveUri`.
|
||||
- `GET /api/event-history/gb32960/telemetry-snapshots`
|
||||
- compact telemetry snapshot by VIN/time.
|
||||
- `GET /api/event-history/gb32960/telemetry-fields`
|
||||
- field projection optimized by `gb32960_telemetry_point`.
|
||||
- `GET /api/event-history/gb32960/diagnostics`
|
||||
- parse status and missing-profile investigation by VIN/time.
|
||||
|
||||
Time semantics:
|
||||
|
||||
- `eventTime` is business query time.
|
||||
- `ingestTime` is kept for operational replay, late-arrival detection, and Kafka lag investigation.
|
||||
- Dates without timezone are interpreted in `Asia/Shanghai`.
|
||||
|
||||
## MySQL Analytics Design
|
||||
|
||||
Credentials are supplied only by environment variables:
|
||||
|
||||
```text
|
||||
MYSQL_HOST
|
||||
MYSQL_PORT
|
||||
MYSQL_DATABASE
|
||||
MYSQL_USERNAME
|
||||
MYSQL_PASSWORD
|
||||
```
|
||||
|
||||
Daily derived metrics are stored in the common metric table, not in a
|
||||
protocol-specific daily-mileage table.
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS vehicle_stat_metric (
|
||||
vin VARCHAR(64) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,6) NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (vin, stat_date, metric_key)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
```
|
||||
|
||||
`vehicle_alarm_timeline`
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS vehicle_alarm_timeline (
|
||||
alarm_id VARCHAR(128) NOT NULL,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
alarm_key VARCHAR(128) NOT NULL,
|
||||
level VARCHAR(32) NOT NULL,
|
||||
start_time DATETIME(3) NOT NULL,
|
||||
end_time DATETIME(3),
|
||||
last_seen_time DATETIME(3) NOT NULL,
|
||||
start_raw_archive_uri VARCHAR(512),
|
||||
last_raw_archive_uri VARCHAR(512),
|
||||
status VARCHAR(16) NOT NULL,
|
||||
details_json JSON,
|
||||
updated_at DATETIME(3) NOT NULL,
|
||||
PRIMARY KEY (alarm_id),
|
||||
KEY vehicle_alarm_vin_time_idx (vin, start_time),
|
||||
KEY vehicle_alarm_open_idx (vin, status, alarm_key)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
```
|
||||
|
||||
`vehicle_stat_event_dedup`
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS vehicle_stat_event_dedup (
|
||||
event_id VARCHAR(128) NOT NULL,
|
||||
consumer_group VARCHAR(128) NOT NULL,
|
||||
processed_at DATETIME(3) NOT NULL,
|
||||
PRIMARY KEY (event_id, consumer_group)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
```
|
||||
|
||||
Writes use `INSERT ... ON DUPLICATE KEY UPDATE` so replayed Kafka messages update the same daily row instead of creating duplicates.
|
||||
|
||||
## Daily Metric Algorithms
|
||||
|
||||
All daily statistics are keyed by `eventTime` in `Asia/Shanghai`.
|
||||
|
||||
### Total Mileage
|
||||
|
||||
Use the latest valid `totalMileageKm` seen for the day.
|
||||
|
||||
Validation:
|
||||
|
||||
- reject negative values;
|
||||
- reject impossible jumps based on configurable max speed and elapsed time;
|
||||
- keep the sample as diagnostic if rejected.
|
||||
|
||||
### Daily Mileage
|
||||
|
||||
JT808 daily mileage uses the GPS total mileage reported in location additional
|
||||
information. The local-day minimum and maximum GPS total mileage values rewrite
|
||||
the same metric row, so ordered streaming and historical replay use one
|
||||
calculation path.
|
||||
|
||||
```text
|
||||
dailyMileage = today.maxTotalMileage - today.minTotalMileage
|
||||
calculationMethod = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
The result is stored in `vehicle_stat_metric` with
|
||||
`metric_key=daily_mileage_km`. There is no JT808-specific daily mileage table,
|
||||
no Redis mileage state, and no previous-day baseline calculation.
|
||||
|
||||
### Daily Power Consumption
|
||||
|
||||
Preferred:
|
||||
|
||||
- vendor cumulative power-consumption field if available.
|
||||
|
||||
Fallback:
|
||||
|
||||
```text
|
||||
deltaKwh = sum(max(0, totalVoltageV * totalCurrentA) * deltaSeconds / 3600000)
|
||||
```
|
||||
|
||||
The sign convention is configurable per vendor profile because GB32960 vehicle current may be positive or negative depending on charging/discharging representation.
|
||||
|
||||
### Daily Hydrogen Consumption
|
||||
|
||||
Use a tiered market-practical algorithm instead of a single fixed formula:
|
||||
|
||||
1. `DIRECT_COUNTER`: vendor cumulative hydrogen consumption or remaining hydrogen mass delta.
|
||||
2. `PVT_MASS_DELTA`: pressure/temperature/tank-volume mass estimate when storage data is available.
|
||||
3. `FUEL_CELL_ENERGY_MODEL`: integrate fuel-cell electric output and divide by calibrated efficiency.
|
||||
4. `UNKNOWN`: keep null when required signals are missing.
|
||||
|
||||
The PVT approach follows the industry pattern behind compressed hydrogen consumption tests, where pressure, temperature, and tank volume can be used to estimate hydrogen mass. The implementation must expose calibration constants per vehicle model/profile:
|
||||
|
||||
```json
|
||||
{
|
||||
"vinPrefix": "LNXNEGRR",
|
||||
"tankVolumeLiter": 140,
|
||||
"hydrogenModel": "PVT_MASS_DELTA",
|
||||
"fuelCellEfficiency": 0.52,
|
||||
"currentDischargeSign": "POSITIVE"
|
||||
}
|
||||
```
|
||||
|
||||
### Daily Hydrogen Consumption Rate
|
||||
|
||||
```text
|
||||
dailyHydrogenKgPer100km = dailyHydrogenKg / dailyMileageKm * 100
|
||||
```
|
||||
|
||||
If mileage is missing or less than the configured minimum distance, keep the rate null and set quality to `INSUFFICIENT_DISTANCE`.
|
||||
|
||||
### Alarm Timeline
|
||||
|
||||
For each telemetry frame:
|
||||
|
||||
- derive active alarm keys from alarm bits and fault code arrays;
|
||||
- open a timeline row when a key first appears;
|
||||
- update `last_seen_time` while it remains active;
|
||||
- close open alarms when the key disappears after a configurable grace window;
|
||||
- keep start/end raw archive URIs for replay.
|
||||
|
||||
## Reliability and Observability
|
||||
|
||||
Required metrics:
|
||||
|
||||
- TCP active connections;
|
||||
- frames received per command;
|
||||
- decode failures by code;
|
||||
- missing platform login rejections;
|
||||
- vendor profile resolution source;
|
||||
- Kafka send latency and failures;
|
||||
- history write batch size, latency, failure count;
|
||||
- DuckDB query latency by endpoint;
|
||||
- MySQL upsert latency and failure count;
|
||||
- consumer lag by group/topic/partition;
|
||||
- late sample count by VIN/date.
|
||||
|
||||
Required logs:
|
||||
|
||||
- one structured line for every dropped or rejected valid frame;
|
||||
- no per-frame INFO spam for successful reports after first frame per VIN/profile unless debug is enabled;
|
||||
- include `vin`, `platformAccount`, `command`, `eventId`, `reasonCode`, and `peer`.
|
||||
|
||||
## Migration Plan
|
||||
|
||||
Phase 1: History foundation
|
||||
|
||||
- Replace Parquet rewrite store with DuckDB hot tables and batch writer.
|
||||
- Keep RAW bytes archive.
|
||||
- Move existing history endpoints onto the new store.
|
||||
- Add diagnostic endpoint for missing VIN/profile cases.
|
||||
|
||||
Phase 2: Ingest association and diagnostics
|
||||
|
||||
- Add local `vin-platform-profile.jsonl` loader.
|
||||
- Use resolved profile before private block parsing.
|
||||
- Add structured parse/reject diagnostics to Kafka metadata and logs.
|
||||
- Add tests for platform login, mapping fallback, and partial private parsing.
|
||||
|
||||
Phase 3: MySQL analytics
|
||||
|
||||
- Add MySQL repository and migrations.
|
||||
- Implement daily stat accumulator with idempotent upsert.
|
||||
- Implement alarm timeline open/update/close logic.
|
||||
- Add integration tests using Testcontainers or a local MySQL profile.
|
||||
|
||||
Phase 4: Production hardening
|
||||
|
||||
- Add backfill/replay commands from DuckDB RAW archive to analytics.
|
||||
- Add retention/export from DuckDB to Parquet for cold storage.
|
||||
- Add dashboards and operational runbook.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A valid GB32960 frame received on port 32960 appears in Kafka with RAW bytes or RAW URI.
|
||||
- A valid frame is queryable by `vin + eventTime` from history.
|
||||
- A returned frame can be replayed from `rawArchiveUri` and decoded with the stored profile.
|
||||
- A VIN with only local mapping, and no current platform login, can still use the correct private parser profile.
|
||||
- A private block parse failure does not drop the frame.
|
||||
- Daily MySQL rows update in real time for mileage, total mileage, power consumption, hydrogen consumption, hydrogen rate, and sample metadata.
|
||||
- Alarm timeline rows preserve full alarm start, update, and close history.
|
||||
- Replaying the same Kafka event does not duplicate daily stats or alarm rows.
|
||||
- No secret appears in committed files.
|
||||
|
||||
## References
|
||||
|
||||
- DuckDB Appender: https://duckdb.org/docs/current/data/appender
|
||||
- DuckDB INSERT performance guidance: https://duckdb.org/docs/current/data/insert
|
||||
- MySQL `INSERT ... ON DUPLICATE KEY UPDATE`: https://dev.mysql.com/doc/refman/9.7/en/insert-on-duplicate.html
|
||||
- SAE J2572 overview: https://h2tools.org/fuel-cell-codes-and-standards/sae-j2572-measuring-exhaust-emissions-energy-consumption-and-range
|
||||
@@ -1,411 +0,0 @@
|
||||
> **Superseded:** This 2026-06-23 split design is historical context.
|
||||
> Use `docs/target-architecture.md` and
|
||||
> `docs/superpowers/specs/2026-06-29-vehicle-ingest-redesign.md` for the
|
||||
> current production architecture: TDengine is the default hot history store,
|
||||
> `sink-archive` owns raw bytes, and `event-file-store` is optional
|
||||
> compatibility only.
|
||||
|
||||
# GB32960 Service Split Design
|
||||
|
||||
Date: 2026-06-23
|
||||
|
||||
## Goal
|
||||
|
||||
Split the current all-in-one vehicle ingest runtime into lower-coupled, higher-cohesion services. The first production slice focuses on GB/T 32960 because it is the active high-pressure path: persistent TCP connections, protocol ACKs, raw frame volume, history queries, and downstream analytics are currently assembled in one `bootstrap-all` process.
|
||||
|
||||
The target outcome is:
|
||||
|
||||
- Keep GB32960 ingest reliable and fast.
|
||||
- Move disk writes and history queries out of the protocol process.
|
||||
- Move statistics and analysis out of the protocol process.
|
||||
- Use Kafka as the durable boundary between ingest, history, and analytics.
|
||||
- Preserve `bootstrap-all` during migration as a local development and rollback entry point.
|
||||
|
||||
## Decision
|
||||
|
||||
Use ACK semantics A:
|
||||
|
||||
> A GB32960 frame is ACKed successfully after the protocol service parses/accepts it and writes the required Kafka message(s) successfully.
|
||||
|
||||
The protocol service must not wait for raw archive, DuckDB/Parquet writes, snapshot generation, or statistics computation before sending the GB32960 response frame.
|
||||
|
||||
## Service Boundaries
|
||||
|
||||
### `gb32960-ingest-app`
|
||||
|
||||
Purpose: own GB32960 connection handling and Kafka production.
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Listen on the GB32960 TCP port.
|
||||
- Handle platform login, VIN authorization, TLS if enabled, and session state required for protocol responses.
|
||||
- Decode GB32960 frames and apply vendor extension parsing.
|
||||
- Produce raw frame records and normalized event records to Kafka.
|
||||
- ACK successful reports only after Kafka production succeeds.
|
||||
- Publish malformed or rejected frames to a DLQ when possible.
|
||||
- Expose operational health and metrics for connections, decode rate, Kafka latency, ACK latency, and DLQ count.
|
||||
|
||||
Non-responsibilities:
|
||||
|
||||
- No local raw archive writes.
|
||||
- No DuckDB or Parquet history writes.
|
||||
- No snapshot query API.
|
||||
- No vehicle statistics or analysis.
|
||||
- No long-running analytical calculations.
|
||||
|
||||
Initial module dependencies:
|
||||
|
||||
- `ingest-api`
|
||||
- `ingest-core`
|
||||
- `ingest-codec-common`
|
||||
- `session-core`
|
||||
- `protocol-gb32960`
|
||||
- `sink-kafka`
|
||||
- `observability`
|
||||
|
||||
### `vehicle-history-app`
|
||||
|
||||
Purpose: own raw storage, historical index storage, and history query APIs.
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume GB32960 raw records and normalized event records from Kafka.
|
||||
- Write raw `.bin` payloads through `sink-archive`.
|
||||
- Write queryable event indexes through `event-file-store`.
|
||||
- Provide APIs for raw frame lookup, decoded frame lookup, history query, and snapshot/source-frame lookup.
|
||||
- Rebuild or backfill the DuckDB sidecar index from stored Parquet files when needed.
|
||||
- Track consumer lag, write latency, archive failures, index failures, and query latency.
|
||||
|
||||
Non-responsibilities:
|
||||
|
||||
- No TCP protocol listener.
|
||||
- No GB32960 ACK decisions.
|
||||
- No online statistics computation.
|
||||
- No direct coupling to `gb32960-ingest-app` internals.
|
||||
|
||||
Initial module dependencies:
|
||||
|
||||
- `ingest-api`
|
||||
- `sink-kafka`
|
||||
- `sink-archive`
|
||||
- `event-file-store`
|
||||
- `event-history-service`
|
||||
- `protocol-gb32960` only for raw-frame decode/query support
|
||||
- `observability`
|
||||
|
||||
### `vehicle-analytics-app`
|
||||
|
||||
Purpose: own vehicle state, daily statistics, alarms, and later analytics.
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume normalized vehicle event records from Kafka.
|
||||
- Maintain latest vehicle state if enabled.
|
||||
- Compute daily vehicle statistics and alarm-derived metrics.
|
||||
- Persist analytical outputs to the selected backend.
|
||||
- Support reprocessing from Kafka offsets when analytical logic changes.
|
||||
- Track consumer lag, per-VIN processing latency, aggregation failures, and output write latency.
|
||||
|
||||
Non-responsibilities:
|
||||
|
||||
- No protocol listener.
|
||||
- No GB32960 ACK decisions.
|
||||
- No raw archive writes.
|
||||
- No normal-path dependency on raw archive or history APIs.
|
||||
|
||||
Initial module dependencies:
|
||||
|
||||
- `ingest-api`
|
||||
- `sink-kafka`
|
||||
- `vehicle-state-service`
|
||||
- `vehicle-stat-service`
|
||||
- `observability`
|
||||
|
||||
## Kafka Contract
|
||||
|
||||
Kafka is the service boundary. Messages must be stable enough that `vehicle-history-app` and `vehicle-analytics-app` do not depend on protocol service internals.
|
||||
|
||||
### Topics
|
||||
|
||||
Use versioned topics:
|
||||
|
||||
- `vehicle.raw.gb32960.v1`
|
||||
- `vehicle.event.gb32960.v1`
|
||||
- `vehicle.dlq.gb32960.v1`
|
||||
|
||||
The current topic names under `lingniu.ingest.sink.kafka.topics` can remain temporarily for compatibility, but the split apps should converge on versioned topic names.
|
||||
|
||||
### Keys
|
||||
|
||||
Use VIN as the Kafka key whenever a VIN is available.
|
||||
|
||||
Benefits:
|
||||
|
||||
- Preserves per-vehicle ordering within a partition.
|
||||
- Lets history and analytics scale by partition.
|
||||
- Keeps stateful analytics simpler.
|
||||
|
||||
When VIN is unavailable, use a stable fallback key such as platform account plus peer address plus receive time bucket.
|
||||
|
||||
### Raw Record
|
||||
|
||||
Topic: `vehicle.raw.gb32960.v1`
|
||||
|
||||
Minimum fields:
|
||||
|
||||
- `schemaVersion`
|
||||
- `protocol = GB32960`
|
||||
- `eventId`
|
||||
- `receiveTime`
|
||||
- `eventTime` if decoded
|
||||
- `peer`
|
||||
- `platformAccount`
|
||||
- `vin` if decoded
|
||||
- `command`
|
||||
- `protocolVersion`
|
||||
- `rawBytes`
|
||||
- `checksumStatus`
|
||||
- `decodeStatus`
|
||||
- `metadata`
|
||||
|
||||
Consumers:
|
||||
|
||||
- `vehicle-history-app` writes archive `.bin` files and raw-frame query indexes.
|
||||
- Future replay tools can regenerate normalized events from raw records.
|
||||
|
||||
### Normalized Event Record
|
||||
|
||||
Topic: `vehicle.event.gb32960.v1`
|
||||
|
||||
Minimum fields:
|
||||
|
||||
- `schemaVersion`
|
||||
- `protocol = GB32960`
|
||||
- `eventId`
|
||||
- `sourceRawEventId`
|
||||
- `receiveTime`
|
||||
- `eventTime`
|
||||
- `platformAccount`
|
||||
- `vin`
|
||||
- `command`
|
||||
- `eventType`
|
||||
- `normalizedFields`
|
||||
- `vendorFields`
|
||||
- `alarmFields`
|
||||
- `metadata`
|
||||
|
||||
Consumers:
|
||||
|
||||
- `vehicle-history-app` writes queryable history records.
|
||||
- `vehicle-analytics-app` updates state and statistics.
|
||||
|
||||
### DLQ Record
|
||||
|
||||
Topic: `vehicle.dlq.gb32960.v1`
|
||||
|
||||
Minimum fields:
|
||||
|
||||
- `schemaVersion`
|
||||
- `stage`
|
||||
- `errorCode`
|
||||
- `errorMessage`
|
||||
- `receiveTime`
|
||||
- `peer`
|
||||
- `platformAccount` if known
|
||||
- `vin` if known
|
||||
- `rawBytes` when available
|
||||
- `metadata`
|
||||
|
||||
Typical stages:
|
||||
|
||||
- `FRAME_DECODE`
|
||||
- `BODY_PARSE`
|
||||
- `AUTH`
|
||||
- `KAFKA_PRODUCE`
|
||||
- `ACK_WRITE`
|
||||
|
||||
## ACK and Failure Semantics
|
||||
|
||||
### Success Path
|
||||
|
||||
1. Receive TCP bytes.
|
||||
2. Decode a complete GB32960 frame.
|
||||
3. Validate checksum/auth/session rules.
|
||||
4. Build raw and normalized Kafka records.
|
||||
5. Produce required Kafka records successfully.
|
||||
6. Send GB32960 success ACK.
|
||||
|
||||
### Kafka Produce Failure
|
||||
|
||||
If required Kafka production fails, the service must not send a success ACK.
|
||||
|
||||
Allowed behavior:
|
||||
|
||||
- Retry within a bounded timeout.
|
||||
- Return a GB32960 failure response when the protocol allows it.
|
||||
- Close the channel after repeated produce failures.
|
||||
- Emit local error logs and metrics.
|
||||
|
||||
Do not silently ACK frames that have not crossed the Kafka durability boundary.
|
||||
|
||||
### History or Analytics Failure
|
||||
|
||||
Failures in `vehicle-history-app` or `vehicle-analytics-app` must not affect GB32960 ACKs. They are handled by Kafka offset retry, DLQ, operational alerting, and backfill/replay.
|
||||
|
||||
## Current Coupling to Remove
|
||||
|
||||
`bootstrap-all` currently assembles protocol modules, Kafka producer/consumer, archive sink, event file store, history API, vehicle state, vehicle statistics, command gateway, and multiple inbound protocols in one application.
|
||||
|
||||
The split should remove these production couplings:
|
||||
|
||||
- Protocol runtime directly containing archive and event-file-store writers.
|
||||
- Protocol runtime directly containing statistics processors.
|
||||
- History query API sharing the same process as TCP ingest.
|
||||
- Kafka consumer workers living in the same all-in-one app as protocol listeners.
|
||||
- Operational scaling tied to one JVM for ingest, storage, queries, and analytics.
|
||||
|
||||
## Migration Plan
|
||||
|
||||
### Phase 1: Add Split App Entrypoints
|
||||
|
||||
Add new app modules:
|
||||
|
||||
- `modules/apps/gb32960-ingest-app`
|
||||
- `modules/apps/vehicle-history-app`
|
||||
- `modules/apps/vehicle-analytics-app`
|
||||
|
||||
Keep `modules/apps/bootstrap-all` unchanged as the fallback runtime.
|
||||
|
||||
### Phase 2: GB32960 Ingest App
|
||||
|
||||
Create a production profile for `gb32960-ingest-app`:
|
||||
|
||||
- Enable `protocol-gb32960`.
|
||||
- Enable Kafka producer.
|
||||
- Disable local archive.
|
||||
- Disable event-file-store.
|
||||
- Disable history API.
|
||||
- Disable vehicle state/stat services.
|
||||
- Disable unrelated protocols.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- GB32960 TCP port starts.
|
||||
- Platform login works.
|
||||
- Realtime report frames are parsed.
|
||||
- Kafka records are produced with VIN keys.
|
||||
- ACK is sent only after Kafka success.
|
||||
|
||||
### Phase 3: History App
|
||||
|
||||
Create `vehicle-history-app`:
|
||||
|
||||
- Enable Kafka consumer.
|
||||
- Enable archive sink.
|
||||
- Enable event-file-store.
|
||||
- Enable event-history HTTP API.
|
||||
- Disable protocol listeners.
|
||||
- Disable analytics processors.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- Consumes `vehicle.raw.gb32960.v1`.
|
||||
- Writes raw `.bin` files.
|
||||
- Consumes `vehicle.event.gb32960.v1`.
|
||||
- Writes Parquet/DuckDB indexes.
|
||||
- Queries can locate raw frames and decoded snapshots.
|
||||
|
||||
### Phase 4: Analytics App
|
||||
|
||||
Create `vehicle-analytics-app`:
|
||||
|
||||
- Enable Kafka consumer.
|
||||
- Enable vehicle state if a state backend is configured.
|
||||
- Enable vehicle statistics.
|
||||
- Disable protocol listeners.
|
||||
- Disable archive and event-file-store.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- Consumes `vehicle.event.gb32960.v1`.
|
||||
- Maintains expected per-VIN state/stat outputs.
|
||||
- Can be restarted and resume from Kafka offsets.
|
||||
- Does not affect ingest ACK latency.
|
||||
|
||||
### Phase 5: Parallel Run and Cutover
|
||||
|
||||
Run old and new paths in parallel for a bounded validation window.
|
||||
|
||||
Compare:
|
||||
|
||||
- Received frame count.
|
||||
- Kafka produced record count.
|
||||
- Raw archive file count.
|
||||
- Unique VIN count.
|
||||
- History query count and sample query correctness.
|
||||
- Analytics output count and sample values.
|
||||
- GB32960 ACK latency before and after split.
|
||||
|
||||
Cut over only after counts and sample queries match within the agreed tolerance.
|
||||
|
||||
## Operational Guidance
|
||||
|
||||
Scale services independently:
|
||||
|
||||
- Scale `gb32960-ingest-app` by connection count and Kafka produce latency.
|
||||
- Scale `vehicle-history-app` by disk throughput, consumer lag, and query latency.
|
||||
- Scale `vehicle-analytics-app` by consumer lag and computation latency.
|
||||
|
||||
Monitor:
|
||||
|
||||
- Kafka producer error rate and p99 produce latency.
|
||||
- ACK p99 latency.
|
||||
- TCP active connections.
|
||||
- Decode failures and DLQ counts.
|
||||
- Consumer lag per group.
|
||||
- Archive write latency and failure rate.
|
||||
- Event-file-store flush latency and failure rate.
|
||||
- Analytics processing latency.
|
||||
|
||||
## Risks and Mitigations
|
||||
|
||||
Risk: Kafka outage blocks GB32960 ACKs.
|
||||
|
||||
Mitigation: bounded producer retry, clear failure ACK/close behavior, Kafka cluster monitoring, and optional local emergency spool only as a later explicit design.
|
||||
|
||||
Risk: Two Kafka topics for raw and normalized events can diverge.
|
||||
|
||||
Mitigation: include `eventId` and `sourceRawEventId`, produce records transactionally if required, or publish a single envelope containing both raw and normalized sections in the first implementation if transaction support is not ready.
|
||||
|
||||
Risk: History app reprocessing duplicates archive/index records.
|
||||
|
||||
Mitigation: deterministic archive keys and idempotent index writes keyed by `eventId` or `rawArchiveUri`.
|
||||
|
||||
Risk: Analytics logic changes require backfill.
|
||||
|
||||
Mitigation: keep raw and normalized topics with sufficient retention; allow analytics consumers to reset offsets or run a backfill group.
|
||||
|
||||
Risk: Query APIs accidentally depend on ingest internals.
|
||||
|
||||
Mitigation: history APIs should depend on `event-file-store`, `sink-archive`, and decode libraries only, not `gb32960-ingest-app`.
|
||||
|
||||
## Open Decisions
|
||||
|
||||
These are intentionally left for implementation planning:
|
||||
|
||||
- Whether Kafka production uses two independent sends or a transaction for raw + normalized records.
|
||||
- Exact protobuf/JSON schema shape for `vehicle.raw.gb32960.v1` and `vehicle.event.gb32960.v1`.
|
||||
- Whether latest snapshot state belongs only to `vehicle-analytics-app` or is also materialized by `vehicle-history-app` for query convenience.
|
||||
- Initial Kafka partition count and retention duration.
|
||||
- Whether command downlink stays with `command-gateway` or gets a separate command service later.
|
||||
|
||||
## Completion Criteria
|
||||
|
||||
The split is complete when:
|
||||
|
||||
- `gb32960-ingest-app`, `vehicle-history-app`, and `vehicle-analytics-app` are separate runnable app modules.
|
||||
- `gb32960-ingest-app` can receive live GB32960 data and ACK after Kafka success.
|
||||
- `vehicle-history-app` can consume Kafka and provide raw/history/snapshot query APIs.
|
||||
- `vehicle-analytics-app` can consume Kafka and compute state/stat outputs.
|
||||
- `bootstrap-all` remains available for development and rollback.
|
||||
- Verification covers build, app startup, Kafka production/consumption, raw archive writes, history queries, analytics outputs, and ACK latency.
|
||||
@@ -1,621 +0,0 @@
|
||||
# Vehicle Ingest Redesign Design
|
||||
|
||||
## Background
|
||||
|
||||
Current GB32960 and JT808 ingestion can receive, decode, publish, and query data, but the responsibilities are split across several partially overlapping paths:
|
||||
|
||||
- Netty handlers produce `RawFrame` and the dispatcher emits both raw archive events and normalized vehicle events.
|
||||
- Kafka envelopes carry raw archive references, while raw bytes are stored separately by the archive sink.
|
||||
- GB32960 historical queries use raw archive records plus file replay decoding.
|
||||
- JT808 location history is materialized separately into TDengine.
|
||||
- DuckDB/Parquet, local raw archive, Kafka, and TDengine each own part of the story, so query and durability semantics are hard to reason about.
|
||||
|
||||
The new design intentionally keeps useful protocol parsers and app deployment knowledge, but replaces the ingestion, durability, storage, and query boundaries with a first-principles model.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Store every original inbound protocol frame, including malformed frames when bytes are available.
|
||||
2. Query historical uploaded data for GB32960 and JT808 by vehicle, protocol, message type, and time range.
|
||||
3. Extend to new fields, protocol messages, and vendor profiles without rewriting the core pipeline.
|
||||
4. Sustain production load for 10,000 vehicles and 1,000 concurrent historical query requests per second.
|
||||
5. Make ACK and durability rules explicit enough that an acknowledged frame is not silently lost.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Do not make Kafka the long-term historical query database.
|
||||
- Do not store large raw byte arrays directly in Kafka topics.
|
||||
- Do not keep DuckDB/Parquet as the production hot query store for GB32960 and JT808.
|
||||
- Do not build one generic API that hides protocol-specific semantics. Shared internals are preferred; external APIs may remain protocol-aware.
|
||||
|
||||
## Recommended Architecture
|
||||
|
||||
Use `Raw Archive + Kafka + TDengine` as the core architecture.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
GB["GB32960 TCP"] --> Edge["Protocol Edge"]
|
||||
JT["JT808 TCP"] --> Edge
|
||||
Edge --> RawStore["Raw Archive"]
|
||||
RawStore --> RawTopic["Kafka vehicle.raw-frame.v1"]
|
||||
Edge --> Decode["Protocol Decoder"]
|
||||
Decode --> FactTopic["Kafka vehicle.decoded-fact.v1"]
|
||||
RawTopic --> Writer["History Writer"]
|
||||
FactTopic --> Writer
|
||||
Writer --> TD["TDengine"]
|
||||
Writer --> Redis["Realtime State"]
|
||||
API["History API"] --> TD
|
||||
API --> RawStore
|
||||
RealtimeAPI["Realtime API"] --> Redis
|
||||
```
|
||||
|
||||
The architecture has three facts:
|
||||
|
||||
- `RawFrameFact`: one row per inbound frame, whether parsing succeeds or fails.
|
||||
- `DecodedFact`: zero or more normalized facts derived from a raw frame.
|
||||
- `RawArchiveObject`: immutable raw bytes addressed by `raw_uri`.
|
||||
|
||||
The raw archive is the forensic source of truth. TDengine is the query source of truth. Kafka is the decoupling and replay pipe.
|
||||
|
||||
## Module Boundaries
|
||||
|
||||
### Protocol Edge Apps
|
||||
|
||||
Apps:
|
||||
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Accept TCP connections.
|
||||
- Split frames.
|
||||
- Perform protocol-level validation, authentication, session binding, and ACK.
|
||||
- Create `RawFrameFact`.
|
||||
- Persist raw bytes through `RawArchiveWriter`.
|
||||
- Publish raw frame fact to Kafka.
|
||||
- Decode frames and publish decoded facts.
|
||||
|
||||
They do not query history, export data, or write business tables directly.
|
||||
|
||||
### Ingest Core
|
||||
|
||||
New module:
|
||||
|
||||
- `modules/core/ingest-facts`
|
||||
|
||||
Core types:
|
||||
|
||||
```java
|
||||
public record RawFrameFact(
|
||||
String frameId,
|
||||
ProtocolId protocol,
|
||||
String vehicleKey,
|
||||
String vin,
|
||||
String phone,
|
||||
int messageId,
|
||||
int subType,
|
||||
Instant eventTime,
|
||||
Instant receivedAt,
|
||||
String peer,
|
||||
String rawUri,
|
||||
String checksum,
|
||||
long rawSizeBytes,
|
||||
ParseStatus parseStatus,
|
||||
String parseError,
|
||||
Map<String, String> metadata
|
||||
) {}
|
||||
```
|
||||
|
||||
```java
|
||||
public sealed interface DecodedFact permits
|
||||
DecodedFact.Location,
|
||||
DecodedFact.Realtime,
|
||||
DecodedFact.Alarm,
|
||||
DecodedFact.Session,
|
||||
DecodedFact.Register,
|
||||
DecodedFact.Extension {
|
||||
|
||||
String factId();
|
||||
String frameId();
|
||||
ProtocolId protocol();
|
||||
String vehicleKey();
|
||||
String vin();
|
||||
Instant eventTime();
|
||||
Instant receivedAt();
|
||||
String rawUri();
|
||||
Map<String, String> metadata();
|
||||
}
|
||||
```
|
||||
|
||||
`vehicleKey` is the stable partition key:
|
||||
|
||||
- GB32960: VIN.
|
||||
- JT808: resolved VIN when known, otherwise `jt808:<phone>`.
|
||||
- Unknown/malformed: `unknown:<protocol>:<hash>`.
|
||||
|
||||
### Raw Archive
|
||||
|
||||
New or refactored module:
|
||||
|
||||
- `modules/sinks/raw-archive-store`
|
||||
|
||||
Interface:
|
||||
|
||||
```java
|
||||
public interface RawArchiveWriter {
|
||||
RawArchiveReceipt write(RawArchiveWriteRequest request) throws IOException;
|
||||
}
|
||||
```
|
||||
|
||||
Receipt:
|
||||
|
||||
```java
|
||||
public record RawArchiveReceipt(
|
||||
String rawUri,
|
||||
String checksum,
|
||||
long sizeBytes
|
||||
) {}
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- Raw bytes are immutable once written.
|
||||
- Write path is content-addressed or deterministic by `protocol/date/vehicleKey/frameId.bin`.
|
||||
- Write must be fsync-capable for local storage and pluggable for object storage.
|
||||
- `rawUri` must be resolvable by history API for single-frame replay.
|
||||
- Malformed frames are written with `parseStatus=FAILED` when bytes are available.
|
||||
|
||||
Deployment constraints:
|
||||
|
||||
- Raw archive, TDengine, and the history service may run on different ECS instances, but should use private-network access in the same VPC and availability zone.
|
||||
- High-frequency historical queries only read TDengine and must not read raw archive.
|
||||
- Raw archive is only used by single-frame detail, troubleshooting, replay, and asynchronous export paths.
|
||||
- `rawUri` must be globally resolvable by services and must not be a private local absolute path on one ECS instance.
|
||||
- The first local raw archive implementation must stay behind a replaceable interface so it can later move to OSS, MinIO, or another object store.
|
||||
|
||||
### Kafka Topics
|
||||
|
||||
Use stable versioned topics:
|
||||
|
||||
- `vehicle.raw-frame.v1`
|
||||
- `vehicle.decoded-fact.v1`
|
||||
- `vehicle.dlq.v1`
|
||||
|
||||
Kafka key:
|
||||
|
||||
```text
|
||||
<protocol>:<vehicleKey>
|
||||
```
|
||||
|
||||
`vehicle.raw-frame.v1` payload contains `RawFrameFact` without raw bytes. It carries `rawUri`, checksum, byte size, message id, parse status, and metadata.
|
||||
|
||||
`vehicle.decoded-fact.v1` payload contains one `DecodedFact`. It always references `frameId` and `rawUri`.
|
||||
|
||||
`vehicle.dlq.v1` carries failed store, publish, decode, and write attempts with enough metadata to replay from raw archive when possible.
|
||||
|
||||
### History Writer
|
||||
|
||||
New app or refactored app:
|
||||
|
||||
- `vehicle-history-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume `vehicle.raw-frame.v1`.
|
||||
- Consume `vehicle.decoded-fact.v1`.
|
||||
- Batch write TDengine.
|
||||
- Update realtime state store for latest vehicle state.
|
||||
- Write DLQ on invalid payloads or storage failures.
|
||||
|
||||
The writer is idempotent by `(frame_id)` for raw facts and `(fact_id)` for decoded facts.
|
||||
|
||||
## TDengine Model
|
||||
|
||||
TDengine is the production historical query store.
|
||||
|
||||
### Raw Frames Super Table
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
message_id INT,
|
||||
sub_type INT,
|
||||
event_time TIMESTAMP,
|
||||
raw_uri NCHAR(512),
|
||||
checksum NCHAR(128),
|
||||
raw_size_bytes BIGINT,
|
||||
parse_status NCHAR(16),
|
||||
parse_error NCHAR(512),
|
||||
peer NCHAR(128),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
Child table:
|
||||
|
||||
```text
|
||||
raw_<protocol>_<hash(vehicleKey)>
|
||||
```
|
||||
|
||||
Primary query patterns:
|
||||
|
||||
- Raw frames by vehicle and time.
|
||||
- Raw frames by protocol, message id, parse status, and time.
|
||||
- Single raw frame by `frame_id` or `raw_uri`.
|
||||
|
||||
### Location Super Table
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg DOUBLE,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
### Realtime Super Table
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_realtime (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
speed_kmh DOUBLE,
|
||||
total_mileage_km DOUBLE,
|
||||
battery_soc DOUBLE,
|
||||
total_voltage_v DOUBLE,
|
||||
total_current_a DOUBLE,
|
||||
running_mode NCHAR(64),
|
||||
vehicle_state NCHAR(64),
|
||||
charging_state NCHAR(64),
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
`fields_json` stores sparse full-field values for low-frequency inspection. High-frequency queried fields should be promoted to explicit columns or a dedicated extension table.
|
||||
|
||||
### Alarm Super Table
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_alarms (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
alarm_level NCHAR(32),
|
||||
alarm_code BIGINT,
|
||||
alarm_name NCHAR(128),
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
### Session and Registration Tables
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_sessions (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
session_type NCHAR(32),
|
||||
result_code INT,
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
JT808 registration details go into `vehicle_sessions` first and may later be projected into MySQL identity bindings for durable identity lookup.
|
||||
|
||||
## ACK and Durability Rules
|
||||
|
||||
For frames that require a protocol ACK:
|
||||
|
||||
1. Frame bytes are accepted from Netty.
|
||||
2. Raw bytes are written to raw archive.
|
||||
3. `RawFrameFact` is published to Kafka.
|
||||
4. Protocol-specific ACK is sent.
|
||||
5. Decoding and `DecodedFact` publication may run before or after ACK, but failures must emit a DLQ event and update raw frame parse status.
|
||||
|
||||
For GB32960 realtime/report frames, this prevents acknowledging a frame whose original bytes cannot be recovered.
|
||||
|
||||
For JT808 register/auth/heartbeat, ACK follows the same raw archive and raw fact boundary. Register ACK can still include token generation before raw fact publish, but must not be flushed to the channel until the durability boundary succeeds.
|
||||
|
||||
If raw archive write fails, close or keep the channel according to protocol tolerance, but do not ACK success.
|
||||
|
||||
If Kafka raw fact publish fails, do not ACK success for durable-ACK frames. Non-durable frames go to local DLQ when configured.
|
||||
|
||||
## Protocol Decoding and Extension
|
||||
|
||||
Use a plugin contract:
|
||||
|
||||
```java
|
||||
public interface ProtocolFrameDecoder {
|
||||
ProtocolId protocol();
|
||||
DecodeResult decode(RawFrameFact rawFact, byte[] rawBytes);
|
||||
}
|
||||
```
|
||||
|
||||
```java
|
||||
public record DecodeResult(
|
||||
ParseStatus status,
|
||||
List<DecodedFact> facts,
|
||||
String errorMessage,
|
||||
Map<String, String> metadata
|
||||
) {}
|
||||
```
|
||||
|
||||
GB32960:
|
||||
|
||||
- Frame command maps to session, realtime, location, alarm, and extension facts.
|
||||
- Vendor extensions are selected by platform account, VIN, or configured profile.
|
||||
- Full raw replay remains possible by reading `rawUri` and running the current decoder.
|
||||
|
||||
JT808:
|
||||
|
||||
- Message id maps to register, auth, heartbeat, location, batch location, media metadata, and extension facts.
|
||||
- Identity resolution updates `vehicleKey`.
|
||||
- Registration facts carry province, city, maker, device id, plate, plate color, and auth token metadata.
|
||||
|
||||
New protocol messages do not require TDengine schema changes unless they become high-frequency query dimensions. Unknown fields are stored as extension facts with `fields_json`.
|
||||
|
||||
## Query APIs
|
||||
|
||||
Keep API surfaces explicit.
|
||||
|
||||
Raw frame APIs:
|
||||
|
||||
- `GET /api/history/raw-frames`
|
||||
- Filters: `protocol`, `vin`, `phone`, `vehicleKey`, `messageId`, `parseStatus`, `dateFrom`, `dateTo`, `pageSize`, `cursor`.
|
||||
- Returns frame metadata, raw URI, checksum, parse status, and compact decoded summary when available.
|
||||
|
||||
Single raw frame API:
|
||||
|
||||
- `GET /api/history/raw-frames/{frameId}`
|
||||
- Returns raw metadata and decoded detail.
|
||||
- Supports `includeRawHex=true` with a strict size limit.
|
||||
|
||||
Location APIs:
|
||||
|
||||
- `GET /api/history/locations`
|
||||
- Filters: `protocol`, `vin`, `phone`, `vehicleKey`, time range, cursor.
|
||||
|
||||
Realtime APIs:
|
||||
|
||||
- `GET /api/history/realtime`
|
||||
- Historical time-series query.
|
||||
|
||||
Realtime state APIs:
|
||||
|
||||
- `GET /api/realtime/vehicles/{vehicleKey}`
|
||||
- Reads Redis latest state, not TDengine.
|
||||
|
||||
Export:
|
||||
|
||||
- Export is asynchronous.
|
||||
- Request creates an export job.
|
||||
- Worker streams TDengine query results to object storage or local export files.
|
||||
- API returns job status and download URI.
|
||||
|
||||
## Pagination
|
||||
|
||||
Use cursor pagination, not deep offset, for high-QPS history queries.
|
||||
|
||||
Cursor fields:
|
||||
|
||||
```text
|
||||
ts, received_at, fact_id
|
||||
```
|
||||
|
||||
Descending query predicate:
|
||||
|
||||
```sql
|
||||
WHERE (
|
||||
ts < :cursorTs
|
||||
OR (ts = :cursorTs AND received_at < :cursorReceivedAt)
|
||||
OR (ts = :cursorTs AND received_at = :cursorReceivedAt AND fact_id < :cursorFactId)
|
||||
)
|
||||
```
|
||||
|
||||
Page size defaults to 100 and is capped at 1000.
|
||||
|
||||
## Capacity Design
|
||||
|
||||
Assumptions:
|
||||
|
||||
- 10,000 vehicles.
|
||||
- Normal report interval: 5 to 10 seconds.
|
||||
- Expected writes: 1,000 to 2,000 frames per second.
|
||||
- Peak burst factor: 3x.
|
||||
- Historical query target: 1,000 requests per second.
|
||||
|
||||
Write path:
|
||||
|
||||
- Netty worker threads stay CPU-light.
|
||||
- Raw archive writes use bounded async workers.
|
||||
- Kafka producer uses compression, batching, idempotence, and vehicle-key partitioning.
|
||||
- History writer batches TDengine inserts by super table and child table.
|
||||
- TDengine child tables are partitioned by vehicle key to make single-vehicle queries cheap.
|
||||
|
||||
Read path:
|
||||
|
||||
- Most high-QPS queries require vehicle key and time range.
|
||||
- Unbounded cross-vehicle queries are admin-only and capped.
|
||||
- Realtime latest state is served from Redis latest-state cache.
|
||||
- Raw detail replay reads one raw object and decodes on demand.
|
||||
- Common dictionary and metadata responses are cached in memory.
|
||||
|
||||
Backpressure:
|
||||
|
||||
- Raw archive worker queue has a fixed bound.
|
||||
- Kafka publish futures are tracked.
|
||||
- If durability boundary cannot complete within timeout, durable ACK fails and the channel is closed or throttled.
|
||||
- History writer lag is observable by Kafka consumer lag and TDengine batch latency.
|
||||
|
||||
## Migration Plan
|
||||
|
||||
Phase 1: Build new fact model beside existing code.
|
||||
|
||||
- Add `ingest-facts`.
|
||||
- Add raw archive receipt contract.
|
||||
- Add serializers for raw frame and decoded fact Kafka messages.
|
||||
- Add tests for stable ids, vehicle key, and raw URI generation.
|
||||
|
||||
Phase 2: Harden raw durability boundary.
|
||||
|
||||
- Refactor GB32960 and JT808 handlers so ACK waits for raw archive and raw fact publish.
|
||||
- Keep existing parsers.
|
||||
- Emit decoded facts to the new Kafka topic.
|
||||
|
||||
Phase 3: Build TDengine writer.
|
||||
|
||||
- Create TDengine schema manager.
|
||||
- Write raw frame facts.
|
||||
- Write location, realtime, alarm, session facts.
|
||||
- Add idempotent insert behavior by stable ids.
|
||||
|
||||
Phase 4: Build query APIs.
|
||||
|
||||
- Raw frame query.
|
||||
- Single frame replay.
|
||||
- Location history query.
|
||||
- Realtime history query.
|
||||
- Latest realtime state query.
|
||||
|
||||
Phase 5: Cut over and retire old hot paths.
|
||||
|
||||
- Stop using DuckDB/Parquet as production hot history for GB32960/JT808.
|
||||
- Keep raw replay compatibility while historical data migrates.
|
||||
- Remove protocol-specific ad hoc history storage once TDengine coverage is verified.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
Unit tests:
|
||||
|
||||
- `RawFrameFact` validation and vehicle key derivation.
|
||||
- Raw archive path generation and checksum.
|
||||
- GB32960 decoder facts from golden frames.
|
||||
- JT808 decoder facts from sample frames.
|
||||
- ACK boundary success and failure cases.
|
||||
- TDengine SQL generation.
|
||||
|
||||
Integration tests:
|
||||
|
||||
- Start app with embedded or mocked Kafka producer and fake raw archive.
|
||||
- Send GB32960 sample frame and verify raw fact, decoded fact, raw archive receipt.
|
||||
- Send JT808 register/auth/location frames and verify registration/session/location facts.
|
||||
- Simulate raw archive failure and verify no success ACK.
|
||||
- Simulate Kafka publish failure and verify durable ACK failure.
|
||||
|
||||
Storage tests:
|
||||
|
||||
- TDengine schema initialization.
|
||||
- Batch insert raw frames and decoded facts.
|
||||
- Query by vehicle key and time range.
|
||||
- Cursor pagination correctness.
|
||||
- Single raw frame replay by `rawUri`.
|
||||
|
||||
Load tests:
|
||||
|
||||
- 2,000 frames/s sustained write for at least 30 minutes.
|
||||
- 6,000 frames/s burst for 5 minutes.
|
||||
- 1,000 QPS location/realtime history queries with bounded p95 latency.
|
||||
- Verify no raw frame count gap between raw archive receipts, Kafka raw facts, and TDengine raw frame rows.
|
||||
|
||||
## Operational Requirements
|
||||
|
||||
Metrics:
|
||||
|
||||
- TCP connections.
|
||||
- Frame receive rate.
|
||||
- Raw archive write latency and failure count.
|
||||
- Kafka publish latency and failure count.
|
||||
- Decoder success/failure count by protocol and message id.
|
||||
- TDengine batch size, latency, and failure count.
|
||||
- Query QPS and latency by endpoint.
|
||||
- Kafka consumer lag.
|
||||
|
||||
Logs:
|
||||
|
||||
- One structured log per failed frame with `frameId`, protocol, vehicle key, message id, raw URI, and error.
|
||||
- No full raw bytes in normal logs.
|
||||
- Raw hex only in explicit debug tools with size limits.
|
||||
|
||||
Alerts:
|
||||
|
||||
- Raw archive write failure.
|
||||
- Durable ACK timeout.
|
||||
- Kafka producer failure.
|
||||
- History writer lag.
|
||||
- TDengine write failure.
|
||||
- Raw archive count and TDengine raw frame row count divergence.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
The redesign is complete only when all of the following are true:
|
||||
|
||||
1. Every accepted GB32960 and JT808 frame creates a raw archive object and a TDengine `raw_frames` row.
|
||||
2. Malformed frames with bytes are queryable from raw history with parse status and error reason.
|
||||
3. GB32960 and JT808 location history can be queried by vehicle and time range with cursor pagination.
|
||||
4. GB32960 realtime history can be queried by vehicle and time range.
|
||||
5. JT808 registration/session data is queryable and can update identity binding.
|
||||
6. Single-frame detail can reload raw bytes by `rawUri` and decode with the current decoder.
|
||||
7. ACK-required frames do not receive success ACK before raw archive and raw fact publication succeed.
|
||||
8. New protocol fields can be added by implementing a decoder/projector and, only when needed, a TDengine table migration.
|
||||
9. Load tests demonstrate sustained 2,000 frames/s writes and 1,000 QPS bounded historical queries.
|
||||
10. Old DuckDB/Parquet hot-history paths are no longer required for GB32960/JT808 production queries.
|
||||
|
||||
## Open Decisions
|
||||
|
||||
The following choices are intentionally fixed for the first implementation:
|
||||
|
||||
- Use local raw archive first, with the interface shaped for object storage later.
|
||||
- Use TDengine as the only production hot historical query store for GB32960/JT808.
|
||||
- Use explicit protocol-aware query APIs instead of a single generic query API.
|
||||
- Use cursor pagination for all high-QPS historical queries.
|
||||
- Keep Kafka payloads byte-light by sending raw references instead of raw bytes.
|
||||
@@ -1,624 +0,0 @@
|
||||
# 车辆接入重设计方案
|
||||
|
||||
## 背景
|
||||
|
||||
当前 GB32960 和 JT808 链路已经能够完成接收、解析、发布和查询，但职责边界被拆散在多条路径里：
|
||||
|
||||
- Netty Handler 生成 `RawFrame`，Dispatcher 同时发出原始归档事件和标准化车辆事件。
|
||||
- Kafka Envelope 只携带 raw archive 引用，原始 bytes 由 archive sink 另行存储。
|
||||
- GB32960 历史查询依赖 raw archive 记录，再回读文件重新解析。
|
||||
- JT808 位置历史又单独物化到 TDengine。
|
||||
- DuckDB/Parquet、本地 raw archive、Kafka、TDengine 分别承担一部分职责，导致查询语义和持久化边界不好推理。
|
||||
|
||||
新设计保留已有协议解析器和部署经验，但重新定义接入、持久化、存储和查询边界。
|
||||
|
||||
## 目标
|
||||
|
||||
1. 保存每一条入站协议原始帧，包括可拿到 bytes 的异常帧。
|
||||
2. 支持按车辆、协议、消息类型、时间范围查询 GB32960 和 JT808 历史上报数据。
|
||||
3. 新增字段、协议消息、厂商扩展时，不需要重写核心管线。
|
||||
4. 支撑 10,000 辆车生产写入，以及 1,000 QPS 历史查询。
|
||||
5. 明确 ACK 和持久化规则，避免“已经 ACK 但数据悄悄丢失”。
|
||||
|
||||
## 非目标
|
||||
|
||||
- 不把 Kafka 作为长期历史查询数据库。
|
||||
- 不把大块原始 bytes 直接塞进 Kafka topic。
|
||||
- 不继续把 DuckDB/Parquet 作为 GB32960/JT808 的生产热查询库。
|
||||
- 不做一个掩盖协议差异的万能接口。内部可以统一，外部 API 保持协议语义清晰。
|
||||
|
||||
## 推荐架构
|
||||
|
||||
采用 `Raw Archive + Kafka + TDengine` 作为核心架构。
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
GB["GB32960 TCP"] --> Edge["协议接入边界"]
|
||||
JT["JT808 TCP"] --> Edge
|
||||
Edge --> RawStore["Raw Archive"]
|
||||
RawStore --> RawTopic["Kafka vehicle.raw-frame.v1"]
|
||||
Edge --> Decode["协议解析器"]
|
||||
Decode --> FactTopic["Kafka vehicle.decoded-fact.v1"]
|
||||
RawTopic --> Writer["History Writer"]
|
||||
FactTopic --> Writer
|
||||
Writer --> TD["TDengine"]
|
||||
Writer --> Redis["实时状态"]
|
||||
API["历史 API"] --> TD
|
||||
API --> RawStore
|
||||
RealtimeAPI["实时 API"] --> Redis
|
||||
```
|
||||
|
||||
架构里只有三类事实：
|
||||
|
||||
- `RawFrameFact`：每一条入站帧一条记录，不管解析成功还是失败。
|
||||
- `DecodedFact`：从 raw frame 派生出来的零条或多条标准事实。
|
||||
- `RawArchiveObject`：不可变原始 bytes，通过 `raw_uri` 寻址。
|
||||
|
||||
raw archive 是排障和回放的事实源。TDengine 是查询事实源。Kafka 是解耦和重放管道。
|
||||
|
||||
## 模块边界
|
||||
|
||||
### 协议接入 APP
|
||||
|
||||
APP：
|
||||
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
|
||||
职责：
|
||||
|
||||
- 接收 TCP 连接。
|
||||
- 完成帧切分。
|
||||
- 执行协议层校验、鉴权、会话绑定和 ACK。
|
||||
- 创建 `RawFrameFact`。
|
||||
- 通过 `RawArchiveWriter` 持久化原始 bytes。
|
||||
- 发布 raw frame fact 到 Kafka。
|
||||
- 解析协议帧并发布 decoded facts。
|
||||
|
||||
接入 APP 不负责历史查询、导出，也不直接写业务查询表。
|
||||
|
||||
### Ingest Core
|
||||
|
||||
新增模块：
|
||||
|
||||
- `modules/core/ingest-facts`
|
||||
|
||||
核心类型：
|
||||
|
||||
```java
|
||||
public record RawFrameFact(
|
||||
String frameId,
|
||||
ProtocolId protocol,
|
||||
String vehicleKey,
|
||||
String vin,
|
||||
String phone,
|
||||
int messageId,
|
||||
int subType,
|
||||
Instant eventTime,
|
||||
Instant receivedAt,
|
||||
String peer,
|
||||
String rawUri,
|
||||
String checksum,
|
||||
long rawSizeBytes,
|
||||
ParseStatus parseStatus,
|
||||
String parseError,
|
||||
Map<String, String> metadata
|
||||
) {}
|
||||
```
|
||||
|
||||
```java
|
||||
public sealed interface DecodedFact permits
|
||||
DecodedFact.Location,
|
||||
DecodedFact.Realtime,
|
||||
DecodedFact.Alarm,
|
||||
DecodedFact.Session,
|
||||
DecodedFact.Register,
|
||||
DecodedFact.Extension {
|
||||
|
||||
String factId();
|
||||
String frameId();
|
||||
ProtocolId protocol();
|
||||
String vehicleKey();
|
||||
String vin();
|
||||
Instant eventTime();
|
||||
Instant receivedAt();
|
||||
String rawUri();
|
||||
Map<String, String> metadata();
|
||||
}
|
||||
```
|
||||
|
||||
`vehicleKey` 是稳定分区键：
|
||||
|
||||
- GB32960：VIN。
|
||||
- JT808：已解析到 VIN 时使用 VIN，否则使用 `jt808:<phone>`。
|
||||
- 未知或异常帧：使用 `unknown:<protocol>:<hash>`。
|
||||
|
||||
### Raw Archive
|
||||
|
||||
新增或重构模块：
|
||||
|
||||
- `modules/sinks/raw-archive-store`
|
||||
|
||||
接口：
|
||||
|
||||
```java
|
||||
public interface RawArchiveWriter {
|
||||
RawArchiveReceipt write(RawArchiveWriteRequest request) throws IOException;
|
||||
}
|
||||
```
|
||||
|
||||
写入回执：
|
||||
|
||||
```java
|
||||
public record RawArchiveReceipt(
|
||||
String rawUri,
|
||||
String checksum,
|
||||
long sizeBytes
|
||||
) {}
|
||||
```
|
||||
|
||||
规则：
|
||||
|
||||
- 原始 bytes 一旦写入，不允许原地修改。
|
||||
- 写入路径可以按内容寻址，也可以按 `protocol/date/vehicleKey/frameId.bin` 生成。
|
||||
- 本地存储需要支持 fsync 能力，接口要预留对象存储实现。
|
||||
- `rawUri` 必须能被历史 API 解析，用于单帧回放。
|
||||
- 异常帧只要有 bytes，也必须写入，并记录 `parseStatus=FAILED`。
|
||||
|
||||
部署约束：
|
||||
|
||||
- Raw archive、TDengine、history 服务可以不在同一台 ECS，但应在同一 VPC 和同一可用区内通过内网访问。
|
||||
- 高频历史查询只访问 TDengine，不访问 raw archive。
|
||||
- Raw archive 只进入单帧详情、问题排查、回放和异步导出路径。
|
||||
- `rawUri` 必须是全局可解析地址，不能是某台 ECS 私有的本地绝对路径。
|
||||
- 第一版本地 raw archive 需要通过可替换接口封装，后续可以切换到 OSS、MinIO 或其它对象存储。
|
||||
|
||||
### Kafka Topic
|
||||
|
||||
使用稳定的版本化 topic：
|
||||
|
||||
- `vehicle.raw-frame.v1`
|
||||
- `vehicle.decoded-fact.v1`
|
||||
- `vehicle.dlq.v1`
|
||||
|
||||
Kafka key：
|
||||
|
||||
```text
|
||||
<protocol>:<vehicleKey>
|
||||
```
|
||||
|
||||
`vehicle.raw-frame.v1` 的 payload 是不含 raw bytes 的 `RawFrameFact`，携带 `rawUri`、checksum、byte size、message id、parse status 和 metadata。
|
||||
|
||||
`vehicle.decoded-fact.v1` 的 payload 是单条 `DecodedFact`，必须引用 `frameId` 和 `rawUri`。
|
||||
|
||||
`vehicle.dlq.v1` 保存存储失败、发布失败、解析失败和写入失败事件，并携带足够元数据，尽量支持从 raw archive 重放。
|
||||
|
||||
### History Writer
|
||||
|
||||
新增或重构 APP：
|
||||
|
||||
- `vehicle-history-app`
|
||||
|
||||
职责：
|
||||
|
||||
- 消费 `vehicle.raw-frame.v1`。
|
||||
- 消费 `vehicle.decoded-fact.v1`。
|
||||
- 批量写 TDengine。
|
||||
- 更新实时状态存储。
|
||||
- 对非法 payload 或存储失败写 DLQ。
|
||||
|
||||
Writer 写入需要幂等：
|
||||
|
||||
- raw fact 以 `(frame_id)` 幂等。
|
||||
- decoded fact 以 `(fact_id)` 幂等。
|
||||
|
||||
## TDengine 模型
|
||||
|
||||
TDengine 是生产历史查询主库。
|
||||
|
||||
### 原始帧超级表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
message_id INT,
|
||||
sub_type INT,
|
||||
event_time TIMESTAMP,
|
||||
raw_uri NCHAR(512),
|
||||
checksum NCHAR(128),
|
||||
raw_size_bytes BIGINT,
|
||||
parse_status NCHAR(16),
|
||||
parse_error NCHAR(512),
|
||||
peer NCHAR(128),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
子表命名：
|
||||
|
||||
```text
|
||||
raw_<protocol>_<hash(vehicleKey)>
|
||||
```
|
||||
|
||||
主要查询模式：
|
||||
|
||||
- 按车辆和时间查询原始帧。
|
||||
- 按协议、消息 ID、解析状态和时间查询原始帧。
|
||||
- 按 `frame_id` 或 `raw_uri` 查询单帧。
|
||||
|
||||
### 位置超级表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg DOUBLE,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
### 实时数据超级表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_realtime (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
speed_kmh DOUBLE,
|
||||
total_mileage_km DOUBLE,
|
||||
battery_soc DOUBLE,
|
||||
total_voltage_v DOUBLE,
|
||||
total_current_a DOUBLE,
|
||||
running_mode NCHAR(64),
|
||||
vehicle_state NCHAR(64),
|
||||
charging_state NCHAR(64),
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
`fields_json` 存放稀疏全字段值，用于低频排查。高频查询字段应提升为明确列，或放入专用扩展表。
|
||||
|
||||
### 报警超级表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_alarms (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
alarm_level NCHAR(32),
|
||||
alarm_code BIGINT,
|
||||
alarm_name NCHAR(128),
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
### 会话和注册表
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_sessions (
|
||||
ts TIMESTAMP,
|
||||
fact_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
session_type NCHAR(32),
|
||||
result_code INT,
|
||||
raw_uri NCHAR(512),
|
||||
fields_json NCHAR(8192),
|
||||
metadata_json NCHAR(4096)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
vin NCHAR(64),
|
||||
phone NCHAR(32)
|
||||
);
|
||||
```
|
||||
|
||||
JT808 注册详情先进入 `vehicle_sessions`，后续可以投影到 MySQL 车辆身份绑定表，用于持久身份解析。
|
||||
|
||||
## ACK 和持久化规则
|
||||
|
||||
对需要协议 ACK 的帧：
|
||||
|
||||
1. Netty 接收到 frame bytes。
|
||||
2. 原始 bytes 写入 raw archive。
|
||||
3. `RawFrameFact` 发布到 Kafka。
|
||||
4. 发送协议 ACK。
|
||||
5. 解码和 `DecodedFact` 发布可以在 ACK 前或 ACK 后执行，但失败必须发 DLQ，并更新 raw frame 解析状态。
|
||||
|
||||
对 GB32960 实时上报和补发帧，这条规则保证不会 ACK 一条无法恢复原始 bytes 的帧。
|
||||
|
||||
对 JT808 注册、鉴权、心跳，也遵循相同 raw archive 和 raw fact 边界。注册 ACK 可以先生成 token，但在 durability boundary 成功前不能 flush 到 channel。
|
||||
|
||||
如果 raw archive 写入失败，不返回成功 ACK。具体是关闭连接还是保留连接，由协议容忍度决定。
|
||||
|
||||
如果 Kafka raw fact 发布失败，durable ACK 帧不返回成功 ACK。非 durable 帧可以写本地 DLQ。
|
||||
|
||||
## 协议解析和扩展
|
||||
|
||||
使用插件契约：
|
||||
|
||||
```java
|
||||
public interface ProtocolFrameDecoder {
|
||||
ProtocolId protocol();
|
||||
DecodeResult decode(RawFrameFact rawFact, byte[] rawBytes);
|
||||
}
|
||||
```
|
||||
|
||||
```java
|
||||
public record DecodeResult(
|
||||
ParseStatus status,
|
||||
List<DecodedFact> facts,
|
||||
String errorMessage,
|
||||
Map<String, String> metadata
|
||||
) {}
|
||||
```
|
||||
|
||||
GB32960：
|
||||
|
||||
- 命令帧映射为会话、实时、位置、报警和扩展事实。
|
||||
- 厂商扩展按平台账号、VIN 或配置 profile 选择。
|
||||
- 全量 raw replay 仍通过读取 `rawUri` 并运行当前解析器完成。
|
||||
|
||||
JT808：
|
||||
|
||||
- 消息 ID 映射为注册、鉴权、心跳、位置、批量位置、多媒体元数据和扩展事实。
|
||||
- 身份解析会更新 `vehicleKey`。
|
||||
- 注册事实携带省、市、厂商、设备 ID、车牌、车牌颜色和 auth token metadata。
|
||||
|
||||
新增协议消息通常不需要改 TDengine schema，除非它成为高频查询维度。未知字段先作为 extension fact 存入 `fields_json`。
|
||||
|
||||
## 查询 API
|
||||
|
||||
API 保持明确，不做一个含糊的统一接口。
|
||||
|
||||
原始帧查询：
|
||||
|
||||
- `GET /api/history/raw-frames`
|
||||
- 过滤条件：`protocol`、`vin`、`phone`、`vehicleKey`、`messageId`、`parseStatus`、`dateFrom`、`dateTo`、`pageSize`、`cursor`。
|
||||
- 返回 frame metadata、raw URI、checksum、parse status，以及可选的紧凑解析摘要。
|
||||
|
||||
单帧详情：
|
||||
|
||||
- `GET /api/history/raw-frames/{frameId}`
|
||||
- 返回 raw metadata 和解码详情。
|
||||
- 支持 `includeRawHex=true`，但必须有严格大小限制。
|
||||
|
||||
位置历史：
|
||||
|
||||
- `GET /api/history/locations`
|
||||
- 过滤条件：`protocol`、`vin`、`phone`、`vehicleKey`、时间范围、cursor。
|
||||
|
||||
实时历史：
|
||||
|
||||
- `GET /api/history/realtime`
|
||||
- 查询历史时序数据。
|
||||
|
||||
实时状态：
|
||||
|
||||
- `GET /api/realtime/vehicles/{vehicleKey}`
|
||||
- 从 Redis 或内存 latest state 读取，不扫 TDengine。
|
||||
|
||||
导出：
|
||||
|
||||
- 导出必须异步。
|
||||
- 请求创建 export job。
|
||||
- Worker 将 TDengine 查询结果流式写入对象存储或本地导出文件。
|
||||
- API 返回 job 状态和下载 URI。
|
||||
|
||||
## 分页
|
||||
|
||||
高 QPS 历史查询使用 cursor pagination，不使用深 offset。
|
||||
|
||||
Cursor 字段：
|
||||
|
||||
```text
|
||||
ts, received_at, fact_id
|
||||
```
|
||||
|
||||
倒序查询谓词：
|
||||
|
||||
```sql
|
||||
WHERE (
|
||||
ts < :cursorTs
|
||||
OR (ts = :cursorTs AND received_at < :cursorReceivedAt)
|
||||
OR (ts = :cursorTs AND received_at = :cursorReceivedAt AND fact_id < :cursorFactId)
|
||||
)
|
||||
```
|
||||
|
||||
默认 page size 为 100，上限为 1000。
|
||||
|
||||
## 容量设计
|
||||
|
||||
假设：
|
||||
|
||||
- 10,000 辆车。
|
||||
- 常规上报间隔 5 到 10 秒。
|
||||
- 预计写入 1,000 到 2,000 frames/s。
|
||||
- 峰值突发系数 3 倍。
|
||||
- 历史查询目标 1,000 requests/s。
|
||||
|
||||
写路径：
|
||||
|
||||
- Netty worker 保持 CPU-light。
|
||||
- Raw archive 写入使用有界异步 worker。
|
||||
- Kafka producer 开启压缩、批量、幂等，并按 vehicle key 分区。
|
||||
- History writer 按超级表和子表批量写 TDengine。
|
||||
- TDengine 子表按 vehicle key 分区，使单车查询足够便宜。
|
||||
|
||||
读路径：
|
||||
|
||||
- 高频查询必须带 vehicle key 和时间范围。
|
||||
- 无边界跨车查询仅限管理接口，并强制限流和限制返回量。
|
||||
- 实时 latest state 从 Redis 或内存状态读取。
|
||||
- Raw 详情只读取单个 raw object 并按需解析。
|
||||
- 字典和元数据接口使用内存缓存。
|
||||
|
||||
背压：
|
||||
|
||||
- Raw archive worker queue 固定上限。
|
||||
- Kafka publish future 必须被跟踪。
|
||||
- Durability boundary 超时时，durable ACK 失败，channel 关闭或被限流。
|
||||
- History writer lag 通过 Kafka consumer lag 和 TDengine batch latency 观察。
|
||||
|
||||
## 迁移计划
|
||||
|
||||
阶段 1：在现有代码旁边建立新的 fact model。
|
||||
|
||||
- 新增 `ingest-facts`。
|
||||
- 新增 raw archive receipt contract。
|
||||
- 新增 raw frame 和 decoded fact 的 Kafka 序列化。
|
||||
- 为 stable id、vehicle key、raw URI 生成规则加测试。
|
||||
|
||||
阶段 2：强化 raw durability boundary。
|
||||
|
||||
- 重构 GB32960 和 JT808 handler，让 ACK 等待 raw archive 和 raw fact publish。
|
||||
- 保留现有解析器。
|
||||
- 向新 Kafka topic 发 decoded facts。
|
||||
|
||||
阶段 3：构建 TDengine writer。
|
||||
|
||||
- 创建 TDengine schema manager。
|
||||
- 写入 raw frame facts。
|
||||
- 写入 location、realtime、alarm、session facts。
|
||||
- 按 stable id 实现幂等写入。
|
||||
|
||||
阶段 4：构建查询 API。
|
||||
|
||||
- Raw frame 查询。
|
||||
- 单帧回放。
|
||||
- 位置历史查询。
|
||||
- 实时历史查询。
|
||||
- 最新实时状态查询。
|
||||
|
||||
阶段 5：切换并下线旧热路径。
|
||||
|
||||
- GB32960/JT808 生产热历史不再依赖 DuckDB/Parquet。
|
||||
- 历史迁移期间保留 raw replay 兼容。
|
||||
- TDengine 覆盖验证完成后，移除协议特定的临时历史存储。
|
||||
|
||||
## 测试策略
|
||||
|
||||
单元测试：
|
||||
|
||||
- `RawFrameFact` 校验和 vehicle key 派生。
|
||||
- Raw archive 路径生成和 checksum。
|
||||
- GB32960 golden frame 到 decoded facts。
|
||||
- JT808 sample frame 到 decoded facts。
|
||||
- ACK boundary 成功和失败场景。
|
||||
- TDengine SQL 生成。
|
||||
|
||||
集成测试：
|
||||
|
||||
- 使用 mocked Kafka producer 和 fake raw archive 启动 APP。
|
||||
- 发送 GB32960 sample frame，验证 raw fact、decoded fact 和 raw archive receipt。
|
||||
- 发送 JT808 注册、鉴权、位置帧，验证 registration、session、location facts。
|
||||
- 模拟 raw archive 失败，验证不发送成功 ACK。
|
||||
- 模拟 Kafka publish 失败，验证 durable ACK 失败。
|
||||
|
||||
存储测试：
|
||||
|
||||
- TDengine schema 初始化。
|
||||
- 批量写入 raw frames 和 decoded facts。
|
||||
- 按 vehicle key 和时间范围查询。
|
||||
- Cursor 分页正确性。
|
||||
- 按 `rawUri` 单帧回放。
|
||||
|
||||
压测：
|
||||
|
||||
- 持续 30 分钟 2,000 frames/s 写入。
|
||||
- 持续 5 分钟 6,000 frames/s 突发写入。
|
||||
- 1,000 QPS location/realtime 历史查询，p95 延迟受控。
|
||||
- 校验 raw archive receipt、Kafka raw fact、TDengine raw frame row 三者数量无缺口。
|
||||
|
||||
## 运维要求
|
||||
|
||||
指标：
|
||||
|
||||
- TCP 连接数。
|
||||
- Frame 接收速率。
|
||||
- Raw archive 写入延迟和失败数。
|
||||
- Kafka 发布延迟和失败数。
|
||||
- 按协议和 message id 统计 decoder 成功/失败数。
|
||||
- TDengine batch size、延迟和失败数。
|
||||
- 各查询 endpoint 的 QPS 和延迟。
|
||||
- Kafka consumer lag。
|
||||
|
||||
日志：
|
||||
|
||||
- 每个失败帧输出一条结构化日志，包含 `frameId`、protocol、vehicle key、message id、raw URI 和 error。
|
||||
- 正常日志不输出完整 raw bytes。
|
||||
- Raw hex 只允许在显式 debug 工具里输出，并限制大小。
|
||||
|
||||
告警：
|
||||
|
||||
- Raw archive 写入失败。
|
||||
- Durable ACK 超时。
|
||||
- Kafka producer 失败。
|
||||
- History writer lag。
|
||||
- TDengine 写入失败。
|
||||
- Raw archive 数量和 TDengine raw frame row 数量不一致。
|
||||
|
||||
## 验收标准
|
||||
|
||||
重设计只有满足以下条件才算完成：
|
||||
|
||||
1. 每一条被接收的 GB32960 和 JT808 帧都会创建 raw archive object 和 TDengine `raw_frames` 行。
|
||||
2. 有 bytes 的异常帧可以从 raw history 查询到，包含 parse status 和错误原因。
|
||||
3. GB32960 和 JT808 位置历史可以按车辆和时间范围 cursor 分页查询。
|
||||
4. GB32960 实时历史可以按车辆和时间范围查询。
|
||||
5. JT808 注册和会话数据可查询，并且可以更新 identity binding。
|
||||
6. 单帧详情可以通过 `rawUri` 重新读取 raw bytes，并使用当前解析器解码。
|
||||
7. 需要 ACK 的帧，在 raw archive 和 raw fact publish 成功前，不会发送成功 ACK。
|
||||
8. 新协议字段可以通过新增 decoder/projector 扩展；只有成为高频查询字段时才需要 TDengine 表迁移。
|
||||
9. 压测证明可以持续 2,000 frames/s 写入，并支持 1,000 QPS 有边界历史查询。
|
||||
10. GB32960/JT808 生产查询不再依赖旧 DuckDB/Parquet 热历史路径。
|
||||
|
||||
## 已固定决策
|
||||
|
||||
第一版实现固定以下选择：
|
||||
|
||||
- 先使用本地 raw archive，实现接口时预留对象存储。
|
||||
- TDengine 是 GB32960/JT808 唯一生产热历史查询库。
|
||||
- 使用协议明确的查询 API，不做一个泛化万能查询 API。
|
||||
- 高频历史查询统一使用 cursor pagination。
|
||||
- Kafka payload 保持轻量，只传 raw 引用，不传 raw bytes。
|
||||
@@ -1,498 +0,0 @@
|
||||
# Go 车辆接入重构设计规格
|
||||
|
||||
## 目标
|
||||
|
||||
使用 Go 重新设计车辆数据接入运行面，让系统回到第一性原则：
|
||||
|
||||
- 接入层只负责可靠收包、协议解析、应答、身份解析和投递。
|
||||
- Kafka 是唯一在线消息通道。
|
||||
- TDengine 保存高吞吐时序明细和完整 RAW 解析 JSON。
|
||||
- MySQL 保存低频配置、身份映射和每日统计窄表。
|
||||
- Redis 保存可重建的准实时状态。
|
||||
- 32960、JT808、宇通 MQTT 使用同一套数据层接口，避免每个协议各自落库。
|
||||
|
||||
第一阶段只做生产链路的骨架和核心数据闭环：接收、解析、Kafka、TDengine、MySQL 统计、Redis 实时查询、ECS 验证。历史查询 API 和业务侧复杂分析放在后续阶段。
|
||||
|
||||
## 参考原则
|
||||
|
||||
本设计只吸收开源项目的边界思想，不复制外部实现代码。
|
||||
|
||||
- GB32960 参考 `DarkInno/gb32960-go-sdk`：连接管理、Handler、Forwarder、鉴权接口分离。
|
||||
- JT808 参考 `cuteLittleDevil/go-jt808`：高并发 Go 原生 TCP、协议核心小而可扩展、事件/适配器机制。
|
||||
- JT808 参考 `fakeyanss/jt808-server-go`：`FramePayload -> PacketData -> JT808Msg -> Reply` 的分层处理、Gateway 模式、保活和版本兼容。
|
||||
- TDengine 参考官方数据模型：按数据采集点建立 supertable，静态维度做 tags，明细数据按子表写入；Go 连接优先使用 WebSocket 驱动。
|
||||
|
||||
## 非目标
|
||||
|
||||
- 第一阶段不继续扩展 Java 服务。
|
||||
- 第一阶段不做统一大查询 API。
|
||||
- 第一阶段不恢复信达 Push。
|
||||
- 第一阶段不把所有协议字段展开成一张超宽 TDengine 表。
|
||||
- Redis 不作为历史事实源。
|
||||
- MySQL 不保存高频原始明细。
|
||||
|
||||
## 目标目录
|
||||
|
||||
新的 Go 运行面放在 `go/vehicle-gateway`，不继续沿用实验性的多 module 分散结构。现有 `go/ingest-edge` 和 `go/vehicle-state` 可以作为迁移素材，最终归并或删除。
|
||||
|
||||
```text
|
||||
go/vehicle-gateway/
|
||||
cmd/
|
||||
gateway/ # 协议接入进程：32960 TCP、JT808 TCP、宇通 MQTT
|
||||
history-writer/ # Kafka -> TDengine RAW/location/mileage points
|
||||
stat-writer/ # Kafka -> MySQL 每日统计窄表
|
||||
realtime-api/ # Redis 准实时查询 API
|
||||
internal/
|
||||
config/ # 环境变量、配置文件、校验
|
||||
gateway/ # TCP/MQTT 生命周期、连接限制、优雅停机
|
||||
protocol/
|
||||
gb32960/ # 32960 编解码
|
||||
jt808/ # 808 编解码
|
||||
yutongmqtt/ # 宇通 MQTT payload 解析
|
||||
identity/ # VIN、phone、device_id、plate 绑定
|
||||
envelope/ # 统一事件模型
|
||||
eventbus/ # Kafka producer/consumer
|
||||
history/ # TDengine adapter
|
||||
stats/ # 每日里程聚合
|
||||
realtime/ # Redis adapter 与快照合并
|
||||
observability/ # 日志、metrics、健康检查
|
||||
```
|
||||
|
||||
## 协议接入设计
|
||||
|
||||
### GB32960
|
||||
|
||||
职责：
|
||||
|
||||
- 监听 TCP `32960`。
|
||||
- 支持车辆登入、实时信息上报、补发信息上报、车辆登出、平台登入/登出、心跳、校时。
|
||||
- 正确返回协议应答。
|
||||
- 完整解析 2016 数据单元：
|
||||
- 整车数据
|
||||
- 驱动电机数据
|
||||
- 燃料电池数据
|
||||
- 发动机数据
|
||||
- 车辆位置数据
|
||||
- 极值数据
|
||||
- 报警数据
|
||||
- 可充电储能装置电压数据
|
||||
- 可充电储能装置温度数据
|
||||
- RAW 中保存完整 parsed JSON。
|
||||
- 关键字段标准化为统一 field：
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `soc_percent`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `vehicle_status`
|
||||
- `charge_status`
|
||||
|
||||
### JT808
|
||||
|
||||
职责：
|
||||
|
||||
- 监听 TCP `808`。
|
||||
- 支持 2011/2013 基础兼容，预留 2019 版本标记。
|
||||
- 分层处理：
|
||||
- Frame：`0x7e` 包边界、转义、校验。
|
||||
- Packet：消息头、手机号 BCD、流水号、分包。
|
||||
- Message：注册、鉴权、心跳、注销、位置、批量位置、未知上行。
|
||||
- Reply：注册应答、平台通用应答。
|
||||
- 终端手机号按协议头 BCD 解析，内部同时保留原始 BCD、规范化 phone、去前导零 phone。
|
||||
- 注册帧和鉴权帧写入 808 registration 表。
|
||||
- 如果设备直接上报位置帧，也要走 identity 解析，尝试用 phone、device_id、plate 找到 VIN。
|
||||
- 0200 附加信息必须解析表 27：
|
||||
- `0x01` GPS 总里程，DWORD，单位 0.1 km。
|
||||
- 其他已知附加项结构化放入 `parsed_json.additional`。
|
||||
- 标准化字段：
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `altitude_m`
|
||||
- `direction_deg`
|
||||
- `alarm_flag`
|
||||
- `status_flag`
|
||||
|
||||
### 宇通 MQTT
|
||||
|
||||
职责：
|
||||
|
||||
- 使用正式 MQTT 配置订阅生产 topic。
|
||||
- 接收 payload 后解析为统一 envelope。
|
||||
- 按 payload 中的车辆标识解析 VIN、车牌、设备号。
|
||||
- 保存完整 payload 和 parsed JSON。
|
||||
- 标准化字段向 32960/808 对齐：
|
||||
- `speed_kmh`
|
||||
- `total_mileage_km`
|
||||
- `longitude`
|
||||
- `latitude`
|
||||
- `soc_percent`
|
||||
- `vehicle_status`
|
||||
|
||||
## 统一 Envelope
|
||||
|
||||
所有协议接入后输出同一结构。
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "string",
|
||||
"trace_id": "string",
|
||||
"protocol": "GB32960|JT808|YUTONG_MQTT",
|
||||
"message_id": "string",
|
||||
"vin": "string",
|
||||
"vehicle_key": "string",
|
||||
"phone": "string",
|
||||
"device_id": "string",
|
||||
"plate": "string",
|
||||
"source_endpoint": "ip:port or mqtt://broker/topic",
|
||||
"event_time_ms": 0,
|
||||
"received_at_ms": 0,
|
||||
"raw_hex": "string",
|
||||
"raw_text": "string",
|
||||
"parsed": {},
|
||||
"fields": {
|
||||
"speed_kmh": 0,
|
||||
"total_mileage_km": 0,
|
||||
"longitude": 0,
|
||||
"latitude": 0
|
||||
},
|
||||
"parse_status": "OK|PARTIAL|BAD_FRAME",
|
||||
"parse_error": ""
|
||||
}
|
||||
```
|
||||
|
||||
规则：
|
||||
|
||||
- `event_id` 使用协议、设备标识、消息流水、事件时间、RAW hash 生成，保证幂等。
|
||||
- Kafka partition key 优先使用 VIN；没有 VIN 时使用 `protocol:phone/device_id`。
|
||||
- `parsed` 保存协议完整结构化字段。
|
||||
- `fields` 只保存跨协议核心字段，用于统计、位置和实时快照。
|
||||
|
||||
## Kafka Topic
|
||||
|
||||
生产只支持 Kafka。
|
||||
|
||||
| Topic | 内容 | Key |
|
||||
|---|---|---|
|
||||
| `vehicle.raw.gb32960.v1` | 32960 完整 RAW envelope | VIN 或 vehicle_key |
|
||||
| `vehicle.raw.jt808.v1` | 808 完整 RAW envelope | VIN 或 phone |
|
||||
| `vehicle.raw.yutong-mqtt.v1` | 宇通 MQTT 完整 RAW envelope | VIN 或 vehicle_key |
|
||||
| `vehicle.event.unified.v1` | 统一轻量事件，用于业务消费 | VIN 或 vehicle_key |
|
||||
|
||||
接入层必须先写 RAW topic，再写 unified event。后续消费者只依赖 Kafka，不直接依赖接入进程内存。
|
||||
|
||||
发布可靠性：
|
||||
|
||||
- Kafka 生产端必须开启同步写入，RAW 写成功后才允许写 unified event。
|
||||
- Kafka 写入必须支持可配置重试、单次写超时和短退避，默认值为 `KAFKA_PUBLISH_ATTEMPTS=3`、`KAFKA_PUBLISH_TIMEOUT_MS=3000`、`KAFKA_PUBLISH_BACKOFF_MS=100`。
|
||||
- gateway 支持本地磁盘 spool/WAL，配置 `KAFKA_SPOOL_DIR` 后启用；Kafka 长时间不可用时，发布失败的 envelope 先原子写入本地 JSON 文件。
|
||||
- spool 补发按文件名顺序执行，补发成功后删除文件；默认补发间隔由 `KAFKA_SPOOL_REPLAY_INTERVAL_MS=1000` 控制。
|
||||
- 如果 RAW 写 Kafka 失败并已落盘，同一个 event 的 unified event 不允许抢先写 Kafka，也必须进入 spool，确保恢复后按 RAW -> unified 顺序补发。
|
||||
- Kafka topic 不允许自动创建，topic 和分区数由部署脚本或运维初始化，避免生产拼写错误造成隐性分流。
|
||||
|
||||
## TDengine 数据库设计
|
||||
|
||||
如果现有 TDengine 库不符合目标，可以新建库并迁移。建议库名：
|
||||
|
||||
```sql
|
||||
CREATE DATABASE IF NOT EXISTS lingniu_vehicle_ts KEEP 7300 DURATION 10 BUFFER 256;
|
||||
```
|
||||
|
||||
### raw_frames
|
||||
|
||||
保存完整 RAW 和完整 parsed JSON。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
event_id NCHAR(64),
|
||||
message_id INT,
|
||||
event_time TIMESTAMP,
|
||||
received_at TIMESTAMP,
|
||||
raw_size_bytes INT,
|
||||
raw_hex BINARY(16374),
|
||||
raw_text BINARY(16374),
|
||||
parsed_json BINARY(16374),
|
||||
fields_json BINARY(4096),
|
||||
parse_status NCHAR(16),
|
||||
parse_error BINARY(1024),
|
||||
source_endpoint NCHAR(128)
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
说明：
|
||||
|
||||
- 子表按 `protocol + vehicle_key hash` 创建。
|
||||
- RAW 查询优先按 tags 和时间范围过滤。
|
||||
- `parsed_json` 是完整协议字段；查询 API 可选择是否返回，避免默认大字段拖慢。
|
||||
|
||||
### vehicle_locations
|
||||
|
||||
只保存位置查询核心字段。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg INT,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
### vehicle_mileage_points
|
||||
|
||||
保存总里程采样点，用于回放重算和查询。
|
||||
|
||||
```sql
|
||||
CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
total_mileage_km DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
);
|
||||
```
|
||||
|
||||
## MySQL 数据库设计
|
||||
|
||||
MySQL 只保存配置、身份映射、808 注册鉴权和每日统计窄表。
|
||||
|
||||
### vehicle_identity_binding
|
||||
|
||||
用于把外部标识定位到 VIN。
|
||||
|
||||
```sql
|
||||
CREATE TABLE vehicle_identity_binding (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
plate VARCHAR(32) NULL,
|
||||
phone VARCHAR(32) NULL,
|
||||
device_id VARCHAR(64) NULL,
|
||||
remark VARCHAR(255) NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_vin (vin),
|
||||
KEY idx_plate (plate),
|
||||
KEY idx_phone (phone),
|
||||
KEY idx_device_id (device_id)
|
||||
);
|
||||
```
|
||||
|
||||
### jt808_registration
|
||||
|
||||
只记录 808 独有的注册、鉴权和在线身份信息。
|
||||
|
||||
```sql
|
||||
CREATE TABLE jt808_registration (
|
||||
phone VARCHAR(32) PRIMARY KEY,
|
||||
vin VARCHAR(32) NULL,
|
||||
plate VARCHAR(32) NULL,
|
||||
device_id VARCHAR(64) NULL,
|
||||
manufacturer_id VARCHAR(32) NULL,
|
||||
terminal_model VARCHAR(64) NULL,
|
||||
terminal_id VARCHAR(64) NULL,
|
||||
auth_code VARCHAR(128) NULL,
|
||||
source_endpoint VARCHAR(128) NULL,
|
||||
first_registered_at DATETIME NULL,
|
||||
latest_registered_at DATETIME NULL,
|
||||
latest_authed_at DATETIME NULL,
|
||||
latest_seen_at DATETIME NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
KEY idx_vin (vin),
|
||||
KEY idx_plate (plate),
|
||||
KEY idx_device_id (device_id)
|
||||
);
|
||||
```
|
||||
|
||||
### vehicle_daily_metric
|
||||
|
||||
32960 和 808 的每日里程、每日总里程都进入同一张窄表。
|
||||
|
||||
```sql
|
||||
CREATE TABLE vehicle_daily_metric (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
metric_key VARCHAR(64) NOT NULL,
|
||||
metric_value DECIMAL(18,3) NOT NULL,
|
||||
metric_unit VARCHAR(16) NOT NULL,
|
||||
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
calculation_method VARCHAR(64) NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_daily_metric (vin, stat_date, protocol, metric_key),
|
||||
KEY idx_stat_date (stat_date),
|
||||
KEY idx_protocol_metric (protocol, metric_key)
|
||||
);
|
||||
```
|
||||
|
||||
指标：
|
||||
|
||||
- `daily_mileage_km = max(total_mileage_km) - min(total_mileage_km)`
|
||||
- `daily_total_mileage_km = max(total_mileage_km)`
|
||||
|
||||
统计消费者可幂等重放。每个样本 upsert 时只维护当天 `min/max total_mileage_km`，不依赖进程内状态。
|
||||
|
||||
## Redis 实时数据设计
|
||||
|
||||
Redis 保存可从 Kafka 重建的数据。
|
||||
|
||||
| Key | 内容 |
|
||||
|---|---|
|
||||
| `vehicle:online:{vin}` | 在线状态、最后活跃时间、协议列表 |
|
||||
| `vehicle:latest:{vin}` | 合并后的 VIN 最新快照 |
|
||||
| `vehicle:latest:{vin}:{protocol}` | 单协议最新快照 |
|
||||
| `vehicle:last_seen` | VIN 到最后接收时间的 sorted set |
|
||||
|
||||
合并规则：
|
||||
|
||||
- 位置以最新 event_time 为准。
|
||||
- 总里程取最新上报值，不做递增假设。
|
||||
- 32960 多帧字段按 field timestamp 合并。
|
||||
- 808 和 MQTT 不覆盖其他协议独有字段。
|
||||
- Redis TTL 用于在线判断，不作为数据删除依据。
|
||||
|
||||
## 数据流
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
source["车辆 / 平台"] --> gateway["Go gateway"]
|
||||
gateway --> parser["protocol parser"]
|
||||
parser --> identity["identity resolver"]
|
||||
identity --> kafkaRaw["Kafka RAW topics"]
|
||||
identity --> kafkaEvent["Kafka unified event"]
|
||||
kafkaRaw --> history["history-writer"]
|
||||
kafkaRaw --> stats["stat-writer"]
|
||||
kafkaEvent --> realtime["realtime consumer"]
|
||||
history --> td["TDengine raw_frames / locations / mileage_points"]
|
||||
stats --> mysql["MySQL vehicle_daily_metric"]
|
||||
realtime --> redis["Redis latest state"]
|
||||
```
|
||||
|
||||
## 错误处理
|
||||
|
||||
- 协议坏帧仍写 RAW topic，`parse_status=BAD_FRAME`。
|
||||
- 可部分解析的帧写 `parse_status=PARTIAL`，保留 parse error。
|
||||
- Kafka 写失败时接入层先按配置重试；重试耗尽后不写 unified event，必须打错误日志和 metrics。
|
||||
- 启用 `KAFKA_SPOOL_DIR` 后，Kafka 重试耗尽会落本地 spool；如果 spool 写失败，才视为接入层最终失败。
|
||||
- TDengine 写失败不提交 Kafka offset。
|
||||
- MySQL 统计写失败不提交 Kafka offset。
|
||||
- Redis 写失败不影响历史和统计，但要通过 metrics 暴露。
|
||||
- 808 相同 phone 新连接时，旧连接应被替换并记录 latest_seen。
|
||||
|
||||
## 部署设计
|
||||
|
||||
生产 ECS 运行 Go 二进制或容器：
|
||||
|
||||
- `vehicle-gateway`
|
||||
- `vehicle-history-writer`
|
||||
- `vehicle-stat-writer`
|
||||
- `vehicle-realtime-api`
|
||||
|
||||
配置来源：
|
||||
|
||||
- 环境变量优先。
|
||||
- Nacos 可作为后续配置中心，但 Go 第一阶段必须能只靠环境变量启动，便于 Portainer 部署和故障恢复。
|
||||
|
||||
TDengine 连接：
|
||||
|
||||
- 优先 WebSocket `taosWS`。
|
||||
- 若当前 ECS 只开放原生端口，可短期兼容 `taosSql`，但代码配置必须显式标记为兼容模式。
|
||||
|
||||
## 验证计划
|
||||
|
||||
### 本地验证
|
||||
|
||||
- 用样例 808 报文验证：
|
||||
- phone BCD 解析。
|
||||
- 0200 经纬度、速度、方向、时间。
|
||||
- 表 27 `0x01` GPS 总里程。
|
||||
- 用真实 32960 报文验证：
|
||||
- 登录应答正确。
|
||||
- 实时帧和补发帧都能写 RAW。
|
||||
- 位置和总里程字段进入统一 fields。
|
||||
- 用宇通 MQTT payload 验证：
|
||||
- payload 完整保存。
|
||||
- 核心字段归一化。
|
||||
- 不连 Kafka 时可输出 JSON log。
|
||||
- 连接 Kafka 时 RAW topic 可消费到 envelope。
|
||||
|
||||
### ECS 验证
|
||||
|
||||
- 部署前先旁路接收或短窗口切流。
|
||||
- 验证端口：
|
||||
- `32960`
|
||||
- `808`
|
||||
- MQTT 订阅连接。
|
||||
- 验证 Kafka topic 有持续写入。
|
||||
- 验证 TDengine：
|
||||
- `raw_frames` 有三种协议数据。
|
||||
- `vehicle_locations` 能按 VIN、协议、时间分页查询。
|
||||
- `vehicle_mileage_points` 有 32960 和 808 总里程采样。
|
||||
- 验证 MySQL：
|
||||
- `vehicle_daily_metric` 有 32960 和 808 的 `daily_mileage_km`、`daily_total_mileage_km`。
|
||||
- 验证 Redis：
|
||||
- 查询 VIN 在线。
|
||||
- 查询 VIN 合并实时快照。
|
||||
- 查询 VIN 分协议实时快照。
|
||||
|
||||
## 验收标准
|
||||
|
||||
- Go gateway 能在 ECS 上接收真实 32960、808、宇通 MQTT 数据。
|
||||
- 三种协议 RAW 都进入 Kafka。
|
||||
- Kafka 短暂写失败时，gateway 按配置重试；单元测试覆盖首次失败后成功和重试耗尽返回错误。
|
||||
- Kafka 长时间不可用时，gateway 可把 RAW/unified 写入本地 spool；单元测试覆盖 RAW 已落盘时 unified 也落盘、恢复后按顺序补发并删除文件。
|
||||
- 三种协议 RAW 和 parsed JSON 都进入 TDengine。
|
||||
- 32960 和 808 的位置进入 `vehicle_locations`。
|
||||
- 32960 和 808 的总里程采样进入 `vehicle_mileage_points`。
|
||||
- 32960 和 808 的每日里程、每日总里程进入 MySQL 窄表。
|
||||
- Redis 可以查询 VIN 在线和实时数据。
|
||||
- 任一消费者宕机后可通过 Kafka offset 继续恢复。
|
||||
- 查询和统计验证使用 ECS 上的真实数据完成。
|
||||
|
||||
## 实施顺序
|
||||
|
||||
1. 创建 `go/vehicle-gateway` 单 module，迁移现有 Go 原型中可用的解析、Kafka、TDengine、MySQL、Redis 代码。
|
||||
2. 先写协议解析单元测试，覆盖 808 样例报文和 32960 核心字段。
|
||||
3. 完成统一 envelope 和 Kafka sink。
|
||||
4. 完成 TDengine schema bootstrap 与 history writer。
|
||||
5. 完成 MySQL schema bootstrap 与 stat writer。
|
||||
6. 完成 Redis realtime writer/API。
|
||||
7. 本地启动 gateway，用 ECS 转发流量验证。
|
||||
8. 构建 Linux 镜像，部署 ECS。
|
||||
9. 按验收标准逐项验证并记录证据。
|
||||
@@ -1,202 +0,0 @@
|
||||
# Target Architecture
|
||||
|
||||
This document records the production target for `lingniu-vehicle-ingest`.
|
||||
The project should stay small at runtime: protocol apps ingest and publish,
|
||||
history consumes and indexes, analytics derives metrics, and business systems
|
||||
read through explicit APIs or Kafka.
|
||||
|
||||
Only supported message backbone: Kafka.
|
||||
|
||||
## Active Scope
|
||||
|
||||
Active production protocols:
|
||||
|
||||
| Protocol | Runtime app | Event topic | Raw topic |
|
||||
|---|---|---|---|
|
||||
| GB/T 32960 | `gb32960-ingest-app` | `vehicle.event.gb32960.v1` | `vehicle.raw.gb32960.v1` |
|
||||
| JT/T 808 | `jt808-ingest-app` | `vehicle.event.jt808.v1` | `vehicle.raw.jt808.v1` |
|
||||
| Yutong MQTT | `yutong-mqtt-app` | `vehicle.event.mqtt-yutong.v1` | `vehicle.raw.mqtt-yutong.v1` |
|
||||
|
||||
Xinda Push source, Maven profile, deployment bindings, and history consumers
|
||||
are removed. New optimization work should target the active protocols above.
|
||||
|
||||
JSATL12 attachment upload is also outside the default production reactor. Keep
|
||||
it available through the `optional-attachments` profile until an attachment app
|
||||
is explicitly deployed.
|
||||
|
||||
## Goals
|
||||
|
||||
- Keep protocol IO, raw archive, history, latest-state, and statistics as
|
||||
separate responsibilities.
|
||||
- Publish normalized Kafka envelopes with VIN as the partition key whenever a
|
||||
VIN is known.
|
||||
- Store complete raw payload metadata and parsed JSON in TDengine `raw_frames`.
|
||||
- Store query-friendly location rows separately from raw payload JSON.
|
||||
- Keep derived statistics in metric tables, not protocol-specific daily tables.
|
||||
- Use Redis only for optional latest-state APIs that can be rebuilt from Kafka.
|
||||
- Keep optional compatibility modules outside the production hot path.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Ingest apps do not write business tables.
|
||||
- Ingest apps do not serve history query APIs.
|
||||
- Redis is not the long-term historical store.
|
||||
- File-based event indexes are not part of the current build surface.
|
||||
- `telemetry_fields` parsing and field-trend APIs are owned by a separate
|
||||
field parsing service. This project stores RAW parsed JSON plus compact
|
||||
location rows and does not expose a telemetry-fields history API.
|
||||
|
||||
## Runtime Responsibilities
|
||||
|
||||
### Protocol Apps
|
||||
|
||||
Apps:
|
||||
|
||||
- `gb32960-ingest-app`
|
||||
- `jt808-ingest-app`
|
||||
- `yutong-mqtt-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Accept protocol traffic.
|
||||
- Decode, authenticate, acknowledge, and maintain sessions where the protocol
|
||||
requires it.
|
||||
- Archive raw bytes through `sink-archive` when bytes are available.
|
||||
- Publish parsed event envelopes to Kafka.
|
||||
- Publish raw envelopes to Kafka.
|
||||
- Resolve vehicle identity through the shared MySQL identity binding table.
|
||||
|
||||
Protocol apps must stay independent from TDengine history readers, Redis state
|
||||
repositories, and statistic calculators.
|
||||
|
||||
### Event Contract
|
||||
|
||||
The Kafka sink owns the protobuf envelope and consumer plumbing.
|
||||
Consumers should use `EnvelopeConsumerProcessor` so invalid protobuf, skipped
|
||||
envelopes, and downstream failures are converted to structured results and DLQ
|
||||
records instead of blocking a partition.
|
||||
|
||||
Core identity fields:
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `event_id` | Parsed event idempotency key |
|
||||
| `trace_id` | Cross-service trace id |
|
||||
| `vin` | Vehicle id and Kafka partition key |
|
||||
| `source` | Source protocol, for example `GB32960`, `JT808`, `YUTONG_MQTT` |
|
||||
| `event_time_ms` | Device event time |
|
||||
| `ingest_time_ms` | Platform receive time |
|
||||
| `raw_uri` | `archive://...` reference to raw bytes when available |
|
||||
|
||||
Full-field telemetry uses `TelemetrySnapshot` fields with stable keys such as
|
||||
`total_mileage_km`, `speed_kmh`, `longitude`, and `latitude`.
|
||||
|
||||
### History App
|
||||
|
||||
App: `vehicle-history-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume active protocol event topics.
|
||||
- Consume active protocol raw topics.
|
||||
- Write raw frame rows into TDengine `raw_frames`.
|
||||
- Write compact location rows into TDengine location tables.
|
||||
- Keep `payloadJson.parsed` on raw rows for full raw inspection.
|
||||
- Expose paged history APIs for raw frames and locations.
|
||||
- Keep specialized APIs off by default unless explicitly enabled.
|
||||
|
||||
Production default:
|
||||
|
||||
- `TDENGINE_HISTORY_ENABLED=true` in deployment.
|
||||
- `EVENT_FILE_STORE_ENABLED=false`.
|
||||
|
||||
File-based event indexes have been removed; this app should keep history
|
||||
queries on TDengine and raw replay on `archive://...` references.
|
||||
|
||||
### Analytics App
|
||||
|
||||
App: `vehicle-analytics-app`
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume JT808 event envelopes.
|
||||
- Calculate 808 daily mileage from the reported GPS total mileage only:
|
||||
|
||||
```text
|
||||
daily_mileage_km = max_total_mileage_km - min_total_mileage_km
|
||||
calculation_method = JT808_TOTAL_MILEAGE_DIFF
|
||||
```
|
||||
|
||||
- Store the metric in `vehicle_stat_metric` with
|
||||
`metric_key = daily_mileage_km`; the local-day minimum and maximum GPS total
|
||||
mileage used for the subtraction stay on the same metric row as calculation
|
||||
source columns.
|
||||
|
||||
Runtime state: none outside `vehicle_stat_metric`. There is no JT808-specific
|
||||
daily mileage table. Restart recovery reads the same metric row.
|
||||
|
||||
### Latest State
|
||||
|
||||
Module: `vehicle-state-service`
|
||||
|
||||
This module is outside the default production reactor. Build it with
|
||||
`-Poptional-latest-state` when Redis latest-state APIs are explicitly needed.
|
||||
|
||||
Responsibilities:
|
||||
|
||||
- Consume normalized Kafka envelopes.
|
||||
- Maintain latest state, location, and safety snapshots in Redis.
|
||||
- Serve latest-state APIs for operational screens.
|
||||
|
||||
Redis state must be rebuildable from Kafka replay and should not be used as the
|
||||
source of truth for historical queries.
|
||||
|
||||
## Data Flow
|
||||
|
||||
Default production flow:
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
vehicle["Vehicles / Platforms"] --> ingest["Protocol apps"]
|
||||
ingest --> archive["sink-archive<br/>raw bytes"]
|
||||
ingest --> kafka["Kafka<br/>event + raw envelopes"]
|
||||
kafka --> history["vehicle-history-app"]
|
||||
kafka --> analytics["vehicle-analytics-app"]
|
||||
history --> tdengine["TDengine<br/>raw_frames + locations"]
|
||||
analytics --> mysql["MySQL<br/>vehicle_stat_metric"]
|
||||
```
|
||||
|
||||
Optional latest-state flow, enabled only with `-Poptional-latest-state`, consumes
|
||||
the same Kafka envelopes and rebuilds Redis snapshots outside the default
|
||||
production deployment.
|
||||
|
||||
## Failure Handling
|
||||
|
||||
- Protocol apps continue accepting traffic when history, analytics, or Redis
|
||||
consumers are unavailable.
|
||||
- Kafka producer failures are handled by sink retry/circuit-breaker behavior and
|
||||
DLQ topics where configured.
|
||||
- History consumers should commit offsets only after TDengine writes succeed.
|
||||
- Analytics daily mileage state can recover from `vehicle_stat_metric` or Kafka
|
||||
replay.
|
||||
- Raw archive failures must be observable, but event publication and history
|
||||
indexing remain separate concerns.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- Active production deployment contains GB32960, JT808, Yutong MQTT,
|
||||
vehicle-history, and vehicle-analytics apps.
|
||||
- Xinda Push source and deployment bindings are removed.
|
||||
- JSATL12 is absent from the default Maven reactor and available through
|
||||
`optional-attachments`.
|
||||
- vehicle-state-service is absent from the default Maven reactor and available
|
||||
through `optional-latest-state`.
|
||||
- raw-archive-store prototype has been removed; raw bytes are written through
|
||||
`sink-archive`.
|
||||
- History APIs read from TDengine; file-based event indexes are not maintained.
|
||||
- Raw frame queries can return complete parsed JSON through `payloadJson.parsed`.
|
||||
- Location queries page over compact rows and reference raw records instead of
|
||||
duplicating full raw JSON.
|
||||
- JT808 daily mileage is stored only in `vehicle_stat_metric`.
|
||||
- Runtime state needed for 808 mileage is also in `vehicle_stat_metric`.
|
||||
- Build validation covers the active modules and their composition tests.
|
||||
@@ -1,146 +0,0 @@
|
||||
# Vehicle Telemetry Internal Fields
|
||||
|
||||
This document defines the internal telemetry field model for Lingniu hydrogen
|
||||
vehicle operations. Protocol fields such as GB/T 32960, JT808, MQTT, and vendor
|
||||
extensions must be mapped into these fields before storage, statistics, or
|
||||
business reporting.
|
||||
|
||||
## Design Rules
|
||||
|
||||
- Business statistics use internal fields, not protocol field names.
|
||||
- Every internal field has one stable unit and one business meaning.
|
||||
- Protocol-specific raw values remain traceable through `source_protocol`,
|
||||
`protocol_version`, `metadata`, and raw archive references.
|
||||
- Core operational fields are stored as typed columns or typed payload fields.
|
||||
- Unstable vendor-specific fields can stay in extension metadata until promoted.
|
||||
- Hydrogen leak safety is a first-class critical safety event, not a generic
|
||||
alarm count.
|
||||
|
||||
## Identity And Time
|
||||
|
||||
| Internal field | Type | Unit | Meaning | GB/T 32960 source |
|
||||
|---|---:|---|---|---|
|
||||
| `vin` | string | - | Vehicle VIN and partition key | header VIN |
|
||||
| `event_time` | instant | - | Device collection time | command body timestamp |
|
||||
| `ingest_time` | instant | - | Platform receive time | ingest server clock |
|
||||
| `source_protocol` | enum | - | Source adapter | `GB32960` |
|
||||
| `protocol_version` | string | - | Source protocol version | `V2016` / `V2025` |
|
||||
|
||||
## Real-Time Vehicle State
|
||||
|
||||
| Internal field | Type | Unit | Meaning | GB/T 32960 source |
|
||||
|---|---:|---|---|---|
|
||||
| `vehicle_state` | enum | - | Started, shutdown, other, invalid | vehicle block `vehicleState` |
|
||||
| `charging_state` | enum | - | Charging state | vehicle block `chargingState` |
|
||||
| `running_mode` | enum | - | Electric, hybrid, fuel, other | vehicle block `runningMode` |
|
||||
| `speed_kmh` | double | km/h | Current speed | vehicle block `speedKmh` |
|
||||
| `total_mileage_km` | double | km | Odometer mileage | vehicle block `totalMileageKm` |
|
||||
| `gear_level` | integer | - | Gear value | vehicle block `gearRaw` low 4 bits |
|
||||
| `accelerator_pedal` | double | % | Accelerator pedal opening | V2016 vehicle block |
|
||||
| `brake_pedal` | double | % | Brake pedal opening | V2016 vehicle block |
|
||||
|
||||
## Location
|
||||
|
||||
| Internal field | Type | Unit | Meaning | GB/T 32960 source |
|
||||
|---|---:|---|---|---|
|
||||
| `longitude` | double | deg | Longitude | position block |
|
||||
| `latitude` | double | deg | Latitude | position block |
|
||||
| `altitude_m` | double | m | Altitude when available | extension or other protocol |
|
||||
| `direction_deg` | double | deg | Direction when available | extension or other protocol |
|
||||
| `location_status_raw` | long | - | Raw location status bits | position block `statusFlag` |
|
||||
|
||||
## Battery And Electricity
|
||||
|
||||
| Internal field | Type | Unit | Meaning | GB/T 32960 source |
|
||||
|---|---:|---|---|---|
|
||||
| `battery_soc` | double | % | Battery SOC | vehicle block `socPercent` |
|
||||
| `battery_voltage_v` | double | V | Battery total voltage | vehicle block `totalVoltageV` |
|
||||
| `battery_current_a` | double | A | Battery total current | vehicle block `totalCurrentA` |
|
||||
| `battery_power_kw` | double | kW | Derived instantaneous battery power | `voltage * current / 1000` in statistics |
|
||||
| `battery_temperature_max_c` | integer | C | Max battery temperature | temperature or extreme blocks |
|
||||
| `battery_temperature_min_c` | integer | C | Min battery temperature | temperature or extreme blocks |
|
||||
| `daily_electricity_kwh` | double | kWh | Daily electricity usage | statistics time integral |
|
||||
|
||||
Daily electricity must be computed in the statistics layer from typed telemetry
|
||||
points. Do not accumulate protocol raw values directly.
|
||||
|
||||
## Hydrogen And Fuel Cell
|
||||
|
||||
| Internal field | Type | Unit | Meaning | GB/T 32960 source |
|
||||
|---|---:|---|---|---|
|
||||
| `fc_voltage_v` | double | V | Fuel cell voltage | fuel cell block |
|
||||
| `fc_current_a` | double | A | Fuel cell current | fuel cell block |
|
||||
| `fc_temp_c` | double | C | Fuel cell or hydrogen system temperature | fuel cell block |
|
||||
| `hydrogen_remaining_kg` | double | kg | Remaining hydrogen mass | vendor extension or capacity conversion |
|
||||
| `hydrogen_remaining_percent` | double | % | Remaining hydrogen percent | V2025 fuel cell block |
|
||||
| `hydrogen_high_pressure_mpa` | double | MPa | Hydrogen high pressure | fuel cell block |
|
||||
| `hydrogen_low_pressure_mpa` | double | MPa | Hydrogen low pressure when available | extension or other protocol |
|
||||
| `daily_hydrogen_kg` | double | kg | Daily hydrogen consumed | statistics layer |
|
||||
| `hydrogen_mileage_efficiency` | double | km/kg | Mileage per kg hydrogen | statistics layer |
|
||||
|
||||
Daily hydrogen usage should be calculated from adjacent hydrogen remaining
|
||||
values. A rising value indicates refueling and must not be counted as negative
|
||||
consumption.
|
||||
|
||||
## Hydrogen Tank Safety
|
||||
|
||||
| Internal field | Type | Unit | Meaning | GB/T 32960 source |
|
||||
|---|---:|---|---|---|
|
||||
| `safety_category` | enum | - | `GENERAL`, `TANK_PRESSURE`, `TANK_TEMPERATURE`, `HYDROGEN_LEAK` | alarm bit classification |
|
||||
| `hydrogen_leak_detected` | boolean | - | Whether hydrogen leak is detected | general alarm bit `HYDROGEN_LEAK` |
|
||||
| `hydrogen_leak_level` | enum | - | `NONE`, `WARNING`, `CRITICAL`, `UNKNOWN` | internal rule |
|
||||
| `hydrogen_leak_action_required` | boolean | - | Whether immediate handling is required | internal rule |
|
||||
| `tank_pressure_status` | enum | - | Normal, warning, critical | hydrogen pressure bit or threshold rule |
|
||||
| `tank_temperature_status` | enum | - | Normal, warning, critical | hydrogen temp bit or threshold rule |
|
||||
|
||||
Safety rule:
|
||||
|
||||
```text
|
||||
If HYDROGEN_LEAK is present:
|
||||
alarm.level = CRITICAL
|
||||
safety_category = HYDROGEN_LEAK
|
||||
hydrogen_leak_detected = true
|
||||
hydrogen_leak_level = CRITICAL
|
||||
hydrogen_leak_action_required = true
|
||||
```
|
||||
|
||||
Hydrogen leak overrides otherwise normal vehicle, speed, battery, or hydrogen
|
||||
state. It must be counted independently from generic alarms.
|
||||
|
||||
## Daily Operation Statistics
|
||||
|
||||
Daily statistics are generated by `stat_date + vin`.
|
||||
|
||||
| Internal field | Type | Unit | Meaning |
|
||||
|---|---:|---|---|
|
||||
| `stat_date` | date | - | Statistics date |
|
||||
| `vin` | string | - | Vehicle VIN |
|
||||
| `first_event_time` | instant | - | First valid telemetry time |
|
||||
| `last_event_time` | instant | - | Last valid telemetry time |
|
||||
| `online_minutes` | double | min | Online duration |
|
||||
| `running_minutes` | double | min | Running duration |
|
||||
| `daily_mileage_km` | double | km | Daily mileage |
|
||||
| `daily_electricity_kwh` | double | kWh | Daily electricity usage |
|
||||
| `daily_hydrogen_kg` | double | kg | Daily hydrogen usage |
|
||||
| `hydrogen_added_kg` | double | kg | Refueled hydrogen |
|
||||
| `km_per_kg_hydrogen` | double | km/kg | Hydrogen efficiency |
|
||||
| `kwh_per_100km` | double | kWh/100km | Electricity efficiency |
|
||||
| `alarm_count` | integer | - | Generic alarm count |
|
||||
| `tank_safety_alarm_count` | integer | - | Tank safety alarm count |
|
||||
| `hydrogen_leak_alarm_count` | integer | - | Hydrogen leak alarm count |
|
||||
| `hydrogen_leak_duration_seconds` | long | s | Hydrogen leak duration |
|
||||
| `data_quality_level` | enum | - | Good, partial, bad |
|
||||
|
||||
## Implementation Status
|
||||
|
||||
- `RealtimePayload` already uses internal operational field names for state,
|
||||
location, battery, fuel cell, and hydrogen fields.
|
||||
- `AlarmPayload` carries internal hydrogen safety fields.
|
||||
- `Gb32960EventMapper` maps GB/T 32960 alarm bits into internal safety fields.
|
||||
- `TelemetrySnapshot` publishes full-field internal telemetry values through the
|
||||
Kafka protobuf envelope.
|
||||
- `event-history-service`, `vehicle-state-service`, and `vehicle-stat-service`
|
||||
consume the same full-field telemetry snapshot contract.
|
||||
- `event-history-service` CSV export can flatten selected internal fields with
|
||||
`fields=field_a,field_b`; media archive references are exposed through
|
||||
`rawArchiveUri` by falling back to `MediaMeta.archiveRef`.
|
||||
Reference in New Issue
Block a user