fix: support tdengine gb32960 raw queries
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

This commit is contained in:
lingniu
2026-06-29 20:19:44 +08:00
parent 12de83e37f
commit 80e96d8075
13 changed files with 398 additions and 51 deletions

View File

@@ -8,6 +8,13 @@
三个服务必须使用同一个 `SINK_ARCHIVE_PATH`。协议接入服务负责落原始 `.bin` 文件Kafka raw envelope 只携带 `archive://...` 引用history 服务按同一个 archive root 读取原始帧并提供查询。
`vehicle-history-app` 的生产高吞吐默认配置是 TDengine-only
- `EVENT_FILE_STORE_ENABLED=false`:避免 DuckDB/EventFileStore 进入实时消费热路径。
- `TDENGINE_TELEMETRY_FIELDS_ENABLED=false`:默认只写 `raw_frames``vehicle_locations`,避免逐字段写放大。
- GB32960 snapshot/fields 接口默认通过 TDengine `raw_frames` 索引找到 `rawUri`,再回读共享 archive 中的原始 `.bin` 即时解析;后续解析器新增字段后可直接查询历史 raw。
- 如果需要兼容旧 `/api/event-history/records` 低层记录接口或逐字段趋势宽表,可以显式打开对应开关;开启后要重新压测 Kafka lag 和 TDengine 写入吞吐。
## 生成 plist
在仓库根目录执行:
@@ -48,6 +55,13 @@ done
如果生产配置由 Nacos 下发,可以把模板里的 `NACOS_CONFIG_ENABLED` 改成 `true`,并补充 `NACOS_SERVER_ADDR``NACOS_NAMESPACE``NACOS_GROUP``NACOS_USERNAME``NACOS_PASSWORD`。端口、Kafka topic、archive root、TDengine 连接建议仍保留为启动环境变量,便于 Portainer、launchctl 和临时压测保持一致。
history 消费者会按协议和 raw/event 自动拆分 consumer group。设置 `KAFKA_GROUP_HISTORY=vehicle-history-live` 后,实际 group 为:
- `vehicle-history-live-gb32960-event`
- `vehicle-history-live-gb32960-raw`
- `vehicle-history-live-jt808-event`
- `vehicle-history-live-jt808-raw`
JT808 身份绑定默认使用内存或文件。需要把注册信息维护到 MySQL 时,将 `com.lingniu.jt808.plist.template` 里的 `VEHICLE_IDENTITY_STORE` 改为 `mysql`,并提供 `VEHICLE_IDENTITY_MYSQL_*` 连接参数。服务会自动创建 `vehicle_identity_binding``vehicle_identity_binding_registration` 两张表:前者保存已确认 VIN 的外部标识绑定,后者保存 0x0100 注册字段;外部系统可把真实 VIN 反写到 registration 表,服务按刷新周期加载后,后续 raw/event 会带真实 VIN。
## 启停

View File

@@ -20,6 +20,8 @@
<string>32960</string>
<key>GB32960_AUTH_ENABLED</key>
<string>false</string>
<key>GB32960_PLATFORM_IP_HYUNDAI</key>
<string>115.29.187.205</string>
<key>KAFKA_ENABLED</key>
<string>true</string>
<key>KAFKA_BROKERS</key>

View File

@@ -26,6 +26,8 @@
<string>vehicle-history-live</string>
<key>KAFKA_GROUP_HISTORY</key>
<string>vehicle-history-live</string>
<key>KAFKA_CONSUMER_CONCURRENCY</key>
<string>3</string>
<key>KAFKA_BROKERS</key>
<string>114.55.58.251:9092</string>
<key>KAFKA_TOPIC_GB32960_EVENT</key>
@@ -37,7 +39,7 @@
<key>KAFKA_TOPIC_JT808_RAW</key>
<string>vehicle.raw.jt808.v1</string>
<key>EVENT_FILE_STORE_ENABLED</key>
<string>true</string>
<string>false</string>
<key>EVENT_FILE_STORE_PATH</key>
<string>__EVENT_STORE_PATH__</string>
<key>EVENT_FILE_STORE_ZONE_ID</key>
@@ -62,6 +64,8 @@
<string>16</string>
<key>TDENGINE_INITIALIZE_SCHEMA</key>
<string>true</string>
<key>TDENGINE_TELEMETRY_FIELDS_ENABLED</key>
<string>false</string>
<key>NACOS_CONFIG_ENABLED</key>
<string>false</string>
<key>MANAGEMENT_HEALTH_REDIS_ENABLED</key>

View File

@@ -56,7 +56,7 @@ services:
restart: unless-stopped
environment:
<<: *common-env
HTTP_PORT: 20400
HTTP_PORT: 20482
JT808_PORT: 808
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer}
@@ -69,7 +69,7 @@ services:
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
ports:
- "${JT808_HTTP_PORT:-20400}:20400"
- "${JT808_HTTP_PORT:-20482}:20482"
- "${JT808_TCP_PORT:-808}:808"
volumes:
- vehicle-history-archive:/archive
@@ -89,20 +89,25 @@ services:
HTTP_PORT: 20200
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history}
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
KAFKA_CONSUMER_CONCURRENCY: ${KAFKA_CONSUMER_CONCURRENCY:-3}
SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true}
SINK_ARCHIVE_PATH: /archive/
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true}
# 高吞吐生产热路径默认只写 TDengine。需要 GB32960 snapshot/fields 旧查询时再显式开启。
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-false}
EVENT_FILE_STORE_PATH: /event-store/
EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai}
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_history}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-RS://tdengine:6041/vehicle_history}
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://tdengine:6041/vehicle_ts}
TDENGINE_DRIVER_CLASS_NAME: ${TDENGINE_DRIVER_CLASS_NAME:-com.taosdata.jdbc.ws.WebSocketDriver}
TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root}
TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata}
TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32}
TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-0}
TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-5000}
TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:--1}
# 默认关闭逐字段宽表,避免 telemetry_fields 写放大;需要字段趋势查询时显式打开。
TDENGINE_TELEMETRY_FIELDS_ENABLED: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:-false}
ports:
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
volumes: