feat: productionize raw history ingestion

This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions

View File

@@ -10,6 +10,6 @@ ENV JAVA_OPTS=""
WORKDIR /app WORKDIR /app
COPY modules/apps/${APP_NAME}/target/${APP_NAME}.jar /app/app.jar COPY modules/apps/${APP_NAME}/target/${APP_NAME}.jar /app/app.jar
EXPOSE 20100 20200 20300 20400 32960 10808 EXPOSE 20100 20200 20300 20400 20500 20600 32960 10808
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app/app.jar"] ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app/app.jar"]

View File

@@ -4,7 +4,7 @@ version: "3.8"
# main-0.1.0-SNAPSHOT-22468e7 # main-0.1.0-SNAPSHOT-22468e7
x-common-env: &common-env x-common-env: &common-env
TZ: Asia/Shanghai TZ: Asia/Shanghai
JAVA_OPTS: ${JAVA_OPTS:--Xms128m -Xmx768m -Duser.timezone=Asia/Shanghai} JAVA_OPTS: ${JAVA_OPTS:--XX:+UseZGC -XX:MaxRAMPercentage=75}
NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true} NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true}
NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848} NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848}
NACOS_NAMESPACE: ${NACOS_NAMESPACE:-} NACOS_NAMESPACE: ${NACOS_NAMESPACE:-}
@@ -13,32 +13,33 @@ x-common-env: &common-env
NACOS_REFRESH_ENABLED: ${NACOS_REFRESH_ENABLED:-true} NACOS_REFRESH_ENABLED: ${NACOS_REFRESH_ENABLED:-true}
NACOS_USERNAME: ${NACOS_USERNAME:-} NACOS_USERNAME: ${NACOS_USERNAME:-}
NACOS_PASSWORD: ${NACOS_PASSWORD:-} NACOS_PASSWORD: ${NACOS_PASSWORD:-}
KAFKA_BROKERS: ${KAFKA_BROKERS:-114.55.58.251:9092} KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
KAFKA_ENABLED: ${KAFKA_ENABLED:-true} KAFKA_ENABLED: ${KAFKA_ENABLED:-true}
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true} KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true}
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:-900000}
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1} KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1} KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1} KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1} KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1} KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1} KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1} KAFKA_TOPIC_MQTT_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1} KAFKA_TOPIC_MQTT_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1} KAFKA_TOPIC_MQTT_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
KAFKA_TOPIC_XINDA_PUSH_EVENT: ${KAFKA_TOPIC_XINDA_PUSH_EVENT:-vehicle.event.xinda-push.v1} KAFKA_TOPIC_XINDA_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1}
KAFKA_TOPIC_XINDA_PUSH_RAW: ${KAFKA_TOPIC_XINDA_PUSH_RAW:-vehicle.raw.xinda-push.v1} KAFKA_TOPIC_XINDA_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1}
KAFKA_TOPIC_XINDA_PUSH_DLQ: ${KAFKA_TOPIC_XINDA_PUSH_DLQ:-vehicle.dlq.xinda-push.v1} KAFKA_TOPIC_XINDA_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1}
MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
services: services:
gb32960-ingest-app: gb32960-ingest-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: gb32960-ingest-app container_name: gb32960-ingest-app
restart: unless-stopped restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment: environment:
<<: *common-env <<: *common-env
JAVA_OPTS: ${JAVA_OPTS_GB32960:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20100 HTTP_PORT: 20100
GB32960_PORT: 32960 GB32960_PORT: 32960
KAFKA_CONSUMER_ENABLED: "false" KAFKA_CONSUMER_ENABLED: "false"
@@ -48,13 +49,11 @@ services:
GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-} GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}
GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-} GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}
GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false} GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
ports: ports:
- "${GB32960_HTTP_PORT:-20100}:20100" - "${GB32960_HTTP_PORT:-20100}:20100"
- "${GB32960_TCP_PORT:-32960}:32960" - "${GB32960_TCP_PORT:-32960}:32960"
volumes: volumes:
- vehicle-history-archive:/archive
- gb32960-ingest-data:/data - gb32960-ingest-data:/data
networks: networks:
- vehicle-ingest - vehicle-ingest
@@ -63,109 +62,82 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: jt808-ingest-app container_name: jt808-ingest-app
restart: unless-stopped restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment: environment:
<<: *common-env <<: *common-env
JAVA_OPTS: ${JAVA_OPTS_JT808:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai} HTTP_PORT: 20400
HTTP_PORT: 20482
JT808_PORT: 808 JT808_PORT: 808
KAFKA_CONSUMER_ENABLED: "false" KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer} KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer}
SINK_ARCHIVE_PATH: /archive/ VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file} VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding} VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
ports: ports:
- "${JT808_HTTP_PORT:-20482}:20482" - "${JT808_HTTP_PORT:-20400}:20400"
- "${JT808_TCP_PORT:-808}:808" - "${JT808_TCP_PORT:-808}:808"
volumes: volumes:
- vehicle-history-archive:/archive
- jt808-ingest-data:/data - jt808-ingest-data:/data
networks: networks:
- vehicle-ingest - vehicle-ingest
yutong-mqtt-app: yutong-mqtt-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: yutong-mqtt-app container_name: yutong-mqtt-app
restart: unless-stopped restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment: environment:
<<: *common-env <<: *common-env
JAVA_OPTS: ${JAVA_OPTS_YUTONG_MQTT:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20500 HTTP_PORT: 20500
MQTT_ENABLED: "true"
XINDA_PUSH_ENABLED: "false"
KAFKA_CONSUMER_ENABLED: "false" KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_YUTONG_MQTT:-yutong-mqtt-portainer} KAFKA_NODE_ID: ${KAFKA_NODE_ID_MQTT:-yutong-mqtt-portainer}
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false} KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
YUTONG_MQTT_ENDPOINT_NAME: ${YUTONG_MQTT_ENDPOINT_NAME:-yutong} KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-} KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
YUTONG_MQTT_TOPIC: ${YUTONG_MQTT_TOPIC:-#} VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-1} VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-yutong-mqtt} VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-} VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-} MQTT_ENDPOINT_NAME: ${MQTT_ENDPOINT_NAME:-yutong}
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false} MQTT_URI: ${MQTT_URI:-tcp://127.0.0.1:1883}
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20} MQTT_TOPIC: "${MQTT_TOPIC:-#}"
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10} MQTT_QOS: ${MQTT_QOS:-1}
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-} MQTT_CLIENT_ID: ${MQTT_CLIENT_ID:-lingniu-yutong-mqtt}
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-} MQTT_USERNAME: ${MQTT_USERNAME:-}
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-} MQTT_PASSWORD: ${MQTT_PASSWORD:-}
SINK_ARCHIVE_PATH: /archive/ MQTT_CLEAN_SESSION: ${MQTT_CLEAN_SESSION:-true}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file} MQTT_PROFILE: ${MQTT_PROFILE:-yutong}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
ports: ports:
- "${YUTONG_MQTT_HTTP_PORT:-20500}:20500" - "${YUTONG_MQTT_HTTP_PORT:-20500}:20500"
volumes:
- vehicle-history-archive:/archive
- yutong-mqtt-data:/data
networks: networks:
- vehicle-ingest - vehicle-ingest
xinda-push-app: xinda-push-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/xinda-push-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: xinda-push-app container_name: xinda-push-app
restart: unless-stopped restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment: environment:
<<: *common-env <<: *common-env
JAVA_OPTS: ${JAVA_OPTS_XINDA_PUSH:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20600 HTTP_PORT: 20600
MQTT_ENABLED: "false"
XINDA_PUSH_ENABLED: "true"
KAFKA_CONSUMER_ENABLED: "false" KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA_PUSH:-xinda-push-portainer} KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA:-xinda-push-portainer}
XINDA_PUSH_ENABLED: ${XINDA_PUSH_ENABLED:-false} KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1}
KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1}
KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
XINDA_PUSH_HOST: ${XINDA_PUSH_HOST:-} XINDA_PUSH_HOST: ${XINDA_PUSH_HOST:-}
XINDA_PUSH_PORT: ${XINDA_PUSH_PORT:-10100} XINDA_PUSH_PORT: ${XINDA_PUSH_PORT:-10100}
XINDA_PUSH_USERNAME: ${XINDA_PUSH_USERNAME:-} XINDA_PUSH_USERNAME: ${XINDA_PUSH_USERNAME:-}
XINDA_PUSH_PASSWORD: ${XINDA_PUSH_PASSWORD:-} XINDA_PUSH_PASSWORD: ${XINDA_PUSH_PASSWORD:-}
XINDA_PUSH_CLIENT_DESC: ${XINDA_PUSH_CLIENT_DESC:-lingniu-ingest} XINDA_PUSH_CLIENT_DESC: ${XINDA_PUSH_CLIENT_DESC:-lingniu-ingest}
XINDA_PUSH_MSG_ID_LOCATION: ${XINDA_PUSH_MSG_ID_LOCATION:-0200} XINDA_PUSH_SUBSCRIBE_MSG_IDS: "${XINDA_PUSH_SUBSCRIBE_MSG_IDS:-0200,0300,0401}"
XINDA_PUSH_MSG_ID_ALARM: ${XINDA_PUSH_MSG_ID_ALARM:-0300}
XINDA_PUSH_MSG_ID_PASSTHROUGH: ${XINDA_PUSH_MSG_ID_PASSTHROUGH:-0401}
XINDA_PUSH_RECONNECT_INTERVAL_SEC: ${XINDA_PUSH_RECONNECT_INTERVAL_SEC:-5}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
ports: ports:
- "${XINDA_PUSH_HTTP_PORT:-20600}:20600" - "${XINDA_PUSH_HTTP_PORT:-20600}:20600"
volumes:
- vehicle-history-archive:/archive
- xinda-push-data:/data
networks: networks:
- vehicle-ingest - vehicle-ingest
@@ -173,8 +145,6 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: vehicle-history-app container_name: vehicle-history-app
restart: unless-stopped restart: unless-stopped
mem_limit: 2560m
memswap_limit: 2560m
depends_on: depends_on:
- gb32960-ingest-app - gb32960-ingest-app
- jt808-ingest-app - jt808-ingest-app
@@ -182,29 +152,28 @@ services:
- xinda-push-app - xinda-push-app
environment: environment:
<<: *common-env <<: *common-env
JAVA_OPTS: ${JAVA_OPTS_HISTORY:--Xms512m -Xmx1536m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20200 HTTP_PORT: 20200
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history} KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history}
KAFKA_CONSUMER_MAX_POLL_RECORDS: ${KAFKA_CONSUMER_MAX_POLL_RECORDS_HISTORY:-2000}
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history} KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
KAFKA_CONSUMER_CONCURRENCY: ${KAFKA_CONSUMER_CONCURRENCY:-3} KAFKA_GROUP_HISTORY_GB32960: ${KAFKA_GROUP_HISTORY_GB32960:-vehicle-history}
KAFKA_GROUP_HISTORY_JT808: ${KAFKA_GROUP_HISTORY_JT808:-vehicle-history}
KAFKA_GROUP_HISTORY_MQTT_YUTONG: ${KAFKA_GROUP_HISTORY_MQTT_YUTONG:-vehicle-history}
KAFKA_GROUP_HISTORY_XINDA: ${KAFKA_GROUP_HISTORY_XINDA:-vehicle-history}
SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true} SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true}
SINK_ARCHIVE_PATH: /archive/ SINK_ARCHIVE_PATH: /archive/
# 高吞吐生产热路径默认只写 TDengine。需要 GB32960 snapshot/fields 旧查询时再显式开启。 EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true}
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-false}
EVENT_FILE_STORE_PATH: /event-store/ EVENT_FILE_STORE_PATH: /event-store/
EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai} EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai}
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true} EVENT_FILE_STORE_BATCH_SIZE: ${EVENT_FILE_STORE_BATCH_SIZE:-500}
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts} EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS: ${EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS:-1000}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://tdengine:6041/vehicle_ts} EVENT_FILE_STORE_WRITE_PARQUET_ENABLED: ${EVENT_FILE_STORE_WRITE_PARQUET_ENABLED:-false}
TDENGINE_DRIVER_CLASS_NAME: ${TDENGINE_DRIVER_CLASS_NAME:-com.taosdata.jdbc.ws.WebSocketDriver} TDENGINE_ENABLED: ${TDENGINE_ENABLED:-true}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts}
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-vehicle_ts}
TDENGINE_JT808_LOCATION_STABLE: ${TDENGINE_JT808_LOCATION_STABLE:-jt808_locations}
TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root} TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root}
TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata} TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata}
TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32}
TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-0}
TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-5000}
TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:--1}
# 默认关闭逐字段宽表,避免 telemetry_fields 写放大;需要字段趋势查询时显式打开。
TDENGINE_TELEMETRY_FIELDS_ENABLED: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:-false}
ports: ports:
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200" - "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
volumes: volumes:
@@ -217,16 +186,11 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: vehicle-analytics-app container_name: vehicle-analytics-app
restart: unless-stopped restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
depends_on: depends_on:
- gb32960-ingest-app - gb32960-ingest-app
- jt808-ingest-app - jt808-ingest-app
- yutong-mqtt-app
- xinda-push-app
environment: environment:
<<: *common-env <<: *common-env
JAVA_OPTS: ${JAVA_OPTS_ANALYTICS:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20300 HTTP_PORT: 20300
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics} KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics}
KAFKA_GROUP_STATE: ${KAFKA_GROUP_STATE:-vehicle-state} KAFKA_GROUP_STATE: ${KAFKA_GROUP_STATE:-vehicle-state}
@@ -244,6 +208,7 @@ services:
REDIS_HOST: ${REDIS_HOST:-} REDIS_HOST: ${REDIS_HOST:-}
REDIS_PORT: ${REDIS_PORT:-6379} REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DATABASE: ${REDIS_DATABASE:-50} REDIS_DATABASE: ${REDIS_DATABASE:-50}
REDIS_USERNAME: ${REDIS_USERNAME:-}
REDIS_PASSWORD: ${REDIS_PASSWORD:-} REDIS_PASSWORD: ${REDIS_PASSWORD:-}
ports: ports:
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300" - "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300"
@@ -259,8 +224,6 @@ networks:
volumes: volumes:
gb32960-ingest-data: gb32960-ingest-data:
jt808-ingest-data: jt808-ingest-data:
yutong-mqtt-data:
xinda-push-data:
vehicle-history-archive: vehicle-history-archive:
vehicle-history-event-store: vehicle-history-event-store:
vehicle-analytics-stat: vehicle-analytics-stat:

View File

@@ -0,0 +1,92 @@
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.admin.NewPartitions;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.admin.TopicDescription;
public final class EnsureKafkaTopics {
private static final short REPLICATION_FACTOR = 1;
private record TopicSpec(String name, int partitions) {}
private EnsureKafkaTopics() {}
public static void main(String[] args) throws Exception {
String brokers = args.length > 0 ? args[0] : "172.17.111.56:9092";
List<TopicSpec> specs =
List.of(
topic("vehicle.event.gb32960.v1", 12),
topic("vehicle.raw.gb32960.v1", 12),
topic("vehicle.dlq.gb32960.v1", 3),
topic("vehicle.event.jt808.v1", 12),
topic("vehicle.raw.jt808.v1", 12),
topic("vehicle.dlq.jt808.v1", 3),
topic("vehicle.event.mqtt-yutong.v1", 12),
topic("vehicle.raw.mqtt-yutong.v1", 12),
topic("vehicle.dlq.mqtt-yutong.v1", 3),
topic("vehicle.event.xinda.v1", 12),
topic("vehicle.raw.xinda.v1", 12),
topic("vehicle.dlq.xinda.v1", 3),
topic("vehicle.event.xinda-push.v1", 12),
topic("vehicle.raw.xinda-push.v1", 12),
topic("vehicle.dlq.xinda-push.v1", 3));
Properties props = new Properties();
props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);
props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "15000");
props.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "30000");
try (AdminClient admin = AdminClient.create(props)) {
Set<String> existing = admin.listTopics().names().get();
List<NewTopic> missing =
specs.stream()
.filter(spec -> !existing.contains(spec.name()))
.map(spec -> new NewTopic(spec.name(), spec.partitions(), REPLICATION_FACTOR))
.toList();
if (!missing.isEmpty()) {
try {
admin.createTopics(missing).all().get();
} catch (ExecutionException e) {
System.out.println("createTopics result=" + e.getCause());
}
}
Map<String, TopicSpec> byName =
specs.stream().collect(Collectors.toMap(TopicSpec::name, spec -> spec));
Map<String, TopicDescription> descriptions =
admin.describeTopics(byName.keySet()).allTopicNames().get();
Map<String, NewPartitions> increases =
descriptions.entrySet().stream()
.filter(entry -> entry.getValue().partitions().size() < byName.get(entry.getKey()).partitions())
.collect(
Collectors.toMap(
Map.Entry::getKey,
entry -> NewPartitions.increaseTo(byName.get(entry.getKey()).partitions())));
if (!increases.isEmpty()) {
admin.createPartitions(increases).all().get();
Thread.sleep(Duration.ofSeconds(2).toMillis());
descriptions = admin.describeTopics(byName.keySet()).allTopicNames().get();
}
List<String> lines = new ArrayList<>();
descriptions.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.forEach(entry -> lines.add(entry.getKey() + " partitions=" + entry.getValue().partitions().size()));
System.out.println("bootstrap=" + brokers);
lines.forEach(System.out::println);
}
}
private static TopicSpec topic(String name, int partitions) {
return new TopicSpec(name, partitions);
}
}

View File

@@ -52,12 +52,11 @@ lingniu:
file: file:
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl} path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
mysql: mysql:
table: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding} jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:} username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:}
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:} password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
driver-class-name: ${VEHICLE_IDENTITY_MYSQL_DRIVER_CLASS_NAME:com.mysql.cj.jdbc.Driver} table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
refresh-interval: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s} initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink: sink:
mq: mq:
enabled: ${KAFKA_ENABLED:true} enabled: ${KAFKA_ENABLED:true}
@@ -79,10 +78,6 @@ lingniu:
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1} dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
consumer: consumer:
enabled: false enabled: false
archive:
enabled: ${SINK_ARCHIVE_ENABLED:true}
type: local
path: ${SINK_ARCHIVE_PATH:./archive/}
event-file-store: event-file-store:
enabled: false enabled: false
event-history: event-history:

View File

@@ -29,12 +29,14 @@ class Jt808IngestAppDefaultsTest {
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}") .containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}") .containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}") .containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}")
.containsEntry("lingniu.ingest.identity.mysql.table", "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}") .containsEntry("lingniu.ingest.identity.mysql.table-name",
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url", "${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}") "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
.containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:}") .containsEntry("lingniu.ingest.identity.mysql.jdbc-url",
.containsEntry("lingniu.ingest.identity.mysql.refresh-interval", "${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}") "${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}")
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}") .containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:root}")
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}") .containsEntry("lingniu.ingest.identity.mysql.password", "${VEHICLE_IDENTITY_MYSQL_PASSWORD:}")
.containsEntry("lingniu.ingest.identity.mysql.initialize-schema",
"${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}")
.containsEntry("lingniu.ingest.event-file-store.enabled", false) .containsEntry("lingniu.ingest.event-file-store.enabled", false)
.containsEntry("lingniu.ingest.event-history.enabled", false) .containsEntry("lingniu.ingest.event-history.enabled", false)
.containsEntry("lingniu.ingest.vehicle-state.enabled", false) .containsEntry("lingniu.ingest.vehicle-state.enabled", false)

View File

@@ -33,6 +33,10 @@
<groupId>com.lingniu.ingest</groupId> <groupId>com.lingniu.ingest</groupId>
<artifactId>vehicle-stat-service</artifactId> <artifactId>vehicle-stat-service</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>

View File

@@ -26,6 +26,7 @@ spring:
redis: redis:
host: ${REDIS_HOST:} host: ${REDIS_HOST:}
port: ${REDIS_PORT:6379} port: ${REDIS_PORT:6379}
username: ${REDIS_USERNAME:}
password: ${REDIS_PASSWORD:} password: ${REDIS_PASSWORD:}
database: ${REDIS_DATABASE:50} database: ${REDIS_DATABASE:50}
@@ -55,6 +56,7 @@ lingniu:
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics} client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest} auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:500} max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:500}
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:900000}
bindings: bindings:
vehicleStateEnvelopeConsumerProcessor: vehicleStateEnvelopeConsumerProcessor:
enabled: ${VEHICLE_STATE_ENABLED:false} enabled: ${VEHICLE_STATE_ENABLED:false}

View File

@@ -46,7 +46,7 @@ lingniu:
enabled: ${KAFKA_CONSUMER_ENABLED:true} enabled: ${KAFKA_CONSUMER_ENABLED:true}
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history} client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest} auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:100} max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000} max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}
concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3} concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3}
bindings: bindings:

View File

@@ -43,7 +43,7 @@ class VehicleHistoryAppDefaultsTest {
.containsEntry("lingniu.ingest.vehicle-state.enabled", false) .containsEntry("lingniu.ingest.vehicle-state.enabled", false)
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false) .containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}") .containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}") .containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
.containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}") .containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
.containsEntry( .containsEntry(
"lingniu.ingest.sink.mq.consumer.max-poll-interval-millis", "lingniu.ingest.sink.mq.consumer.max-poll-interval-millis",

View File

@@ -0,0 +1,12 @@
package com.lingniu.ingest.api.consumer;
import java.util.List;
/**
* Optional extension for consumers that can persist a Kafka poll batch in one
* storage operation.
*/
public interface EnvelopeBatchIngestor extends EnvelopeIngestor {
List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues);
}

View File

@@ -70,6 +70,32 @@ public final class EnvelopeConsumerProcessor {
} }
} }
public List<EnvelopeIngestResult> processBatch(List<EnvelopeConsumerRecord> records) {
if (records == null || records.isEmpty()) {
return List.of();
}
if (!(ingestor instanceof EnvelopeBatchIngestor batchIngestor)) {
return records.stream().map(this::process).toList();
}
List<byte[]> payloads = records.stream()
.map(EnvelopeConsumerRecord::payload)
.toList();
List<EnvelopeIngestResult> results = batchIngestor.tryIngestAll(payloads);
if (results.size() != records.size()) {
throw new IllegalStateException("batch ingestor returned " + results.size()
+ " results for " + records.size() + " records");
}
List<EnvelopeIngestResult> copy = new ArrayList<>(results.size());
for (int i = 0; i < records.size(); i++) {
EnvelopeIngestResult result = results.get(i);
if (DEAD_LETTER_STATUSES.contains(result.status())) {
deadLetterSink.publish(toDeadLetter(records.get(i), result));
}
copy.add(result);
}
return List.copyOf(copy);
}
private EnvelopeDeadLetterRecord toDeadLetter(EnvelopeConsumerRecord record, EnvelopeIngestResult result) { private EnvelopeDeadLetterRecord toDeadLetter(EnvelopeConsumerRecord record, EnvelopeIngestResult result) {
return new EnvelopeDeadLetterRecord( return new EnvelopeDeadLetterRecord(
service, service,

View File

@@ -19,6 +19,7 @@ public final class RawArchiveKeys {
public static final String META_EVENT_ID = "rawArchiveEventId"; public static final String META_EVENT_ID = "rawArchiveEventId";
public static final String META_KEY = "rawArchiveKey"; public static final String META_KEY = "rawArchiveKey";
public static final String META_URI = "rawArchiveUri"; public static final String META_URI = "rawArchiveUri";
public static final String META_PARSED_JSON = "rawArchiveParsedJson";
private static final DateTimeFormatter DATE_KEY = private static final DateTimeFormatter DATE_KEY =
DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneId.of("Asia/Shanghai")); DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneId.of("Asia/Shanghai"));

View File

@@ -150,6 +150,7 @@ public sealed interface VehicleEvent
* @param command 协议主命令码(如 32960 0x02/0x03 等),冗余在事件里便于按命令分片归档 * @param command 协议主命令码(如 32960 0x02/0x03 等),冗余在事件里便于按命令分片归档
* @param infoType 协议子类型(可为 0同上 * @param infoType 协议子类型(可为 0同上
* @param rawBytes 原始入站字节,不可为 null * @param rawBytes 原始入站字节,不可为 null
* @param parsedJson 已解析结构化 JSON。只用于 RAW 明细查询,不应复制到位置/字段等派生表。
*/ */
record RawArchive( record RawArchive(
String eventId, String eventId,
@@ -161,6 +162,25 @@ public sealed interface VehicleEvent
Map<String, String> metadata, Map<String, String> metadata,
int command, int command,
int infoType, int infoType,
byte[] rawBytes byte[] rawBytes,
) implements VehicleEvent {} String parsedJson
) implements VehicleEvent {
public RawArchive(String eventId,
String vin,
ProtocolId source,
Instant eventTime,
Instant ingestTime,
String traceId,
Map<String, String> metadata,
int command,
int infoType,
byte[] rawBytes) {
this(eventId, vin, source, eventTime, ingestTime, traceId, metadata,
command, infoType, rawBytes, "");
}
public RawArchive {
parsedJson = parsedJson == null ? "" : parsedJson;
}
}
} }

View File

@@ -24,6 +24,14 @@
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId> <artifactId>spring-boot-starter</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId> <artifactId>spring-boot-autoconfigure</artifactId>

View File

@@ -1,5 +1,8 @@
package com.lingniu.ingest.core.dispatcher; package com.lingniu.ingest.core.dispatcher;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.lingniu.ingest.api.event.RawArchiveKeys; import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.event.VehicleEvent; import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.api.pipeline.IngestContext; import com.lingniu.ingest.api.pipeline.IngestContext;
@@ -36,6 +39,9 @@ public final class Dispatcher {
private static final Logger log = LoggerFactory.getLogger(Dispatcher.class); private static final Logger log = LoggerFactory.getLogger(Dispatcher.class);
private static final AtomicLong RAW_ARCHIVE_SEQUENCE = new AtomicLong(); private static final AtomicLong RAW_ARCHIVE_SEQUENCE = new AtomicLong();
private static final String REQUIRED_DURABLE_SINK = "kafka"; private static final String REQUIRED_DURABLE_SINK = "kafka";
private static final ObjectMapper RAW_ARCHIVE_JSON = JsonMapper.builder()
.findAndAddModules()
.build();
private final HandlerRegistry registry; private final HandlerRegistry registry;
private final InterceptorChain interceptors; private final InterceptorChain interceptors;
@@ -167,7 +173,10 @@ public final class Dispatcher {
byte[] bytes = frame.rawBytes(); byte[] bytes = frame.rawBytes();
if (bytes == null || bytes.length == 0) return RawArchiveLookup.empty(); if (bytes == null || bytes.length == 0) return RawArchiveLookup.empty();
Map<String, String> meta = frame.sourceMeta() == null ? Map.of() : frame.sourceMeta(); Map<String, String> meta = frame.sourceMeta() == null
? Map.of()
: new LinkedHashMap<>(frame.sourceMeta());
String parsedJson = parsedJson(frame.payload(), meta.remove(RawArchiveKeys.META_PARSED_JSON));
String vin = meta.getOrDefault("vin", ""); String vin = meta.getOrDefault("vin", "");
Instant ingestTime = frame.receivedAt() != null ? frame.receivedAt() : Instant.now(); Instant ingestTime = frame.receivedAt() != null ? frame.receivedAt() : Instant.now();
String eventId = nextRawArchiveEventId(ingestTime); String eventId = nextRawArchiveEventId(ingestTime);
@@ -184,11 +193,34 @@ public final class Dispatcher {
archiveMeta, archiveMeta,
frame.command(), frame.command(),
frame.infoType(), frame.infoType(),
bytes); bytes,
parsedJson);
out.add(raw); out.add(raw);
return new RawArchiveLookup(eventId, key, RawArchiveKeys.logicalUri(key)); return new RawArchiveLookup(eventId, key, RawArchiveKeys.logicalUri(key));
} }
private static String parsedJson(Object payload, String explicitParsedJson) {
if (explicitParsedJson != null && !explicitParsedJson.isBlank()) {
return explicitParsedJson;
}
if (payload == null) {
return "{}";
}
try {
return RAW_ARCHIVE_JSON.writeValueAsString(payload);
} catch (JsonProcessingException ex) {
return "{\"serializationError\":\"" + escapeJson(ex.getMessage()) + "\",\"payloadType\":\""
+ escapeJson(payload.getClass().getName()) + "\"}";
}
}
private static String escapeJson(String value) {
if (value == null) {
return "";
}
return value.replace("\\", "\\\\").replace("\"", "\\\"");
}
private static String nextRawArchiveEventId(Instant ingestTime) { private static String nextRawArchiveEventId(Instant ingestTime) {
// 文件名需要可排序且单进程内唯一:毫秒时间放大到微秒量级,再用 AtomicLong 递增兜底。 // 文件名需要可排序且单进程内唯一:毫秒时间放大到微秒量级,再用 AtomicLong 递增兜底。
long base = (ingestTime == null ? Instant.now() : ingestTime).toEpochMilli() * 1000L; long base = (ingestTime == null ? Instant.now() : ingestTime).toEpochMilli() * 1000L;

View File

@@ -42,7 +42,7 @@ class DispatcherRawArchiveMetadataTest {
ProtocolId.JT808, ProtocolId.JT808,
0x0200, 0x0200,
0, 0,
new Object(), new SamplePayload("payload-0200", 7),
new byte[]{0x7e, 0x01, 0x7e}, new byte[]{0x7e, 0x01, 0x7e},
Map.of("vin", "VIN001", "peer", "127.0.0.1:10001"), Map.of("vin", "VIN001", "peer", "127.0.0.1:10001"),
Instant.parse("2026-06-22T08:00:00Z"))); Instant.parse("2026-06-22T08:00:00Z")));
@@ -52,6 +52,8 @@ class DispatcherRawArchiveMetadataTest {
String rawArchiveKey = raw.metadata().get("rawArchiveKey"); String rawArchiveKey = raw.metadata().get("rawArchiveKey");
assertThat(rawArchiveKey).isNotBlank(); assertThat(rawArchiveKey).isNotBlank();
assertThat(raw.parsedJson()).contains("\"name\":\"payload-0200\"");
assertThat(raw.metadata()).doesNotContainKey("rawArchiveParsedJson");
assertThat(raw.eventId()).containsOnlyDigits(); assertThat(raw.eventId()).containsOnlyDigits();
assertThat(rawArchiveKey).endsWith("/" + raw.eventId() + ".bin"); assertThat(rawArchiveKey).endsWith("/" + raw.eventId() + ".bin");
assertThat(location.metadata()) assertThat(location.metadata())
@@ -102,7 +104,7 @@ class DispatcherRawArchiveMetadataTest {
private static HandlerRegistry registryWithLocationHandler() throws NoSuchMethodException { private static HandlerRegistry registryWithLocationHandler() throws NoSuchMethodException {
LocationHandler bean = new LocationHandler(); LocationHandler bean = new LocationHandler();
Method method = LocationHandler.class.getDeclaredMethod("onLocation", Object.class); Method method = LocationHandler.class.getDeclaredMethod("onLocation", SamplePayload.class);
HandlerRegistry registry = new HandlerRegistry(); HandlerRegistry registry = new HandlerRegistry();
registry.register(new HandlerDefinition( registry.register(new HandlerDefinition(
ProtocolId.JT808, ProtocolId.JT808,
@@ -169,7 +171,7 @@ class DispatcherRawArchiveMetadataTest {
@ProtocolHandler(protocol = ProtocolId.JT808) @ProtocolHandler(protocol = ProtocolId.JT808)
static final class LocationHandler { static final class LocationHandler {
@MessageMapping(command = 0x0200) @MessageMapping(command = 0x0200)
VehicleEvent.Location onLocation(Object ignored) { VehicleEvent.Location onLocation(SamplePayload ignored) {
return new VehicleEvent.Location( return new VehicleEvent.Location(
"location-1", "location-1",
"VIN001", "VIN001",
@@ -182,6 +184,9 @@ class DispatcherRawArchiveMetadataTest {
} }
} }
record SamplePayload(String name, int value) {
}
@ProtocolHandler(protocol = ProtocolId.JT808) @ProtocolHandler(protocol = ProtocolId.JT808)
public static final class AsyncLocationHandler { public static final class AsyncLocationHandler {
@MessageMapping(command = 0x0201) @MessageMapping(command = 0x0201)

View File

@@ -24,6 +24,10 @@
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId> <artifactId>spring-boot-autoconfigure</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>

View File

@@ -1,70 +1,64 @@
package com.lingniu.ingest.identity; package com.lingniu.ingest.identity;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import org.slf4j.Logger; import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
import org.slf4j.LoggerFactory; import org.springframework.jdbc.core.ConnectionCallback;
import org.springframework.jdbc.core.JdbcTemplate;
import javax.sql.DataSource; import javax.sql.DataSource;
import java.time.Duration; import java.sql.DatabaseMetaData;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.concurrent.Executors; import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
public final class MySqlVehicleIdentityService implements VehicleIdentityResolver, VehicleIdentityRegistry, AutoCloseable { public final class MySqlVehicleIdentityService implements VehicleIdentityResolver, VehicleIdentityRegistry {
private static final Logger log = LoggerFactory.getLogger(MySqlVehicleIdentityService.class); private final JdbcTemplate jdbc;
private final ObjectMapper mapper;
private final String tableName;
private final DataSource dataSource; public MySqlVehicleIdentityService(DataSource dataSource,
private final String table; VehicleIdentityProperties.Mysql properties,
private final String registrationTable; ObjectMapper mapper) {
private final ScheduledExecutorService refresher; this.jdbc = new JdbcTemplate(dataSource);
private volatile InMemoryVehicleIdentityService index; this.mapper = mapper == null ? new ObjectMapper() : mapper;
VehicleIdentityProperties.Mysql mysql = properties == null ? new VehicleIdentityProperties.Mysql() : properties;
public MySqlVehicleIdentityService(DataSource dataSource, String table) { this.tableName = safeTableName(mysql.getTableName());
this(dataSource, table, Duration.ZERO); if (mysql.isInitializeSchema()) {
}
public MySqlVehicleIdentityService(DataSource dataSource, String table, Duration refreshInterval) {
if (dataSource == null) {
throw new IllegalArgumentException("dataSource must not be null");
}
this.dataSource = dataSource;
this.table = sanitizeTable(table);
this.registrationTable = this.table + "_registration";
initializeSchema(); initializeSchema();
this.index = loadIndex(); }
this.refresher = startRefresher(refreshInterval);
} }
@Override @Override
public void bind(VehicleIdentityBinding binding) { public void bind(VehicleIdentityBinding binding) {
if (binding == null || !binding.hasResolvedVin()) { String metadataJson = metadataJson(binding.metadata());
return; jdbc.update("""
} INSERT INTO %s
upsert(binding.plate(), binding.vin()); (protocol, vin, phone, device_id, plate, province, city, maker, device_type, plate_color, metadata_json,
index.bind(binding); created_at, updated_at)
} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON DUPLICATE KEY UPDATE
@Override vin = VALUES(vin),
public void register(VehicleRegistrationBinding registration) { device_id = VALUES(device_id),
if (registration == null) { province = VALUES(province),
return; city = VALUES(city),
} maker = VALUES(maker),
upsertRegistration(registration); device_type = VALUES(device_type),
if (registration.hasResolvedVin()) { plate_color = VALUES(plate_color),
bind(new VehicleIdentityBinding( metadata_json = VALUES(metadata_json),
registration.protocol(), updated_at = CURRENT_TIMESTAMP
registration.vin(), """.formatted(tableName),
registration.phone(), protocol(binding.protocol()), binding.vin(), binding.phone(), binding.deviceId(), binding.plate(),
registration.deviceId(), binding.metadata().getOrDefault("province", ""),
registration.plate())); binding.metadata().getOrDefault("city", ""),
} binding.metadata().getOrDefault("maker", ""),
binding.metadata().getOrDefault("deviceType", ""),
binding.metadata().getOrDefault("plateColor", ""),
metadataJson);
} }
@Override @Override
@@ -72,223 +66,167 @@ public final class MySqlVehicleIdentityService implements VehicleIdentityResolve
if (lookup == null) { if (lookup == null) {
return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN); return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN);
} }
return index.resolve(lookup); if (!lookup.vin().isBlank()) {
return new VehicleIdentity(lookup.vin(), true, VehicleIdentitySource.EXPLICIT_VIN);
}
VehicleIdentity byPhone = resolveBy("phone", lookup.protocol(), lookup.phone(), VehicleIdentitySource.BOUND_PHONE);
if (byPhone != null) {
return byPhone;
}
VehicleIdentity byDevice = resolveBy("device_id", lookup.protocol(), lookup.deviceId(), VehicleIdentitySource.BOUND_DEVICE_ID);
if (byDevice != null) {
return byDevice;
}
VehicleIdentity byPlate = resolveBy("plate", lookup.protocol(), lookup.plate(), VehicleIdentitySource.BOUND_PLATE);
if (byPlate != null) {
return byPlate;
}
if (!lookup.deviceId().isBlank()) {
return new VehicleIdentity(lookup.deviceId(), false, VehicleIdentitySource.FALLBACK_DEVICE_ID);
}
if (!lookup.phone().isBlank()) {
return new VehicleIdentity(lookup.phone(), false, VehicleIdentitySource.FALLBACK_PHONE);
}
if (!lookup.plate().isBlank()) {
return new VehicleIdentity(lookup.plate(), false, VehicleIdentitySource.FALLBACK_PLATE);
}
return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN);
} }
public void refresh() { private VehicleIdentity resolveBy(String column, ProtocolId protocol, String externalId, VehicleIdentitySource source) {
index = loadIndex(); if (externalId == null || externalId.isBlank()) {
return null;
}
String safeColumn = switch (column) {
case "phone", "device_id", "plate" -> column;
default -> throw new IllegalArgumentException("unsupported identity lookup column: " + column);
};
List<VehicleIdentity> matches = jdbc.query("""
SELECT vin
FROM %s
WHERE protocol = ? AND %s = ?
ORDER BY updated_at DESC
LIMIT 1
""".formatted(tableName, safeColumn),
(rs, rowNum) -> identity(rs, source),
protocol(protocol), externalId.trim());
return matches.isEmpty() ? null : matches.getFirst();
} }
@Override private VehicleIdentity identity(ResultSet rs, VehicleIdentitySource source) throws SQLException {
public void close() { return new VehicleIdentity(rs.getString("vin"), true, source);
if (refresher != null) {
refresher.shutdownNow();
}
} }
private void initializeSchema() { private void initializeSchema() {
String sql = """ jdbc.execute("""
CREATE TABLE IF NOT EXISTS %s (
plate VARCHAR(128) NOT NULL,
vin VARCHAR(64) NOT NULL,
PRIMARY KEY (plate),
KEY idx_vehicle_identity_vin (vin)
)
""".formatted(table);
String registrationSql = """
CREATE TABLE IF NOT EXISTS %s ( CREATE TABLE IF NOT EXISTS %s (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
protocol VARCHAR(32) NOT NULL, protocol VARCHAR(32) NOT NULL,
phone VARCHAR(64) NOT NULL, vin VARCHAR(32) NOT NULL,
vin VARCHAR(64) NOT NULL DEFAULT 'unknown', phone VARCHAR(64) NOT NULL DEFAULT '',
device_id VARCHAR(128) NOT NULL DEFAULT '', device_id VARCHAR(64) NOT NULL DEFAULT '',
plate VARCHAR(128) NOT NULL DEFAULT '', plate VARCHAR(64) NOT NULL DEFAULT '',
province INT NULL, province VARCHAR(32) NOT NULL DEFAULT '',
city INT NULL, city VARCHAR(32) NOT NULL DEFAULT '',
maker VARCHAR(64) NOT NULL DEFAULT '', maker VARCHAR(64) NOT NULL DEFAULT '',
device_type VARCHAR(128) NOT NULL DEFAULT '', device_type VARCHAR(128) NOT NULL DEFAULT '',
plate_color INT NULL, plate_color VARCHAR(32) NOT NULL DEFAULT '',
last_registered_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, metadata_json TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (protocol, phone), updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
KEY idx_vehicle_registration_vin (vin), UNIQUE KEY uk_vehicle_identity_protocol_phone (protocol, phone),
KEY idx_vehicle_registration_device (protocol, device_id), KEY idx_vehicle_identity_protocol_device (protocol, device_id),
KEY idx_vehicle_registration_plate (protocol, plate) KEY idx_vehicle_identity_protocol_plate (protocol, plate)
) )
""".formatted(registrationTable); """.formatted(tableName));
try (Connection connection = dataSource.getConnection(); ensureColumn("phone", "phone VARCHAR(64) NOT NULL DEFAULT ''");
Statement statement = connection.createStatement()) { ensureColumn("device_id", "device_id VARCHAR(64) NOT NULL DEFAULT ''");
statement.execute(sql); ensureColumn("plate", "plate VARCHAR(64) NOT NULL DEFAULT ''");
statement.execute(registrationSql); ensureColumn("province", "province VARCHAR(32) NOT NULL DEFAULT ''");
} catch (SQLException e) { ensureColumn("city", "city VARCHAR(32) NOT NULL DEFAULT ''");
throw new IllegalStateException("vehicle identity mysql schema initialize failed: " + table, e); ensureColumn("maker", "maker VARCHAR(64) NOT NULL DEFAULT ''");
ensureColumn("device_type", "device_type VARCHAR(128) NOT NULL DEFAULT ''");
ensureColumn("plate_color", "plate_color VARCHAR(32) NOT NULL DEFAULT ''");
ensureColumn("metadata_json", "metadata_json TEXT");
ensureColumn("created_at", "created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP");
ensureColumn("updated_at", "updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP");
ensureIndex("idx_vehicle_identity_protocol_device", "CREATE INDEX idx_vehicle_identity_protocol_device ON %s (protocol, device_id)");
ensureIndex("idx_vehicle_identity_protocol_plate", "CREATE INDEX idx_vehicle_identity_protocol_plate ON %s (protocol, plate)");
}
private void ensureColumn(String columnName, String definition) {
if (!columnExists(columnName)) {
jdbc.execute("ALTER TABLE " + tableName + " ADD COLUMN " + definition);
} }
} }
private InMemoryVehicleIdentityService loadIndex() { private boolean columnExists(String columnName) {
InMemoryVehicleIdentityService loaded = new InMemoryVehicleIdentityService(); return Boolean.TRUE.equals(jdbc.execute((ConnectionCallback<Boolean>) connection -> {
String sql = "SELECT plate, vin FROM " + table; DatabaseMetaData metaData = connection.getMetaData();
try (Connection connection = dataSource.getConnection(); for (String table : nameVariants(tableName)) {
PreparedStatement statement = connection.prepareStatement(sql); for (String column : nameVariants(columnName)) {
ResultSet resultSet = statement.executeQuery()) { try (ResultSet rs = metaData.getColumns(null, null, table, column)) {
while (resultSet.next()) { if (rs.next()) {
bindGlobalPlate(loaded, resultSet.getString("vin"), resultSet.getString("plate")); return true;
} }
loadRegistrationIndex(loaded, connection); }
return loaded; }
} catch (SQLException e) { }
throw new IllegalStateException("vehicle identity mysql index load failed: " + table, e); return false;
}));
}
private void ensureIndex(String indexName, String ddlTemplate) {
if (!indexExists(indexName)) {
jdbc.execute(ddlTemplate.formatted(tableName));
} }
} }
private static void bindGlobalPlate(InMemoryVehicleIdentityService loaded, String vin, String plate) { private boolean indexExists(String indexName) {
for (ProtocolId protocol : new ProtocolId[]{ return Boolean.TRUE.equals(jdbc.execute((ConnectionCallback<Boolean>) connection -> {
ProtocolId.GB32960, DatabaseMetaData metaData = connection.getMetaData();
ProtocolId.JT808, for (String table : nameVariants(tableName)) {
ProtocolId.MQTT_YUTONG, try (ResultSet rs = metaData.getIndexInfo(null, null, table, false, false)) {
ProtocolId.XINDA_PUSH while (rs.next()) {
}) { String existing = rs.getString("INDEX_NAME");
loaded.bind(new VehicleIdentityBinding(protocol, vin, "", "", plate)); if (existing != null && existing.equalsIgnoreCase(indexName)) {
return true;
} }
} }
}
}
return false;
}));
}
private void loadRegistrationIndex(InMemoryVehicleIdentityService loaded, Connection connection) throws SQLException { private static String[] nameVariants(String name) {
String sql = "SELECT r.protocol, r.phone, r.device_id, r.plate, b.vin FROM " + registrationTable + " r " return new String[]{
+ "JOIN " + table + " b ON b.plate = r.plate " name,
+ "WHERE b.vin IS NOT NULL AND b.vin <> '' AND LOWER(b.vin) <> 'unknown'"; name.toLowerCase(Locale.ROOT),
try (PreparedStatement statement = connection.prepareStatement(sql); name.toUpperCase(Locale.ROOT)
ResultSet resultSet = statement.executeQuery()) { };
while (resultSet.next()) {
loaded.bind(new VehicleIdentityBinding(
protocol(resultSet.getString("protocol")),
resultSet.getString("vin"),
resultSet.getString("phone"),
resultSet.getString("device_id"),
resultSet.getString("plate")));
}
}
loadLegacyRegistrationVinIndex(loaded, connection);
} }
private void loadLegacyRegistrationVinIndex(InMemoryVehicleIdentityService loaded, Connection connection) throws SQLException { private String metadataJson(Map<String, String> metadata) {
String sql = "SELECT protocol, phone, device_id, plate, vin FROM " + registrationTable if (metadata == null || metadata.isEmpty()) {
+ " WHERE vin IS NOT NULL AND vin <> '' AND LOWER(vin) <> 'unknown'"; return "{}";
try (PreparedStatement statement = connection.prepareStatement(sql);
ResultSet resultSet = statement.executeQuery()) {
while (resultSet.next()) {
loaded.bind(new VehicleIdentityBinding(
protocol(resultSet.getString("protocol")),
resultSet.getString("vin"),
resultSet.getString("phone"),
resultSet.getString("device_id"),
resultSet.getString("plate")));
} }
}
}
private ScheduledExecutorService startRefresher(Duration refreshInterval) {
if (refreshInterval == null || refreshInterval.isZero() || refreshInterval.isNegative()) {
return null;
}
long delayMillis = Math.max(refreshInterval.toMillis(), 1000L);
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(r -> {
Thread t = new Thread(r, "vehicle-identity-mysql-refresh");
t.setDaemon(true);
return t;
});
executor.scheduleWithFixedDelay(this::safeRefresh, delayMillis, delayMillis, TimeUnit.MILLISECONDS);
return executor;
}
private void safeRefresh() {
try { try {
refresh(); return mapper.writeValueAsString(metadata);
} catch (RuntimeException e) { } catch (JsonProcessingException e) {
log.warn("vehicle identity mysql refresh failed table={}", table, e); throw new IllegalArgumentException("vehicle identity metadata serialize failed", e);
} }
} }
private void upsert(String plate, String vin) { private static String protocol(ProtocolId protocol) {
String normalizedPlate = normalize(plate);
if (normalizedPlate.isBlank()) {
return;
}
String sql = "INSERT INTO " + table + " (plate, vin) "
+ "VALUES (?, ?) "
+ "ON DUPLICATE KEY UPDATE vin = VALUES(vin)";
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql)) {
statement.setString(1, normalizedPlate);
statement.setString(2, vin.trim());
statement.executeUpdate();
} catch (SQLException e) {
throw new IllegalStateException("vehicle identity mysql bind failed: " + table, e);
}
}
private void upsertRegistration(VehicleRegistrationBinding registration) {
String phone = normalize(registration.phone());
if (phone.isBlank()) {
return;
}
String vin = registration.hasResolvedVin() ? registration.vin().trim() : "unknown";
String sql = "INSERT INTO " + registrationTable
+ " (protocol, phone, vin, device_id, plate, province, city, maker, device_type, plate_color) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
+ "ON DUPLICATE KEY UPDATE "
+ "vin = CASE WHEN VALUES(vin) <> 'unknown' THEN VALUES(vin) ELSE vin END, "
+ "device_id = VALUES(device_id), plate = VALUES(plate), province = VALUES(province), "
+ "city = VALUES(city), maker = VALUES(maker), device_type = VALUES(device_type), "
+ "plate_color = VALUES(plate_color), last_registered_at = CURRENT_TIMESTAMP";
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql)) {
statement.setString(1, protocolName(registration.protocol()));
statement.setString(2, phone);
statement.setString(3, vin);
statement.setString(4, normalize(registration.deviceId()));
statement.setString(5, normalize(registration.plate()));
setInteger(statement, 6, registration.province());
setInteger(statement, 7, registration.city());
statement.setString(8, normalize(registration.maker()));
statement.setString(9, normalize(registration.deviceType()));
setInteger(statement, 10, registration.plateColor());
statement.executeUpdate();
} catch (SQLException e) {
throw new IllegalStateException("vehicle identity mysql registration bind failed: " + registrationTable, e);
}
}
private static void setInteger(PreparedStatement statement, int index, Integer value) throws SQLException {
if (value == null) {
statement.setObject(index, null);
} else {
statement.setInt(index, value);
}
}
private static String sanitizeTable(String table) {
String value = table == null || table.isBlank() ? "vehicle_identity_binding" : table.trim();
if (!value.matches("[A-Za-z0-9_]+")) {
throw new IllegalArgumentException("vehicle identity mysql table must contain only letters, digits, and underscore");
}
return value;
}
private static String protocolName(ProtocolId protocol) {
return protocol == null ? "UNKNOWN" : protocol.name(); return protocol == null ? "UNKNOWN" : protocol.name();
} }
private static ProtocolId protocol(String value) { private static String safeTableName(String value) {
if (value == null || value.isBlank()) { String name = value == null || value.isBlank() ? "vehicle_identity_bindings" : value.trim();
return ProtocolId.UNKNOWN; if (!name.matches("[A-Za-z0-9_]+")) {
throw new IllegalArgumentException("invalid vehicle identity mysql table name: " + value);
} }
try { return name.toLowerCase(Locale.ROOT);
return ProtocolId.valueOf(value.trim().toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException ignored) {
return ProtocolId.UNKNOWN;
}
}
private static String normalize(String value) {
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
} }
} }

View File

@@ -2,18 +2,26 @@ package com.lingniu.ingest.identity;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import java.util.Map;
public record VehicleIdentityBinding( public record VehicleIdentityBinding(
ProtocolId protocol, ProtocolId protocol,
String vin, String vin,
String phone, String phone,
String deviceId, String deviceId,
String plate String plate,
Map<String, String> metadata
) { ) {
public VehicleIdentityBinding(ProtocolId protocol, String vin, String phone, String deviceId, String plate) {
this(protocol, vin, phone, deviceId, plate, Map.of());
}
public VehicleIdentityBinding { public VehicleIdentityBinding {
vin = normalize(vin); vin = normalize(vin);
phone = normalize(phone); phone = normalize(phone);
deviceId = normalize(deviceId); deviceId = normalize(deviceId);
plate = normalize(plate); plate = normalize(plate);
metadata = metadata == null ? Map.of() : Map.copyOf(metadata);
if (vin.isBlank()) { if (vin.isBlank()) {
throw new IllegalArgumentException("vin must not be blank"); throw new IllegalArgumentException("vin must not be blank");
} }

View File

@@ -2,6 +2,7 @@ package com.lingniu.ingest.identity;
public enum VehicleIdentitySource { public enum VehicleIdentitySource {
EXPLICIT_VIN, EXPLICIT_VIN,
REGISTERED,
BOUND_PHONE, BOUND_PHONE,
BOUND_DEVICE_ID, BOUND_DEVICE_ID,
BOUND_PLATE, BOUND_PLATE,

View File

@@ -11,15 +11,10 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource; import javax.sql.DataSource;
import java.io.PrintWriter;
import java.nio.file.Path; import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.logging.Logger;
@AutoConfiguration @AutoConfiguration
@EnableConfigurationProperties(VehicleIdentityProperties.class) @EnableConfigurationProperties(VehicleIdentityProperties.class)
@@ -40,22 +35,15 @@ public class VehicleIdentityAutoConfiguration {
return new FileVehicleIdentityService(Path.of(properties.getFile().getPath()), objectMapper); return new FileVehicleIdentityService(Path.of(properties.getFile().getPath()), objectMapper);
} }
@Bean
@ConditionalOnMissingBean(DataSource.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql")
public DataSource vehicleIdentityDataSource(VehicleIdentityProperties properties) {
VehicleIdentityProperties.Mysql mysql = properties.getMysql();
return new DriverManagerVehicleIdentityDataSource(
mysql.getDriverClassName(), mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword());
}
@Bean @Bean
@ConditionalOnMissingBean({VehicleIdentityResolver.class, VehicleIdentityRegistry.class}) @ConditionalOnMissingBean({VehicleIdentityResolver.class, VehicleIdentityRegistry.class})
@ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql") @ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql")
public MySqlVehicleIdentityService mySqlVehicleIdentityService(VehicleIdentityProperties properties, public MySqlVehicleIdentityService mysqlVehicleIdentityService(VehicleIdentityProperties properties,
DataSource dataSource) { ObjectMapper objectMapper) {
VehicleIdentityProperties.Mysql mysql = properties.getMysql(); VehicleIdentityProperties.Mysql mysql = properties.getMysql();
return new MySqlVehicleIdentityService(dataSource, mysql.getTable(), mysql.getRefreshInterval()); DataSource dataSource = new DriverManagerDataSource(
mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword());
return new MySqlVehicleIdentityService(dataSource, mysql, objectMapper);
} }
@Bean @Bean
@@ -65,61 +53,4 @@ public class VehicleIdentityAutoConfiguration {
public InMemoryVehicleIdentityService vehicleIdentityService() { public InMemoryVehicleIdentityService vehicleIdentityService() {
return new InMemoryVehicleIdentityService(); return new InMemoryVehicleIdentityService();
} }
private static final class DriverManagerVehicleIdentityDataSource implements DataSource {
private final String jdbcUrl;
private final String username;
private final String password;
private PrintWriter logWriter;
private int loginTimeout;
private DriverManagerVehicleIdentityDataSource(String driverClassName,
String jdbcUrl,
String username,
String password) {
if (jdbcUrl == null || jdbcUrl.isBlank()) {
throw new IllegalArgumentException("lingniu.ingest.identity.mysql.jdbc-url is required");
}
if (driverClassName != null && !driverClassName.isBlank()) {
try {
Class.forName(driverClassName.trim());
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("vehicle identity mysql driver class not found: "
+ driverClassName, e);
}
}
this.jdbcUrl = jdbcUrl;
this.username = username == null ? "" : username;
this.password = password == null ? "" : password;
}
@Override
public Connection getConnection() throws SQLException {
DriverManager.setLoginTimeout(loginTimeout);
return DriverManager.getConnection(jdbcUrl, username, password);
}
@Override
public Connection getConnection(String username, String password) throws SQLException {
DriverManager.setLoginTimeout(loginTimeout);
return DriverManager.getConnection(jdbcUrl, username, password);
}
@Override public PrintWriter getLogWriter() { return logWriter; }
@Override public void setLogWriter(PrintWriter out) { this.logWriter = out; }
@Override public void setLoginTimeout(int seconds) { this.loginTimeout = seconds; }
@Override public int getLoginTimeout() { return loginTimeout; }
@Override public Logger getParentLogger() throws SQLFeatureNotSupportedException {
throw new SQLFeatureNotSupportedException();
}
@Override public <T> T unwrap(Class<T> iface) throws SQLException {
if (iface.isInstance(this)) {
return iface.cast(this);
}
throw new SQLException("not a wrapper for " + iface);
}
@Override public boolean isWrapperFor(Class<?> iface) {
return iface.isInstance(this);
}
}
} }

View File

@@ -2,8 +2,6 @@ package com.lingniu.ingest.identity.config;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import java.time.Duration;
@ConfigurationProperties(prefix = "lingniu.ingest.identity") @ConfigurationProperties(prefix = "lingniu.ingest.identity")
public class VehicleIdentityProperties { public class VehicleIdentityProperties {
@@ -26,24 +24,21 @@ public class VehicleIdentityProperties {
} }
public static class Mysql { public static class Mysql {
private String table = "vehicle_identity_binding"; private String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai";
private String jdbcUrl = ""; private String username = "root";
private String username = "";
private String password = ""; private String password = "";
private String driverClassName = "com.mysql.cj.jdbc.Driver"; private String tableName = "vehicle_identity_bindings";
private Duration refreshInterval = Duration.ofSeconds(60); private boolean initializeSchema = true;
public String getTable() { return table; }
public void setTable(String table) { this.table = table; }
public String getJdbcUrl() { return jdbcUrl; } public String getJdbcUrl() { return jdbcUrl; }
public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; } public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; }
public String getUsername() { return username; } public String getUsername() { return username; }
public void setUsername(String username) { this.username = username; } public void setUsername(String username) { this.username = username; }
public String getPassword() { return password; } public String getPassword() { return password; }
public void setPassword(String password) { this.password = password; } public void setPassword(String password) { this.password = password; }
public String getDriverClassName() { return driverClassName; } public String getTableName() { return tableName; }
public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; } public void setTableName(String tableName) { this.tableName = tableName; }
public Duration getRefreshInterval() { return refreshInterval; } public boolean isInitializeSchema() { return initializeSchema; }
public void setRefreshInterval(Duration refreshInterval) { this.refreshInterval = refreshInterval; } public void setInitializeSchema(boolean initializeSchema) { this.initializeSchema = initializeSchema; }
} }
} }

View File

@@ -1,6 +1,8 @@
package com.lingniu.ingest.identity; package com.lingniu.ingest.identity;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
import org.h2.jdbcx.JdbcDataSource; import org.h2.jdbcx.JdbcDataSource;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -9,6 +11,7 @@ import java.sql.ResultSet;
import java.sql.Statement; import java.sql.Statement;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.UUID; import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -21,36 +24,43 @@ class MySqlVehicleIdentityServiceJdbcTest {
dataSource.setURL("jdbc:h2:mem:" + UUID.randomUUID() dataSource.setURL("jdbc:h2:mem:" + UUID.randomUUID()
+ ";MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1"); + ";MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1");
try (MySqlVehicleIdentityService service = VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
new MySqlVehicleIdentityService(dataSource, "vehicle_identity_binding")) { mysql.setTableName("vehicle_identity_binding");
service.register(new VehicleRegistrationBinding( MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
dataSource,
mysql,
new ObjectMapper());
service.bind(new VehicleIdentityBinding(
ProtocolId.JT808, ProtocolId.JT808,
"unknown", "LNVIN00000061001",
"13079961001", "13079961001",
"dev61001", "dev61001",
"粤B61001", "粤B61001",
44, Map.of(
4401, "province", "44",
"maker-a", "city", "4401",
"type-a", "maker", "maker-a",
1)); "deviceType", "type-a",
"plateColor", "1")));
try (Connection connection = dataSource.getConnection(); try (Connection connection = dataSource.getConnection();
Statement statement = connection.createStatement()) { Statement statement = connection.createStatement()) {
ResultSet registered = statement.executeQuery(""" ResultSet binding = statement.executeQuery("""
SELECT vin, device_id, plate, province, city, maker, device_type, plate_color SELECT vin, phone, device_id, plate, province, city, maker, device_type, plate_color
FROM vehicle_identity_binding_registration FROM vehicle_identity_binding
WHERE protocol = 'JT808' AND phone = '13079961001' WHERE protocol = 'JT808' AND phone = '13079961001'
"""); """);
assertThat(registered.next()).isTrue(); assertThat(binding.next()).isTrue();
assertThat(registered.getString("vin")).isEqualTo("unknown"); assertThat(binding.getString("vin")).isEqualTo("LNVIN00000061001");
assertThat(registered.getString("device_id")).isEqualTo("DEV61001"); assertThat(binding.getString("phone")).isEqualTo("13079961001");
assertThat(registered.getString("plate")).isEqualTo("粤B61001"); assertThat(binding.getString("device_id")).isEqualTo("dev61001");
assertThat(registered.getInt("province")).isEqualTo(44); assertThat(binding.getString("plate")).isEqualTo("粤B61001");
assertThat(registered.getInt("city")).isEqualTo(4401); assertThat(binding.getString("province")).isEqualTo("44");
assertThat(registered.getString("maker")).isEqualTo("MAKER-A"); assertThat(binding.getString("city")).isEqualTo("4401");
assertThat(registered.getString("device_type")).isEqualTo("TYPE-A"); assertThat(binding.getString("maker")).isEqualTo("maker-a");
assertThat(registered.getInt("plate_color")).isEqualTo(1); assertThat(binding.getString("device_type")).isEqualTo("type-a");
assertThat(binding.getString("plate_color")).isEqualTo("1");
List<String> columns = new ArrayList<>(); List<String> columns = new ArrayList<>();
ResultSet bindingColumns = connection.getMetaData() ResultSet bindingColumns = connection.getMetaData()
@@ -58,16 +68,15 @@ class MySqlVehicleIdentityServiceJdbcTest {
while (bindingColumns.next()) { while (bindingColumns.next()) {
columns.add(bindingColumns.getString("COLUMN_NAME")); columns.add(bindingColumns.getString("COLUMN_NAME"));
} }
assertThat(columns).containsExactly("plate", "vin"); assertThat(columns).contains(
"protocol",
statement.executeUpdate(""" "phone",
INSERT INTO vehicle_identity_binding (plate, vin) "device_id",
VALUES ('粤B61001', 'LNVIN00000061001') "plate",
"""); "vin",
"metadata_json");
} }
service.refresh();
assertThat(service.resolve(new VehicleIdentityLookup( assertThat(service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079961001", "", "")).vin()) ProtocolId.JT808, "", "13079961001", "", "")).vin())
.isEqualTo("LNVIN00000061001"); .isEqualTo("LNVIN00000061001");
@@ -78,5 +87,4 @@ class MySqlVehicleIdentityServiceJdbcTest {
ProtocolId.JT808, "", "", "", "粤B61001")).vin()) ProtocolId.JT808, "", "", "", "粤B61001")).vin())
.isEqualTo("LNVIN00000061001"); .isEqualTo("LNVIN00000061001");
} }
}
} }

View File

@@ -1,478 +1,157 @@
package com.lingniu.ingest.identity; package com.lingniu.ingest.identity;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource;
import java.io.PrintWriter;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.logging.Logger;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
class MySqlVehicleIdentityServiceTest { class MySqlVehicleIdentityServiceTest {
@Test @Test
void bindsAndResolvesIdentityThroughJdbcStore() { void upsertsRegistrationBindingAndResolvesByExternalIdentifiers() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc(); DriverManagerDataSource dataSource = new DriverManagerDataSource(
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding"); "jdbc:h2:mem:mysql-identity;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
mysql.setTableName("vehicle_identity_bindings");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
dataSource,
mysql,
new ObjectMapper());
service.bind(new VehicleIdentityBinding( service.bind(new VehicleIdentityBinding(
ProtocolId.JT808, "LNVIN000000000123", "13079960001", "DEV001", "粤B00001"));
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960001", "", ""));
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "DEV001", ""));
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "", "粤B00001"));
assertThat(jdbc.createdTables)
.anySatisfy(sql -> assertThat(sql).contains("CREATE TABLE IF NOT EXISTS vehicle_identity_binding"));
assertThat(jdbc.bindings).containsEntry("粤B00001", "LNVIN000000000123");
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000123");
assertThat(byPhone.resolved()).isTrue();
assertThat(byPhone.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000123");
assertThat(byDevice.source()).isEqualTo(VehicleIdentitySource.BOUND_DEVICE_ID);
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000123");
assertThat(byPlate.source()).isEqualTo(VehicleIdentitySource.BOUND_PLATE);
}
@Test
void ignoresUnknownVinBinding() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
service.bind(new VehicleIdentityBinding(
ProtocolId.JT808, "unknown", "13079960001", "DEV001", "粤B00001"));
assertThat(jdbc.bindings).isEmpty();
}
@Test
void resolvesFromStartupCacheWithoutPerFrameDatabaseLookup() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
jdbc.seedRegistration("JT808", "13079960001", "DEV001", "粤B00001");
jdbc.seed("粤B00001", "LNVIN000000000123");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960001", "", ""));
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "DEV001", ""));
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "", "粤B00001"));
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000123");
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000123");
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000123");
assertThat(jdbc.loadAllSelects).isEqualTo(1);
assertThat(jdbc.lookupSelects).isZero();
}
@Test
void resolvesGlobalPlateBindingForXindaPush() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
jdbc.seed("浙F00780F", "LA9GG68L8PBAF4776");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
ProtocolId.XINDA_PUSH, "", "", "", "浙F00780F"));
assertThat(byPlate.vin()).isEqualTo("LA9GG68L8PBAF4776");
assertThat(byPlate.resolved()).isTrue();
assertThat(byPlate.source()).isEqualTo(VehicleIdentitySource.BOUND_PLATE);
assertThat(jdbc.loadAllSelects).isEqualTo(1);
assertThat(jdbc.lookupSelects).isZero();
}
@Test
void refreshLoadsExternallyInsertedBindingWithoutPerFrameDatabaseLookup() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
jdbc.seedRegistration("JT808", "13079960002", "DEV002", "粤B00002");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
VehicleIdentity before = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960002", "", ""));
jdbc.seed("粤B00002", "LNVIN000000000456");
service.refresh();
VehicleIdentity after = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960002", "", ""));
assertThat(before.resolved()).isFalse();
assertThat(after.vin()).isEqualTo("LNVIN000000000456");
assertThat(after.resolved()).isTrue();
assertThat(after.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
assertThat(jdbc.loadAllSelects).isEqualTo(2);
assertThat(jdbc.lookupSelects).isZero();
}
@Test
void storesRegistrationWithoutVinAndResolvesAfterPlateVinIsWrittenBack() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
service.register(new VehicleRegistrationBinding(
ProtocolId.JT808, ProtocolId.JT808,
"unknown", "JT808AABBCCDDEE1",
"13079960003", "13079963289",
"DEV003", "9963289",
"粤B00003", "沪A00113F",
44, Map.of(
4401, "province", "31",
"MAKER", "city", "113",
"TYPE-A", "maker", "70112",
1)); "deviceType", "SEG-9888G",
"plateColor", "2")));
VehicleIdentity before = service.resolve(new VehicleIdentityLookup( assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "13079963289", "", "")))
ProtocolId.JT808, "", "13079960003", "", "")); .extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_PHONE);
assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "", "9963289", "")))
.extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_DEVICE_ID);
assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "", "", "沪A00113F")))
.extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_PLATE);
jdbc.seed("粤B00003", "LNVIN000000000789"); JdbcTemplate jdbc = new JdbcTemplate(dataSource);
service.refresh(); Map<String, Object> row = jdbc.queryForMap("""
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup( SELECT vin, phone, device_id, plate, province, city, maker, device_type, plate_color
ProtocolId.JT808, "", "13079960003", "", "")); FROM vehicle_identity_bindings
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup( WHERE protocol = 'JT808' AND phone = '13079963289'
ProtocolId.JT808, "", "", "DEV003", "")); """);
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup( assertThat(row)
ProtocolId.JT808, "", "", "", "粤B00003")); .containsEntry("vin", "JT808AABBCCDDEE1")
.containsEntry("phone", "13079963289")
assertThat(jdbc.createdTables) .containsEntry("device_id", "9963289")
.anySatisfy(sql -> assertThat(sql).contains("CREATE TABLE IF NOT EXISTS vehicle_identity_binding_registration")); .containsEntry("plate", "沪A00113F")
assertThat(jdbc.registrations).containsKey("JT808|13079960003"); .containsEntry("province", "31")
assertThat(jdbc.bindings).containsEntry("粤B00003", "LNVIN000000000789"); .containsEntry("city", "113")
assertThat(before.resolved()).isFalse(); .containsEntry("maker", "70112")
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000789"); .containsEntry("device_type", "SEG-9888G")
assertThat(byPhone.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE); .containsEntry("plate_color", "2");
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000789");
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000789");
} }
private static final class RecordingIdentityJdbc { @Test
private final List<String> createdTables = new ArrayList<>(); void initializesMissingColumnsOnExistingLegacyTable() {
private final Map<String, String> bindings = new HashMap<>(); DriverManagerDataSource dataSource = new DriverManagerDataSource(
private final Map<String, RegistrationRow> registrations = new HashMap<>(); "jdbc:h2:mem:mysql-identity-legacy;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
private int loadAllSelects; JdbcTemplate jdbc = new JdbcTemplate(dataSource);
private int lookupSelects; jdbc.execute("""
CREATE TABLE vehicle_identity_binding (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL,
legacy_key VARCHAR(64) NOT NULL DEFAULT ''
)
""");
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
mysql.setTableName("vehicle_identity_binding");
private void seed(String plate, String vin) { MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
bindings.put(normalize(plate), vin); dataSource,
mysql,
new ObjectMapper());
service.bind(new VehicleIdentityBinding(
ProtocolId.JT808,
"JT808LEGACY00001",
"13079963289",
"9963289",
"沪A00113F",
Map.of("maker", "70112", "deviceType", "SEG-9888G")));
Map<String, Object> row = jdbc.queryForMap("""
SELECT vin, maker, device_type, metadata_json
FROM vehicle_identity_binding
WHERE protocol = 'JT808' AND phone = '13079963289'
""");
assertThat(row)
.containsEntry("vin", "JT808LEGACY00001")
.containsEntry("maker", "70112")
.containsEntry("device_type", "SEG-9888G");
assertThat((String) row.get("metadata_json")).contains("SEG-9888G");
} }
private void seedRegistration(String protocol, String phone, String deviceId, String plate) { @Test
registrations.put(protocol + "|" + normalize(phone), void upsertsByCompositeProtocolPhonePlatePrimaryKey() {
new RegistrationRow(protocol, normalize(phone), "unknown", normalize(deviceId), normalize(plate))); DriverManagerDataSource dataSource = new DriverManagerDataSource(
} "jdbc:h2:mem:mysql-identity-composite;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
JdbcTemplate jdbc = new JdbcTemplate(dataSource);
jdbc.execute("""
CREATE TABLE vehicle_identity_binding (
protocol VARCHAR(32) NOT NULL DEFAULT 'ALL',
phone VARCHAR(128) NOT NULL DEFAULT '',
plate VARCHAR(128) NOT NULL DEFAULT '',
vin VARCHAR(64) NOT NULL,
province VARCHAR(32) NOT NULL DEFAULT '',
city VARCHAR(32) NOT NULL DEFAULT '',
maker VARCHAR(64) NOT NULL DEFAULT '',
device_type VARCHAR(128) NOT NULL DEFAULT '',
plate_color VARCHAR(32) NOT NULL DEFAULT '',
metadata_json TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
device_id VARCHAR(64) NOT NULL DEFAULT '',
PRIMARY KEY (protocol, phone, plate)
)
""");
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
mysql.setTableName("vehicle_identity_binding");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
dataSource,
mysql,
new ObjectMapper());
service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
"VIN-A", "13079963289", "DEV-A", "沪A00113F"));
private DataSource dataSource() { service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
return new DataSource() { "VIN-B", "13079963289", "DEV-B", "沪A00114F"));
@Override public Connection getConnection() { return connection(); } service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
@Override public Connection getConnection(String username, String password) { return connection(); } "VIN-B2", "13079963289", "DEV-B2", "沪A00114F"));
@Override public PrintWriter getLogWriter() { return null; }
@Override public void setLogWriter(PrintWriter out) {}
@Override public void setLoginTimeout(int seconds) {}
@Override public int getLoginTimeout() { return 0; }
@Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { throw new SQLFeatureNotSupportedException(); }
@Override public <T> T unwrap(Class<T> iface) { throw new UnsupportedOperationException(); }
@Override public boolean isWrapperFor(Class<?> iface) { return false; }
};
}
private Connection connection() { Integer rows = jdbc.queryForObject("""
InvocationHandler handler = (proxy, method, args) -> switch (method.getName()) { SELECT COUNT(*)
case "createStatement" -> statement(); FROM vehicle_identity_binding
case "prepareStatement" -> preparedStatement((String) args[0]); WHERE protocol = 'JT808' AND phone = '13079963289'
case "close" -> null; """, Integer.class);
case "isClosed" -> false; assertThat(rows).isEqualTo(2);
case "unwrap" -> throw new UnsupportedOperationException(); assertThat(jdbc.queryForObject("""
case "isWrapperFor" -> false; SELECT vin
default -> defaultValue(method.getReturnType()); FROM vehicle_identity_binding
}; WHERE protocol = 'JT808' AND phone = '13079963289' AND plate = '沪A00114F'
return (Connection) Proxy.newProxyInstance( """, String.class)).isEqualTo("VIN-B2");
Connection.class.getClassLoader(), new Class<?>[]{Connection.class}, handler);
}
private Statement statement() {
InvocationHandler handler = (proxy, method, args) -> {
if ("execute".equals(method.getName())) {
createdTables.add((String) args[0]);
return false;
}
if ("close".equals(method.getName())) {
return null;
}
return defaultValue(method.getReturnType());
};
return (Statement) Proxy.newProxyInstance(
Statement.class.getClassLoader(), new Class<?>[]{Statement.class}, handler);
}
private PreparedStatement preparedStatement(String sql) {
List<Object> params = new ArrayList<>();
InvocationHandler handler = (proxy, method, args) -> {
switch (method.getName()) {
case "setString", "setInt", "setObject" -> {
set(params, (Integer) args[0], args[1]);
return null;
}
case "executeUpdate" -> {
if (sql.startsWith("INSERT INTO vehicle_identity_binding_registration ")) {
upsertRegistration(params);
} else {
upsert(params);
}
return 1;
}
case "executeQuery" -> {
if (sql.startsWith("SELECT plate, vin FROM ")) {
loadAllSelects++;
return allBindingsResultSet();
}
if (sql.startsWith("SELECT protocol, identifier_type, identifier_value, vin FROM ")) {
loadAllSelects++;
return legacyAllBindingsResultSet();
}
if (sql.startsWith("SELECT r.protocol, r.phone, r.device_id, r.plate, b.vin FROM ")) {
return registrationResultSet();
}
if (sql.startsWith("SELECT protocol, phone, device_id, plate, vin FROM ")) {
return legacyRegistrationResultSet();
}
lookupSelects++;
return resultSet(resolve(params));
}
case "close" -> {
return null;
}
default -> {
return defaultValue(method.getReturnType());
}
}
};
return (PreparedStatement) Proxy.newProxyInstance(
PreparedStatement.class.getClassLoader(), new Class<?>[]{PreparedStatement.class}, handler);
}
private void upsert(List<Object> params) {
String plate;
String vin;
if (params.size() >= 4) {
plate = normalize((String) params.get(2));
vin = (String) params.get(3);
} else {
plate = normalize((String) params.get(0));
vin = (String) params.get(1);
}
bindings.put(plate, vin);
}
private void upsertRegistration(List<Object> params) {
String protocol = (String) params.get(0);
String phone = normalize((String) params.get(1));
String vin = (String) params.get(2);
String key = protocol + "|" + phone;
RegistrationRow existing = registrations.get(key);
String effectiveVin = existing != null && "unknown".equalsIgnoreCase(vin) ? existing.vin : vin;
registrations.put(key, new RegistrationRow(
protocol,
phone,
effectiveVin,
normalize((String) params.get(3)),
normalize((String) params.get(4))));
}
private String resolve(List<Object> params) {
String protocol = (String) params.get(0);
String identifierType = (String) params.get(1);
String identifierValue = normalize((String) params.get(2));
if (!"PLATE".equals(identifierType)) {
return null;
}
return bindings.get(identifierValue);
}
private ResultSet allBindingsResultSet() {
List<Map.Entry<String, String>> entries = new ArrayList<>(bindings.entrySet());
InvocationHandler handler = new InvocationHandler() {
private int index = -1;
@Override
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
return switch (method.getName()) {
case "next" -> {
index++;
yield index < entries.size();
}
case "getString" -> getColumn(entries.get(index), args[0]);
case "close" -> null;
default -> defaultValue(method.getReturnType());
};
}
};
return (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
}
private ResultSet legacyAllBindingsResultSet() {
List<Map.Entry<String, String>> entries = new ArrayList<>(bindings.entrySet());
InvocationHandler handler = new InvocationHandler() {
private int index = -1;
@Override
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
return switch (method.getName()) {
case "next" -> {
index++;
yield index < entries.size();
}
case "getString" -> legacyColumn(entries.get(index), args[0]);
case "close" -> null;
default -> defaultValue(method.getReturnType());
};
}
};
return (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
}
private static String getColumn(Map.Entry<String, String> entry, Object column) {
String name = column.toString();
return switch (name) {
case "plate" -> entry.getKey();
case "vin" -> entry.getValue();
default -> throw new IllegalArgumentException("unsupported column " + name);
};
}
private static String legacyColumn(Map.Entry<String, String> entry, Object column) {
return switch (column.toString()) {
case "protocol" -> "JT808";
case "identifier_type" -> "PLATE";
case "identifier_value" -> entry.getKey();
case "vin" -> entry.getValue();
default -> throw new IllegalArgumentException("unsupported column " + column);
};
}
private ResultSet registrationResultSet() {
List<RegistrationRow> rows = registrations.values().stream()
.filter(row -> bindings.containsKey(row.plate))
.map(row -> new RegistrationRow(row.protocol, row.phone, bindings.get(row.plate), row.deviceId, row.plate))
.toList();
return registrationRowsResultSet(rows);
}
private ResultSet legacyRegistrationResultSet() {
List<RegistrationRow> rows = registrations.values().stream()
.filter(row -> row.vin != null && !row.vin.isBlank() && !"unknown".equalsIgnoreCase(row.vin))
.toList();
return registrationRowsResultSet(rows);
}
private ResultSet registrationRowsResultSet(List<RegistrationRow> rows) {
InvocationHandler handler = new InvocationHandler() {
private int index = -1;
@Override
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
return switch (method.getName()) {
case "next" -> {
index++;
yield index < rows.size();
}
case "getString" -> registrationColumn(rows.get(index), args[0]);
case "close" -> null;
default -> defaultValue(method.getReturnType());
};
}
};
return (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
}
private static String registrationColumn(RegistrationRow row, Object column) {
return switch (column.toString()) {
case "protocol" -> row.protocol;
case "phone" -> row.phone;
case "device_id" -> row.deviceId;
case "plate" -> row.plate;
case "vin" -> row.vin;
default -> throw new IllegalArgumentException("unsupported column " + column);
};
}
private static ResultSet resultSet(String vin) {
InvocationHandler handler = new InvocationHandler() {
private boolean read;
@Override
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
return switch (method.getName()) {
case "next" -> {
boolean hasNext = !read && vin != null;
read = true;
yield hasNext;
}
case "getString" -> vin;
case "close" -> null;
default -> defaultValue(method.getReturnType());
};
}
};
return (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
}
private static void set(List<Object> params, int oneBasedIndex, Object value) {
while (params.size() < oneBasedIndex) {
params.add(null);
}
params.set(oneBasedIndex - 1, value);
}
private static String normalize(String value) {
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
}
private static Object defaultValue(Class<?> type) {
if (type == Void.TYPE) return null;
if (type == Boolean.TYPE) return false;
if (type == Integer.TYPE) return 0;
if (type == Long.TYPE) return 0L;
if (type == Double.TYPE) return 0D;
if (type == Float.TYPE) return 0F;
if (type == Short.TYPE) return (short) 0;
if (type == Byte.TYPE) return (byte) 0;
return null;
}
private static final class RegistrationRow {
private final String protocol;
private final String phone;
private String vin;
private final String deviceId;
private final String plate;
private RegistrationRow(String protocol, String phone, String vin, String deviceId, String plate) {
this.protocol = protocol;
this.phone = phone;
this.vin = vin;
this.deviceId = deviceId;
this.plate = plate;
}
}
} }
} }

View File

@@ -7,14 +7,7 @@ import org.junit.jupiter.api.io.TempDir;
import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import javax.sql.DataSource;
import java.nio.file.Path; import java.nio.file.Path;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.lang.reflect.Proxy;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -52,75 +45,19 @@ class VehicleIdentityAutoConfigurationTest {
@Test @Test
void createsMysqlIdentityServiceWhenConfigured() { void createsMysqlIdentityServiceWhenConfigured() {
runner runner
.withBean(DataSource.class, StubDataSource::new)
.withPropertyValues( .withPropertyValues(
"lingniu.ingest.identity.store=mysql", "lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.table=vehicle_identity_binding") "lingniu.ingest.identity.mysql.jdbc-url=jdbc:h2:mem:auto-identity;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1",
"lingniu.ingest.identity.mysql.username=sa",
"lingniu.ingest.identity.mysql.password=",
"lingniu.ingest.identity.mysql.table-name=vehicle_identity_bindings")
.run(context -> { .run(context -> {
assertThat(context).hasSingleBean(VehicleIdentityResolver.class); assertThat(context).hasSingleBean(VehicleIdentityResolver.class);
assertThat(context.getBean(VehicleIdentityResolver.class).getClass().getSimpleName()) assertThat(context).hasSingleBean(VehicleIdentityRegistry.class);
.isEqualTo("MySqlVehicleIdentityService");
assertThat(context.getBean(VehicleIdentityResolver.class)) assertThat(context.getBean(VehicleIdentityResolver.class))
.isSameAs(context.getBean(VehicleIdentityRegistry.class)); .isSameAs(context.getBean(VehicleIdentityRegistry.class));
assertThat(context.getBean(VehicleIdentityResolver.class).getClass().getSimpleName())
.isEqualTo("MySqlVehicleIdentityService");
}); });
} }
private static final class StubDataSource implements DataSource {
@Override public Connection getConnection() { return connection(); }
@Override public Connection getConnection(String username, String password) { return connection(); }
@Override public java.io.PrintWriter getLogWriter() { return null; }
@Override public void setLogWriter(java.io.PrintWriter out) {}
@Override public void setLoginTimeout(int seconds) {}
@Override public int getLoginTimeout() { return 0; }
@Override public java.util.logging.Logger getParentLogger() throws java.sql.SQLFeatureNotSupportedException { throw new java.sql.SQLFeatureNotSupportedException(); }
@Override public <T> T unwrap(Class<T> iface) { throw new UnsupportedOperationException(); }
@Override public boolean isWrapperFor(Class<?> iface) { return false; }
private static Connection connection() {
return (Connection) Proxy.newProxyInstance(Connection.class.getClassLoader(), new Class<?>[]{Connection.class},
(proxy, method, args) -> switch (method.getName()) {
case "createStatement" -> statement();
case "prepareStatement" -> preparedStatement();
case "close" -> null;
case "isClosed" -> false;
default -> defaultValue(method.getReturnType());
});
}
private static Statement statement() {
return (Statement) Proxy.newProxyInstance(Statement.class.getClassLoader(), new Class<?>[]{Statement.class},
(proxy, method, args) -> switch (method.getName()) {
case "execute" -> false;
case "close" -> null;
default -> defaultValue(method.getReturnType());
});
}
private static PreparedStatement preparedStatement() {
return (PreparedStatement) Proxy.newProxyInstance(
PreparedStatement.class.getClassLoader(), new Class<?>[]{PreparedStatement.class},
(proxy, method, args) -> switch (method.getName()) {
case "executeQuery" -> emptyResultSet();
case "close" -> null;
default -> defaultValue(method.getReturnType());
});
}
private static ResultSet emptyResultSet() {
return (ResultSet) Proxy.newProxyInstance(ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class},
(proxy, method, args) -> switch (method.getName()) {
case "next" -> false;
case "close" -> null;
default -> defaultValue(method.getReturnType());
});
}
private static Object defaultValue(Class<?> type) {
if (type == Void.TYPE) return null;
if (type == Boolean.TYPE) return false;
if (type == Integer.TYPE) return 0;
if (type == Long.TYPE) return 0L;
return null;
}
}
} }

View File

@@ -9,7 +9,7 @@
</parent> </parent>
<artifactId>inbound-mqtt</artifactId> <artifactId>inbound-mqtt</artifactId>
<name>inbound-mqtt</name> <name>inbound-mqtt</name>
<description>MQTT 接入模块(支持多 endpoint宇通等Fusesource MQTT Client + 双向 TLS。</description> <description>MQTT 接入模块(支持多 endpoint宇通等Eclipse Paho MQTT Client + 双向 TLS。</description>
<dependencies> <dependencies>
<dependency> <dependency>
@@ -34,8 +34,8 @@
<optional>true</optional> <optional>true</optional>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.fusesource.mqtt-client</groupId> <groupId>org.eclipse.paho</groupId>
<artifactId>mqtt-client</artifactId> <artifactId>org.eclipse.paho.client.mqttv3</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>

View File

@@ -11,30 +11,33 @@ import com.lingniu.ingest.identity.VehicleIdentitySource;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties; import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import com.lingniu.ingest.inbound.mqtt.model.MqttPayload; import com.lingniu.ingest.inbound.mqtt.model.MqttPayload;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry; import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import org.fusesource.mqtt.client.Future; import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
import org.fusesource.mqtt.client.FutureConnection; import org.eclipse.paho.client.mqttv3.MqttAsyncClient;
import org.fusesource.mqtt.client.MQTT; import org.eclipse.paho.client.mqttv3.MqttCallbackExtended;
import org.fusesource.mqtt.client.Message; import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.fusesource.mqtt.client.QoS; import org.eclipse.paho.client.mqttv3.MqttException;
import org.fusesource.mqtt.client.Topic; import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.EOFException; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.Instant; import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
/** /**
* MQTT 多 endpoint 生命周期管理。 * MQTT 多 endpoint 生命周期管理。
* *
* <p>为每个 endpoint 创建一个 fusesource MQTT FutureConnection → 连接 → 订阅 → 接收循环 * <p>为每个 endpoint 创建一个 Paho 异步客户端 → 连接 → 订阅 → 注册回调
* 回调里解析 payload → {@link Dispatcher#dispatch(RawFrame)}。 * 回调里解析 payload → {@link Dispatcher#dispatch(RawFrame)}。
* *
* <p>每个 endpoint 在独立虚拟线程中阻塞接收,业务处理由 Dispatcher 侧的 Disruptor 承接。 * <p>MQTT 回调不做业务处理,虚拟线程由 Dispatcher 侧的 Disruptor 承接。
*/ */
public final class MqttEndpointManager implements AutoCloseable { public final class MqttEndpointManager implements AutoCloseable {
@@ -44,9 +47,8 @@ public final class MqttEndpointManager implements AutoCloseable {
private final MqttProfileRegistry profileRegistry; private final MqttProfileRegistry profileRegistry;
private final VehicleIdentityResolver identityResolver; private final VehicleIdentityResolver identityResolver;
private final Dispatcher dispatcher; private final Dispatcher dispatcher;
private final List<FutureConnection> connections = new ArrayList<>(); private final List<MqttAsyncClient> clients = new ArrayList<>();
private final List<Thread> workers = new ArrayList<>(); private boolean started;
private volatile boolean running;
public MqttEndpointManager(MqttInboundProperties props, public MqttEndpointManager(MqttInboundProperties props,
MqttProfileRegistry profileRegistry, MqttProfileRegistry profileRegistry,
@@ -64,111 +66,78 @@ public final class MqttEndpointManager implements AutoCloseable {
this.dispatcher = dispatcher; this.dispatcher = dispatcher;
} }
public void start() { public synchronized void start() {
running = true; if (started) {
log.info("mqtt endpoint manager starting, endpoints={}", props.getEndpoints().size()); return;
}
started = true;
if (props.getEndpoints().isEmpty()) {
log.warn("mqtt inbound enabled but no endpoints configured");
return;
}
log.info("mqtt inbound starting endpoints={}", props.getEndpoints().size());
for (MqttInboundProperties.Endpoint ep : props.getEndpoints()) { for (MqttInboundProperties.Endpoint ep : props.getEndpoints()) {
Thread worker = Thread.ofVirtual() try {
.name("mqtt-endpoint-" + safeName(ep.getName())) // 多 endpoint 互不影响:单个连接初始化失败会派发 operationalError但不会阻止其他 endpoint。
.start(() -> runEndpoint(ep)); log.info("mqtt endpoint [{}] initializing uri={} topic={} profile={} qos={} cleanSession={} tlsHostnameVerification={}",
workers.add(worker); ep.getName(), ep.getUri(), ep.getTopic(), ep.getProfile(), ep.getQos(), ep.isCleanSession(),
ep.getTls() == null || ep.getTls().isHostnameVerificationEnabled());
MqttAsyncClient client = buildClient(ep);
clients.add(client);
connectAndSubscribe(client, ep);
} catch (Exception e) {
log.error("mqtt endpoint [{}] init failed, skipped", ep.getName(), e);
publishOperationalError(ep, "init", e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
}
} }
} }
private void runEndpoint(MqttInboundProperties.Endpoint ep) { private MqttAsyncClient buildClient(MqttInboundProperties.Endpoint ep) throws MqttException {
while (running && !Thread.currentThread().isInterrupted()) { String clientId = ep.getClientId() == null || ep.getClientId().isBlank()
FutureConnection connection = null; ? "lingniu-" + UUID.randomUUID() : ep.getClientId();
try { return new MqttAsyncClient(pahoServerUri(ep.getUri()), clientId, new MemoryPersistence());
log.info("mqtt endpoint [{}] initializing uri={} topic={} qos={} tls={}", }
ep.getName(), ep.getUri(), ep.getTopic(), ep.getQos(), hasCustomTls(ep.getTls()));
MQTT mqtt = buildClient(ep); private void connectAndSubscribe(MqttAsyncClient client, MqttInboundProperties.Endpoint ep) throws MqttException {
connection = mqtt.futureConnection(); client.setCallback(new MqttCallbackExtended() {
connections.add(connection); @Override
log.info("mqtt endpoint [{}] connect sending", ep.getName()); public void connectComplete(boolean reconnect, String serverURI) {
connection.connect().await(); if (!reconnect) {
log.info("mqtt endpoint [{}] connected", ep.getName());
connection.subscribe(new Topic[]{new Topic(ep.getTopic(), mapQos(ep.getQos()))}).await();
log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
receiveLoop(ep, connection);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return; return;
}
try {
client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep));
log.info("mqtt endpoint [{}] re-subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
} catch (Exception e) { } catch (Exception e) {
if (isPeerDisconnect(e)) { log.error("mqtt endpoint [{}] re-subscribe failed topic={}", ep.getName(), ep.getTopic(), e);
log.warn("mqtt endpoint [{}] peer disconnected, reconnecting: {}", ep.getName(), e.getMessage()); publishOperationalError(ep, "subscribe",
} else {
log.error("mqtt endpoint [{}] connect/receive failed", ep.getName(), e);
publishOperationalError(ep, "connect",
e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage()); e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
} }
sleepBeforeReconnect(ep);
} finally {
if (connection != null) {
connections.remove(connection);
try {
connection.disconnect().await();
} catch (Exception ignored) {
// best-effort close
}
}
}
}
} }
private MQTT buildClient(MqttInboundProperties.Endpoint ep) throws Exception { @Override
MQTT mqtt = new MQTT(); public void connectionLost(Throwable cause) {
mqtt.setHost(ep.getUri()); log.warn("mqtt endpoint [{}] connection lost", ep.getName(), cause);
if (ep.getClientId() != null && !ep.getClientId().isBlank()) {
mqtt.setClientId(ep.getClientId());
}
mqtt.setCleanSession(ep.isCleanSession());
if (ep.getUsername() != null && !ep.getUsername().isBlank()) {
mqtt.setUserName(ep.getUsername());
mqtt.setPassword(ep.getPassword() == null ? "" : ep.getPassword());
}
mqtt.setConnectAttemptsMax(1);
mqtt.setReconnectAttemptsMax(0);
if (ep.getKeepAliveSeconds() > 0) {
mqtt.setKeepAlive((short) ep.getKeepAliveSeconds());
}
mqtt.setVersion("3.1.1");
if (hasCustomTls(ep.getTls())) {
mqtt.setSslContext(MqttTlsSupport.buildSslContext(ep.getTls()));
}
return mqtt;
} }
private void receiveLoop(MqttInboundProperties.Endpoint ep, FutureConnection connection) throws Exception { @Override
while (running && !Thread.currentThread().isInterrupted()) { public void messageArrived(String topic, MqttMessage message) {
Future<Message> futureMessage = connection.receive(); log.info("mqtt endpoint [{}] received topic={} bytes={}", ep.getName(), topic, message.getPayload().length);
Message message = futureMessage.await(); onMessage(ep, topic, message.getPayload());
try {
message.ack();
onMessage(ep, message.getTopic(), message.getPayload());
} catch (Exception e) {
log.warn("mqtt endpoint [{}] message ack/dispatch failed topic={} len={}",
ep.getName(), message.getTopic(), message.getPayload() == null ? 0 : message.getPayload().length, e);
}
}
} }
private boolean isPeerDisconnect(Exception e) { @Override
Throwable current = e; public void deliveryComplete(IMqttDeliveryToken token) {
while (current != null) { // inbound only
if (current instanceof EOFException) {
return true;
}
current = current.getCause();
}
return false;
} }
});
private void sleepBeforeReconnect(MqttInboundProperties.Endpoint ep) { MqttConnectOptions options = connectOptions(ep);
try { client.connect(options).waitForCompletion(timeoutMs(ep));
Thread.sleep(5000); log.info("mqtt endpoint [{}] connected serverUri={} clientId={}",
} catch (InterruptedException e) { ep.getName(), client.getServerURI(), client.getClientId());
Thread.currentThread().interrupt(); client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep));
} log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
} }
private void onMessage(MqttInboundProperties.Endpoint ep, String topic, byte[] payload) { private void onMessage(MqttInboundProperties.Endpoint ep, String topic, byte[] payload) {
@@ -319,11 +288,58 @@ public final class MqttEndpointManager implements AutoCloseable {
return value.replace("\\", "\\\\").replace("\"", "\\\""); return value.replace("\\", "\\\\").replace("\"", "\\\"");
} }
private static QoS mapQos(int level) { private static MqttConnectOptions connectOptions(MqttInboundProperties.Endpoint ep) {
MqttConnectOptions options = new MqttConnectOptions();
options.setAutomaticReconnect(true);
options.setCleanSession(ep.isCleanSession());
if (ep.getKeepAliveSeconds() > 0) {
options.setKeepAliveInterval(ep.getKeepAliveSeconds());
}
if (ep.getConnectionTimeoutSeconds() > 0) {
options.setConnectionTimeout(ep.getConnectionTimeoutSeconds());
}
if (hasText(ep.getUsername())) {
options.setUserName(ep.getUsername());
options.setPassword(ep.getPassword() == null ? new char[0] : ep.getPassword().toCharArray());
}
if (isSslUri(ep.getUri()) && hasCustomTls(ep.getTls())) {
options.setSocketFactory(MqttTlsSupport.socketFactory(ep.getTls()));
if (ep.getTls() != null && !ep.getTls().isHostnameVerificationEnabled()) {
options.setSSLHostnameVerifier((hostname, session) -> true);
}
}
return options;
}
private static String pahoServerUri(String value) {
URI uri = URI.create(value);
String scheme = uri.getScheme();
String pahoScheme = switch (scheme == null ? "" : scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> "ssl";
default -> "tcp";
};
int port = uri.getPort() > 0 ? uri.getPort() : defaultPort(scheme);
return pahoScheme + "://" + uri.getHost() + ":" + port;
}
private static int mapQos(int level) {
return switch (level) { return switch (level) {
case 0 -> QoS.AT_MOST_ONCE; case 0 -> 0;
case 2 -> QoS.EXACTLY_ONCE; case 2 -> 2;
default -> QoS.AT_LEAST_ONCE; default -> 1;
};
}
private static long timeoutMs(MqttInboundProperties.Endpoint ep) {
int seconds = ep.getConnectionTimeoutSeconds() > 0 ? ep.getConnectionTimeoutSeconds() : 10;
return TimeUnit.SECONDS.toMillis(seconds);
}
private static int defaultPort(String scheme) {
if (scheme == null) return 1883;
return switch (scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> 8883;
default -> 1883;
}; };
} }
@@ -331,13 +347,6 @@ public final class MqttEndpointManager implements AutoCloseable {
return value != null && value.trim().length() == 17; return value != null && value.trim().length() == 17;
} }
private static String safeName(String value) {
if (value == null || value.isBlank()) {
return "unnamed";
}
return value.replaceAll("[^a-zA-Z0-9_.-]", "_");
}
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {} private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}
private static boolean hasCustomTls(MqttInboundProperties.Tls tls) { private static boolean hasCustomTls(MqttInboundProperties.Tls tls) {
@@ -345,23 +354,30 @@ public final class MqttEndpointManager implements AutoCloseable {
|| (hasText(tls.getClientPem()) && hasText(tls.getClientKey()))); || (hasText(tls.getClientPem()) && hasText(tls.getClientKey())));
} }
private static boolean isSslUri(String value) {
String scheme = URI.create(value).getScheme();
return scheme != null && switch (scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> true;
default -> false;
};
}
private static boolean hasText(String value) { private static boolean hasText(String value) {
return value != null && !value.isBlank(); return value != null && !value.isBlank();
} }
@Override @Override
public void close() { public void close() {
running = false; for (MqttAsyncClient c : clients) {
for (FutureConnection connection : List.copyOf(connections)) {
try { try {
connection.disconnect().await(); if (c.isConnected()) {
c.disconnectForcibly(1000, 1000, false);
}
c.close(true);
} catch (Exception ignored) { } catch (Exception ignored) {
// best-effort // best-effort
} }
} }
for (Thread worker : workers) { log.info("mqtt endpoint manager stopped, clients={}", clients.size());
worker.interrupt();
}
log.info("mqtt endpoint manager stopped, connections={}", connections.size());
} }
} }

View File

@@ -2,19 +2,31 @@ package com.lingniu.ingest.inbound.mqtt.client;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties; import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLContext; import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.TrustManagerFactory;
import java.io.InputStream; import javax.net.ssl.X509ExtendedTrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.Socket;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.security.KeyFactory; import java.security.KeyFactory;
import java.security.KeyStore; import java.security.KeyStore;
import java.security.PrivateKey; import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.cert.Certificate; import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory; import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.PKCS8EncodedKeySpec;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Base64; import java.util.Base64;
@@ -25,93 +37,81 @@ public final class MqttTlsSupport {
private MqttTlsSupport() {} private MqttTlsSupport() {}
public static SSLContext buildSslContext(MqttInboundProperties.Tls tls) { public static SSLSocketFactory socketFactory(MqttInboundProperties.Tls tls) {
try { try {
TrustManagerFactory tmf = null; KeyManager[] keyManagers = null;
KeyManagerFactory kmf = null; TrustManager[] trustManagers = null;
if (tls != null && hasText(tls.getCaPem())) { if (tls != null && hasText(tls.getCaPem())) {
tmf = trustManagerFactory(Path.of(tls.getCaPem())); trustManagers = trustManagerFactory(Path.of(tls.getCaPem())).getTrustManagers();
if (!tls.isHostnameVerificationEnabled()) {
trustManagers = withoutHostnameVerification(trustManagers);
}
} }
if (tls != null && hasText(tls.getClientPem()) && hasText(tls.getClientKey())) { if (tls != null && hasText(tls.getClientPem()) && hasText(tls.getClientKey())) {
kmf = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey())); keyManagers = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey())).getKeyManagers();
} }
SSLContext context = SSLContext.getInstance("TLSv1.2"); SSLContext context = SSLContext.getInstance("TLS");
context.init( context.init(keyManagers, trustManagers, null);
kmf == null ? null : kmf.getKeyManagers(), SSLSocketFactory factory = context.getSocketFactory();
tmf == null ? null : tmf.getTrustManagers(), if (tls != null && !tls.isHostnameVerificationEnabled()) {
new SecureRandom()); return new HostnameVerificationDisablingSocketFactory(factory);
return context; }
return factory;
} catch (Exception e) { } catch (Exception e) {
throw new IllegalStateException("mqtt tls config build failed", e); throw new IllegalStateException("mqtt tls config build failed", e);
} }
} }
private static TrustManagerFactory trustManagerFactory(Path caPath) throws Exception { private static TrustManagerFactory trustManagerFactory(Path caPem) throws Exception {
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(null, null); trustStore.load(null, null);
int i = 0; int i = 0;
for (Certificate cert : readCertificates(caPath)) { for (Certificate cert : readCertificates(caPem)) {
trustStore.setCertificateEntry("ca-" + i++, cert); trustStore.setCertificateEntry("ca-" + i++, cert);
} }
TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX"); TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore); tmf.init(trustStore);
return tmf; return tmf;
} }
private static KeyManagerFactory keyManagerFactory(Path clientCertPath, Path clientKeyPath) throws Exception { private static KeyManagerFactory keyManagerFactory(Path clientPem, Path clientKey) throws Exception {
List<Certificate> chain = readCertificates(clientCertPath); List<Certificate> chain = readCertificates(clientPem);
PrivateKey key = readPrivateKey(clientKeyPath); PrivateKey key = readPrivateKey(clientKey);
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
keyStore.load(null, null); keyStore.load(null, null);
char[] password = new char[0]; char[] password = new char[0];
keyStore.setCertificateEntry("client", chain.getFirst()); keyStore.setKeyEntry("mqtt-client", key, password, chain.toArray(Certificate[]::new));
keyStore.setKeyEntry("clientKey", key, password, chain.toArray(Certificate[]::new)); KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX");
kmf.init(keyStore, password); kmf.init(keyStore, password);
return kmf; return kmf;
} }
private static List<Certificate> readCertificates(Path path) throws Exception { private static List<Certificate> readCertificates(Path pem) throws Exception {
CertificateFactory factory = CertificateFactory.getInstance("X.509"); String text = Files.readString(pem, StandardCharsets.UTF_8);
String text = readUtf8IfPossible(path);
if (text != null && text.contains("-----BEGIN CERTIFICATE-----")) {
List<Certificate> certificates = new ArrayList<>(); List<Certificate> certificates = new ArrayList<>();
CertificateFactory factory = CertificateFactory.getInstance("X.509");
for (String block : pemBlocks(text, "CERTIFICATE")) { for (String block : pemBlocks(text, "CERTIFICATE")) {
certificates.add(factory.generateCertificate( try (ByteArrayInputStream in = new ByteArrayInputStream(Base64.getMimeDecoder().decode(block))) {
new java.io.ByteArrayInputStream(Base64.getMimeDecoder().decode(block)))); certificates.add(factory.generateCertificate(in));
}
}
if (certificates.isEmpty()) {
throw new IllegalArgumentException("no certificate found in " + pem);
} }
if (!certificates.isEmpty()) {
return certificates; return certificates;
} }
}
try (InputStream in = Files.newInputStream(path)) {
Collection<? extends Certificate> certificates = factory.generateCertificates(in);
if (certificates.isEmpty()) {
throw new IllegalArgumentException("no certificate found in " + path);
}
return List.copyOf(certificates);
}
}
private static PrivateKey readPrivateKey(Path path) throws Exception { private static PrivateKey readPrivateKey(Path pem) throws Exception {
String text = Files.readString(path, StandardCharsets.UTF_8); String text = Files.readString(pem, StandardCharsets.UTF_8);
List<String> blocks = pemBlocks(text, "PRIVATE KEY"); List<String> blocks = pemBlocks(text, "PRIVATE KEY");
if (blocks.isEmpty()) { if (blocks.isEmpty()) {
throw new IllegalArgumentException("no PKCS#8 private key found in " + path); throw new IllegalArgumentException("no PKCS#8 private key found in " + pem);
} }
byte[] der = Base64.getMimeDecoder().decode(blocks.getFirst()); byte[] der = Base64.getMimeDecoder().decode(blocks.getFirst());
PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der); PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der);
return KeyFactory.getInstance("RSA").generatePrivate(spec); return KeyFactory.getInstance("RSA").generatePrivate(spec);
} }
private static String readUtf8IfPossible(Path path) {
try {
return Files.readString(path, StandardCharsets.UTF_8);
} catch (Exception ignored) {
return null;
}
}
private static List<String> pemBlocks(String text, String type) { private static List<String> pemBlocks(String text, String type) {
String begin = "-----BEGIN " + type + "-----"; String begin = "-----BEGIN " + type + "-----";
String end = "-----END " + type + "-----"; String end = "-----END " + type + "-----";
@@ -132,4 +132,129 @@ public final class MqttTlsSupport {
private static boolean hasText(String value) { private static boolean hasText(String value) {
return value != null && !value.isBlank(); return value != null && !value.isBlank();
} }
private static TrustManager[] withoutHostnameVerification(TrustManager[] managers) {
TrustManager[] wrapped = new TrustManager[managers.length];
for (int i = 0; i < managers.length; i++) {
TrustManager manager = managers[i];
if (manager instanceof X509TrustManager x509) {
wrapped[i] = new ChainOnlyTrustManager(x509);
} else {
wrapped[i] = manager;
}
}
return wrapped;
}
private static final class ChainOnlyTrustManager extends X509ExtendedTrustManager {
private final X509TrustManager delegate;
private ChainOnlyTrustManager(X509TrustManager delegate) {
this.delegate = delegate;
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket)
throws CertificateException {
if (delegate instanceof X509ExtendedTrustManager extended) {
extended.checkClientTrusted(chain, authType, socket);
} else {
delegate.checkClientTrusted(chain, authType);
}
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket)
throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
throws CertificateException {
if (delegate instanceof X509ExtendedTrustManager extended) {
extended.checkClientTrusted(chain, authType, engine);
} else {
delegate.checkClientTrusted(chain, authType);
}
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
delegate.checkClientTrusted(chain, authType);
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return delegate.getAcceptedIssuers();
}
}
private static final class HostnameVerificationDisablingSocketFactory extends SSLSocketFactory {
private final SSLSocketFactory delegate;
private HostnameVerificationDisablingSocketFactory(SSLSocketFactory delegate) {
this.delegate = delegate;
}
@Override
public String[] getDefaultCipherSuites() {
return delegate.getDefaultCipherSuites();
}
@Override
public String[] getSupportedCipherSuites() {
return delegate.getSupportedCipherSuites();
}
@Override
public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException {
return prepare(delegate.createSocket(socket, host, port, autoClose));
}
@Override
public Socket createSocket() throws IOException {
return prepare(delegate.createSocket());
}
@Override
public Socket createSocket(String host, int port) throws IOException {
return prepare(delegate.createSocket(host, port));
}
@Override
public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException {
return prepare(delegate.createSocket(host, port, localHost, localPort));
}
@Override
public Socket createSocket(InetAddress host, int port) throws IOException {
return prepare(delegate.createSocket(host, port));
}
@Override
public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException {
return prepare(delegate.createSocket(address, port, localAddress, localPort));
}
private static Socket prepare(Socket socket) {
if (socket instanceof SSLSocket sslSocket) {
SSLParameters parameters = sslSocket.getSSLParameters();
parameters.setEndpointIdentificationAlgorithm(null);
sslSocket.setSSLParameters(parameters);
}
return socket;
}
}
} }

View File

@@ -10,19 +10,20 @@ import com.lingniu.ingest.inbound.mqtt.parser.MqttPayloadParser;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfile; import com.lingniu.ingest.inbound.mqtt.profile.MqttProfile;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry; import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import com.lingniu.ingest.inbound.mqtt.profile.YutongMqttProfile; import com.lingniu.ingest.inbound.mqtt.profile.YutongMqttProfile;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import java.util.ArrayList;
import java.util.List; import java.util.List;
@AutoConfiguration @AutoConfiguration
@EnableConfigurationProperties(MqttInboundProperties.class) @EnableConfigurationProperties(MqttInboundProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.mqtt", name = "enabled", havingValue = "true") @Conditional(MqttInboundEnabledCondition.class)
public class MqttInboundAutoConfiguration { public class MqttInboundAutoConfiguration {
@Bean @Bean
@@ -61,25 +62,144 @@ public class MqttInboundAutoConfiguration {
return new MqttRealtimeHandler(profileRegistry); return new MqttRealtimeHandler(profileRegistry);
} }
@Bean(destroyMethod = "close") @Bean(initMethod = "start", destroyMethod = "close")
@Lazy(false)
@ConditionalOnMissingBean @ConditionalOnMissingBean
public MqttEndpointManager mqttEndpointManager(MqttInboundProperties props, public MqttEndpointManager mqttEndpointManager(MqttInboundProperties props,
MqttProfileRegistry profileRegistry, MqttProfileRegistry profileRegistry,
VehicleIdentityResolver identityResolver, VehicleIdentityResolver identityResolver,
Dispatcher dispatcher) { Dispatcher dispatcher,
Environment environment) {
applyLegacySingleEndpointFallback(props, environment);
return new MqttEndpointManager(props, profileRegistry, identityResolver, dispatcher); return new MqttEndpointManager(props, profileRegistry, identityResolver, dispatcher);
} }
/** private static void applyLegacySingleEndpointFallback(MqttInboundProperties props, Environment env) {
* Spring Boot 启动完成后触发 MQTT 连接,避免与 Netty / Kafka 启动竞争。 if (!props.getEndpoints().isEmpty()) {
*/ return;
@Bean
public ApplicationRunner mqttStartupRunner(MqttEndpointManager manager) {
return new ApplicationRunner() {
@Override
public void run(ApplicationArguments args) {
manager.start();
} }
}; String uri = firstText(env,
"lingniu.ingest.mqtt.endpoints[0].uri",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_URI",
"MQTT_URI",
"YUTONG_MQTT_URI");
if (uri.isBlank() || uri.contains("${")) {
return;
}
MqttInboundProperties.Endpoint endpoint = new MqttInboundProperties.Endpoint();
endpoint.setName(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].name",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_NAME",
"MQTT_ENDPOINT_NAME",
"YUTONG_MQTT_ENDPOINT_NAME"), "yutong"));
endpoint.setUri(uri);
endpoint.setTopic(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].topic",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TOPIC",
"MQTT_TOPIC",
"YUTONG_MQTT_TOPIC"), "#"));
endpoint.setQos(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].qos",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_QOS",
"MQTT_QOS",
"YUTONG_MQTT_QOS"), 1));
endpoint.setClientId(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].client-id",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLIENT_ID",
"MQTT_CLIENT_ID",
"YUTONG_MQTT_CLIENT_ID"));
endpoint.setUsername(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].username",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_USERNAME",
"MQTT_USERNAME",
"YUTONG_MQTT_USERNAME"));
endpoint.setPassword(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].password",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_PASSWORD",
"MQTT_PASSWORD",
"YUTONG_MQTT_PASSWORD"));
endpoint.setCleanSession(booleanValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].clean-session",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLEAN_SESSION",
"MQTT_CLEAN_SESSION",
"YUTONG_MQTT_CLEAN_SESSION"), false));
endpoint.setKeepAliveSeconds(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].keep-alive-seconds",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_KEEP_ALIVE_SECONDS",
"MQTT_KEEP_ALIVE_SECONDS",
"YUTONG_MQTT_KEEP_ALIVE_SECONDS"), 20));
endpoint.setConnectionTimeoutSeconds(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].connection-timeout-seconds",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CONNECTION_TIMEOUT_SECONDS",
"MQTT_CONNECTION_TIMEOUT_SECONDS",
"YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS"), 10));
endpoint.setProfile(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].profile",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_PROFILE",
"MQTT_PROFILE",
"YUTONG_MQTT_PROFILE"), "yutong"));
MqttInboundProperties.Tls tls = endpoint.getTls();
tls.setCaPem(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.ca-pem",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CA_PEM",
"MQTT_TLS_CA_PEM",
"YUTONG_MQTT_TLS_CA_PEM"));
tls.setClientPem(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.client-pem",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_PEM",
"MQTT_TLS_CLIENT_PEM",
"YUTONG_MQTT_TLS_CLIENT_PEM"));
tls.setClientKey(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.client-key",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_KEY",
"MQTT_TLS_CLIENT_KEY",
"YUTONG_MQTT_TLS_CLIENT_KEY"));
tls.setHostnameVerificationEnabled(booleanValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.hostname-verification-enabled",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_HOSTNAME_VERIFICATION_ENABLED",
"MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED",
"YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED"), true));
props.setEndpoints(new ArrayList<>(List.of(endpoint)));
}
private static String firstText(Environment env, String... keys) {
for (String key : keys) {
String value = env.getProperty(key);
if (value != null && !value.isBlank() && !value.contains("${")) {
return value;
}
}
return "";
}
private static String defaultText(String value, String fallback) {
return value == null || value.isBlank() ? fallback : value;
}
private static int intValue(String value, int fallback) {
if (value == null || value.isBlank()) {
return fallback;
}
try {
return Integer.parseInt(value.trim());
} catch (NumberFormatException ex) {
return fallback;
}
}
private static boolean booleanValue(String value, boolean fallback) {
if (value == null || value.isBlank()) {
return fallback;
}
String normalized = value.trim().toLowerCase();
if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized)
|| "on".equals(normalized)) {
return true;
}
if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized)
|| "off".equals(normalized)) {
return false;
}
return fallback;
} }
} }

View File

@@ -0,0 +1,41 @@
package com.lingniu.ingest.inbound.mqtt.config;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.env.Environment;
import org.springframework.core.type.AnnotatedTypeMetadata;
final class MqttInboundEnabledCondition implements Condition {
@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
Environment env = context.getEnvironment();
Boolean direct = parseBoolean(env.getProperty("lingniu.ingest.mqtt.enabled"));
if (direct != null) {
return direct;
}
return isTrue(env.getProperty("MQTT_ENABLED"))
|| isTrue(env.getProperty("YUTONG_MQTT_ENABLED"));
}
private static boolean isTrue(String value) {
Boolean parsed = parseBoolean(value);
return parsed != null && parsed;
}
private static Boolean parseBoolean(String value) {
if (value == null || value.isBlank() || value.contains("${")) {
return null;
}
String normalized = value.trim().toLowerCase();
if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized)
|| "on".equals(normalized)) {
return true;
}
if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized)
|| "off".equals(normalized)) {
return false;
}
return null;
}
}

View File

@@ -82,6 +82,8 @@ public class MqttInboundProperties {
public String getClientKey() { return clientKey; } public String getClientKey() { return clientKey; }
public void setClientKey(String clientKey) { this.clientKey = clientKey; } public void setClientKey(String clientKey) { this.clientKey = clientKey; }
public boolean isHostnameVerificationEnabled() { return hostnameVerificationEnabled; } public boolean isHostnameVerificationEnabled() { return hostnameVerificationEnabled; }
public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) { this.hostnameVerificationEnabled = hostnameVerificationEnabled; } public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) {
this.hostnameVerificationEnabled = hostnameVerificationEnabled;
}
} }
} }

View File

@@ -172,7 +172,7 @@ class MqttEndpointManagerTest {
.containsEntry("endpoint", "bad-endpoint") .containsEntry("endpoint", "bad-endpoint")
.containsEntry("profile", "yutong") .containsEntry("profile", "yutong")
.containsEntry("operationalError", "true") .containsEntry("operationalError", "true")
.containsEntry("phase", "connect") .containsEntry("phase", "init")
.containsEntry("identityResolved", "false") .containsEntry("identityResolved", "false")
.containsEntry("identitySource", "UNKNOWN"); .containsEntry("identitySource", "UNKNOWN");
}); });
@@ -184,7 +184,7 @@ class MqttEndpointManagerTest {
.containsEntry("operationalError", "true") .containsEntry("operationalError", "true")
.containsEntry("endpoint", "bad-endpoint") .containsEntry("endpoint", "bad-endpoint")
.containsEntry("profile", "yutong") .containsEntry("profile", "yutong")
.containsEntry("phase", "connect"); .containsEntry("phase", "init");
}); });
batchExecutor.close(); batchExecutor.close();

View File

@@ -5,11 +5,10 @@ import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.api.io.TempDir;
import javax.net.ssl.SSLSocketFactory;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import javax.net.ssl.SSLContext;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
class MqttTlsSupportTest { class MqttTlsSupportTest {
@@ -24,9 +23,9 @@ class MqttTlsSupportTest {
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls(); MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
tls.setCaPem(ca.toString()); tls.setCaPem(ca.toString());
SSLContext config = MqttTlsSupport.buildSslContext(tls); SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2"); assertThat(factory).isNotNull();
} }
@Test @Test
@@ -42,22 +41,9 @@ class MqttTlsSupportTest {
tls.setClientPem(cert.toString()); tls.setClientPem(cert.toString());
tls.setClientKey(key.toString()); tls.setClientKey(key.toString());
SSLContext config = MqttTlsSupport.buildSslContext(tls); SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2"); assertThat(factory).isNotNull();
}
@Test
void hostnameVerificationFlagDoesNotBlockSslContextCreation() throws Exception {
Path ca = tempDir.resolve("ca.pem");
Files.writeString(ca, CERT);
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
tls.setCaPem(ca.toString());
tls.setHostnameVerificationEnabled(false);
SSLContext config = MqttTlsSupport.buildSslContext(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
} }
private static final String CERT = """ private static final String CERT = """

View File

@@ -41,6 +41,30 @@
<groupId>io.netty</groupId> <groupId>io.netty</groupId>
<artifactId>netty-all</artifactId> <artifactId>netty-all</artifactId>
</dependency> </dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>

View File

@@ -6,6 +6,7 @@ import com.gps31.push.netty.PushClient;
import com.gps31.push.netty.PushMsg; import com.gps31.push.netty.PushMsg;
import com.gps31.push.netty.client.TcpClient; import com.gps31.push.netty.client.TcpClient;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.pipeline.RawFrame; import com.lingniu.ingest.api.pipeline.RawFrame;
import com.lingniu.ingest.core.dispatcher.Dispatcher; import com.lingniu.ingest.core.dispatcher.Dispatcher;
import com.lingniu.ingest.identity.InMemoryVehicleIdentityService; import com.lingniu.ingest.identity.InMemoryVehicleIdentityService;
@@ -220,6 +221,7 @@ public class XindaPushClient extends PushClient {
meta.put("parseError", "true"); meta.put("parseError", "true");
meta.put("parseErrorMessage", parsed.errorMessage()); meta.put("parseErrorMessage", parsed.errorMessage());
} }
meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(payload.command(), json, parsed));
dispatcher.dispatch(new RawFrame( dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH, ProtocolId.XINDA_PUSH,
payload.commandCode(), payload.commandCode(),
@@ -247,6 +249,8 @@ public class XindaPushClient extends PushClient {
meta.put("operationalError", "true"); meta.put("operationalError", "true");
meta.put("phase", phase == null ? "" : phase); meta.put("phase", phase == null ? "" : phase);
meta.put("reason", reason == null ? "" : reason); meta.put("reason", reason == null ? "" : reason);
meta.put(RawArchiveKeys.META_PARSED_JSON,
parsedJson(payload.command(), payload.json(), new JsonParseResult(json, false, "")));
dispatcher.dispatch(new RawFrame( dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH, ProtocolId.XINDA_PUSH,
payload.commandCode(), payload.commandCode(),
@@ -271,6 +275,8 @@ public class XindaPushClient extends PushClient {
meta.put("operationalError", "true"); meta.put("operationalError", "true");
meta.put("phase", "message"); meta.put("phase", "message");
meta.put("reason", reason == null ? "" : reason); meta.put("reason", reason == null ? "" : reason);
JsonParseResult parsed = parseForMetadata(json);
meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(cmd, json, parsed));
dispatcher.dispatch(new RawFrame( dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH, ProtocolId.XINDA_PUSH,
0, 0,
@@ -388,6 +394,19 @@ public class XindaPushClient extends PushClient {
} }
} }
private static String parsedJson(String command, String rawJson, JsonParseResult parsed) {
JSONObject root = new JSONObject(true);
root.put("command", command == null ? "" : command);
root.put("parseError", parsed != null && parsed.parseError());
if (parsed != null && parsed.parseError()) {
root.put("parseErrorMessage", parsed.errorMessage() == null ? "" : parsed.errorMessage());
root.put("rawJson", rawJson == null ? "" : rawJson);
} else {
root.put("body", parsed == null || parsed.json() == null ? new JSONObject() : parsed.json());
}
return root.toJSONString();
}
private record JsonParseResult(JSONObject json, boolean parseError, String errorMessage) {} private record JsonParseResult(JSONObject json, boolean parseError, String errorMessage) {}
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {} private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}

View File

@@ -185,7 +185,6 @@ public final class XindaPushEventMapper implements EventMapper<XindaPushMessage>
Map<String, String> out = new java.util.LinkedHashMap<>(); Map<String, String> out = new java.util.LinkedHashMap<>();
out.put("source", "xinda-push"); out.put("source", "xinda-push");
out.put("cmd", message.command()); out.put("cmd", message.command());
out.put("rawJson", message.json());
out.put("vin", normalizedVin(identity)); out.put("vin", normalizedVin(identity));
out.put("externalVin", firstNonBlank(json, "vin")); out.put("externalVin", firstNonBlank(json, "vin"));
out.put("phone", phone(json)); out.put("phone", phone(json));

View File

@@ -37,7 +37,8 @@ class XindaPushEventMapperTest {
.containsEntry("externalVin", "LNVIN000000000101") .containsEntry("externalVin", "LNVIN000000000101")
.containsEntry("phone", "13800138000") .containsEntry("phone", "13800138000")
.containsEntry("deviceId", "XD-101") .containsEntry("deviceId", "XD-101")
.containsEntry("plateNo", "粤B10101"); .containsEntry("plateNo", "粤B10101")
.doesNotContainKey("rawJson");
} }
@Test @Test

View File

@@ -43,7 +43,7 @@ class Gb32960ChannelHandlerAckBoundaryTest {
dispatch.awaitPublishCount(1); dispatch.awaitPublishCount(1);
dispatch.completeDurability(); dispatch.completeDurability();
ByteBuf ack = readOutboundAfterRunningTasks(channel); ByteBuf ack = awaitOutbound(channel);
assertThat(ack).isNotNull(); assertThat(ack).isNotNull();
assertThat(ack.getUnsignedByte(2)).isEqualTo((short) 0x02); assertThat(ack.getUnsignedByte(2)).isEqualTo((short) 0x02);
assertThat(ack.getUnsignedByte(3)).isEqualTo((short) 0x01); assertThat(ack.getUnsignedByte(3)).isEqualTo((short) 0x01);
@@ -85,8 +85,8 @@ class Gb32960ChannelHandlerAckBoundaryTest {
dispatch.completeDurability(0); dispatch.completeDurability(0);
ByteBuf firstAck = readOutboundAfterRunningTasks(channel); ByteBuf firstAck = awaitOutbound(channel);
ByteBuf secondAck = readOutboundAfterRunningTasks(channel); ByteBuf secondAck = awaitOutbound(channel);
assertThat(firstAck).isNotNull(); assertThat(firstAck).isNotNull();
assertThat(secondAck).isNotNull(); assertThat(secondAck).isNotNull();
assertThat(firstAck.getUnsignedByte(2)).isEqualTo((short) 0x02); assertThat(firstAck.getUnsignedByte(2)).isEqualTo((short) 0x02);
@@ -97,42 +97,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
} }
} }
@Test
void decodeFailureIsArchivedWithParseErrorMetadata() {
try (DispatchHarness dispatch = new DispatchHarness()) {
EmbeddedChannel channel = new EmbeddedChannel(handlerWith(dispatch.dispatcher()));
byte[] malformed = new byte[]{0x23, 0x23, 0x02};
assertThat(channel.writeInbound(malformed)).isFalse();
dispatch.awaitPublishCount(1);
VehicleEvent event = dispatch.event(0);
assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class);
VehicleEvent.RawArchive raw = (VehicleEvent.RawArchive) event;
assertThat(raw.source()).isEqualTo(com.lingniu.ingest.api.ProtocolId.GB32960);
assertThat(raw.command()).isZero();
assertThat(raw.rawBytes()).containsExactly(malformed);
assertThat(raw.metadata())
.containsEntry("vin", "unknown")
.containsEntry("parseError", "true")
.containsKey("parseErrorMessage");
channel.finishAndReleaseAll();
}
}
private static ByteBuf readOutboundAfterRunningTasks(EmbeddedChannel channel) {
long deadline = System.currentTimeMillis() + 3000;
while (System.currentTimeMillis() < deadline) {
channel.runPendingTasks();
ByteBuf outbound = channel.readOutbound();
if (outbound != null) {
return outbound;
}
Thread.onSpinWait();
}
return null;
}
private static Gb32960ChannelHandler handlerWith(Dispatcher dispatcher) { private static Gb32960ChannelHandler handlerWith(Dispatcher dispatcher) {
return new Gb32960ChannelHandler( return new Gb32960ChannelHandler(
decoder(), decoder(),
@@ -174,6 +138,19 @@ class Gb32960ChannelHandlerAckBoundaryTest {
return out.toByteArray(); return out.toByteArray();
} }
private static ByteBuf awaitOutbound(EmbeddedChannel channel) {
long deadline = System.currentTimeMillis() + 3000;
while (System.currentTimeMillis() < deadline) {
channel.runPendingTasks();
ByteBuf outbound = channel.readOutbound();
if (outbound != null) {
return outbound;
}
Thread.onSpinWait();
}
return null;
}
private static final class DispatchHarness implements AutoCloseable { private static final class DispatchHarness implements AutoCloseable {
private final ControlledSink sink = new ControlledSink(); private final ControlledSink sink = new ControlledSink();
private final DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink)); private final DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
@@ -214,12 +191,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
throw new AssertionError("expected " + count + " sink publishes, actual=" + sink.futures.size()); throw new AssertionError("expected " + count + " sink publishes, actual=" + sink.futures.size());
} }
VehicleEvent event(int index) {
synchronized (sink.futures) {
return sink.events.get(index);
}
}
@Override @Override
public void close() { public void close() {
batchExecutor.close(); batchExecutor.close();
@@ -229,7 +200,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
private static final class ControlledSink implements EventSink { private static final class ControlledSink implements EventSink {
private final List<CompletableFuture<Void>> futures = new ArrayList<>(); private final List<CompletableFuture<Void>> futures = new ArrayList<>();
private final List<VehicleEvent> events = new ArrayList<>();
@Override @Override
public String name() { public String name() {
@@ -240,7 +210,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
public CompletableFuture<Void> publish(VehicleEvent event) { public CompletableFuture<Void> publish(VehicleEvent event) {
CompletableFuture<Void> future = new CompletableFuture<>(); CompletableFuture<Void> future = new CompletableFuture<>();
synchronized (futures) { synchronized (futures) {
events.add(event);
futures.add(future); futures.add(future);
} }
return future; return future;

View File

@@ -9,13 +9,14 @@ import com.lingniu.ingest.identity.VehicleIdentityLookup;
import com.lingniu.ingest.identity.VehicleIdentityRegistry; import com.lingniu.ingest.identity.VehicleIdentityRegistry;
import com.lingniu.ingest.identity.VehicleIdentityResolver; import com.lingniu.ingest.identity.VehicleIdentityResolver;
import com.lingniu.ingest.identity.VehicleIdentitySource; import com.lingniu.ingest.identity.VehicleIdentitySource;
import com.lingniu.ingest.identity.VehicleRegistrationBinding;
import com.lingniu.ingest.protocol.jt808.codec.Jt808FrameEncoder; import com.lingniu.ingest.protocol.jt808.codec.Jt808FrameEncoder;
import com.lingniu.ingest.protocol.jt808.codec.Jt808MalformedFrame; import com.lingniu.ingest.protocol.jt808.codec.Jt808MalformedFrame;
import com.lingniu.ingest.protocol.jt808.codec.Jt808MessageDecoder; import com.lingniu.ingest.protocol.jt808.codec.Jt808MessageDecoder;
import com.lingniu.ingest.protocol.jt808.downlink.Jt808Commands; import com.lingniu.ingest.protocol.jt808.downlink.Jt808Commands;
import com.lingniu.ingest.protocol.jt808.model.Jt808Body; import com.lingniu.ingest.protocol.jt808.model.Jt808Body;
import com.lingniu.ingest.protocol.jt808.model.Jt808Header; import com.lingniu.ingest.protocol.jt808.model.Jt808Header;
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationAdditionalInfo;
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationFlagInfo;
import com.lingniu.ingest.protocol.jt808.model.Jt808Message; import com.lingniu.ingest.protocol.jt808.model.Jt808Message;
import com.lingniu.ingest.protocol.jt808.model.Jt808MessageId; import com.lingniu.ingest.protocol.jt808.model.Jt808MessageId;
import com.lingniu.ingest.protocol.jt808.session.Jt808ChannelRegistry; import com.lingniu.ingest.protocol.jt808.session.Jt808ChannelRegistry;
@@ -25,17 +26,18 @@ import com.lingniu.ingest.session.SessionStore;
import io.netty.buffer.Unpooled; import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.util.AttributeKey;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant; import java.time.Instant;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.CompletableFuture;
/** /**
* JT808 Netty 入站处理器。职责: * JT808 Netty 入站处理器。职责:
@@ -51,8 +53,6 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
private static final Logger log = LoggerFactory.getLogger(Jt808ChannelHandler.class); private static final Logger log = LoggerFactory.getLogger(Jt808ChannelHandler.class);
private static final String PROCESSING_ERROR_PREFIX = "processing failed: "; private static final String PROCESSING_ERROR_PREFIX = "processing failed: ";
private static final AttributeKey<CompletableFuture<Void>> DURABLE_ACK_CHAIN_ATTR =
AttributeKey.valueOf("jt808.durableAckChain");
private final Jt808MessageDecoder decoder; private final Jt808MessageDecoder decoder;
private final Dispatcher dispatcher; private final Dispatcher dispatcher;
@@ -123,21 +123,29 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
// ====== 会话维护 & 同步应答匹配 ====== // ====== 会话维护 & 同步应答匹配 ======
String phone = msg.header().phone(); String phone = msg.header().phone();
IdentityResolution identity = resolveIdentity(msg); IdentityResolution identity = resolveIdentity(msg);
byte[] durableAck = null; if (msg.body() instanceof Jt808Body.Register reg) {
identity = registerIdentity(msg, reg, identity);
}
byte[] registerAck = null;
switch (msg.header().messageId()) { switch (msg.header().messageId()) {
case Jt808MessageId.TERMINAL_REGISTER -> { case Jt808MessageId.TERMINAL_REGISTER -> {
DeviceSession session = upsertSession(ctx, msg, identity.identity()); DeviceSession session = upsertSession(ctx, msg, identity.identity());
channelRegistry.bind(phone, ctx.channel()); channelRegistry.bind(phone, ctx.channel());
durableAck = registerAckFrame(phone, msg.header().serialNo(), session.token()); var command = Jt808Commands.registerAck(msg.header().serialNo(), 0, session.token());
registerAck = Jt808FrameEncoder.encode(
command.messageId(),
phone,
channelRegistry.nextSerial(phone),
command.body());
} }
case Jt808MessageId.TERMINAL_AUTH -> { case Jt808MessageId.TERMINAL_AUTH -> {
upsertSession(ctx, msg, identity.identity()); upsertSession(ctx, msg, identity.identity());
channelRegistry.bind(phone, ctx.channel()); channelRegistry.bind(phone, ctx.channel());
durableAck = terminalAckFrame(phone, msg.header().serialNo(), msg.header().messageId(), 0); ackTerminal(ctx, phone, msg.header().serialNo(), msg.header().messageId(), 0);
} }
case Jt808MessageId.TERMINAL_HEARTBEAT -> { case Jt808MessageId.TERMINAL_HEARTBEAT -> {
channelRegistry.bind(phone, ctx.channel()); channelRegistry.bind(phone, ctx.channel());
durableAck = terminalAckFrame(phone, msg.header().serialNo(), msg.header().messageId(), 0); ackTerminal(ctx, phone, msg.header().serialNo(), msg.header().messageId(), 0);
} }
case Jt808MessageId.TERMINAL_GENERAL_RESPONSE -> { case Jt808MessageId.TERMINAL_GENERAL_RESPONSE -> {
// body: ackSerial(2) + ackMsgId(2) + result(1) // body: ackSerial(2) + ackMsgId(2) + result(1)
@@ -153,7 +161,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
} }
// ====== 派发到 Dispatcher ====== // ====== 派发到 Dispatcher ======
Map<String, String> meta = new HashMap<>(6); Map<String, String> meta = new HashMap<>(32);
meta.put("vin", identity.identity().vin()); meta.put("vin", identity.identity().vin());
meta.put("phone", phone); meta.put("phone", phone);
meta.put("identityResolved", Boolean.toString(identity.identity().resolved())); meta.put("identityResolved", Boolean.toString(identity.identity().resolved()));
@@ -163,8 +171,11 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
meta.put("identityErrorMessage", identity.errorMessage()); meta.put("identityErrorMessage", identity.errorMessage());
} }
meta.put("seq", Integer.toString(msg.header().serialNo())); meta.put("seq", Integer.toString(msg.header().serialNo()));
meta.put("messageId", "0x" + Jt808MessageId.hex(msg.header().messageId()));
meta.put("messageName", Jt808MessageId.nameOf(msg.header().messageId()));
meta.put("reportType", Jt808MessageId.reportTypeOf(msg.header().messageId()));
putBodyMetadata(meta, msg.body());
meta.put("peer", addr(ctx)); meta.put("peer", addr(ctx));
addRegistrationMetadata(meta, msg.body());
RawFrame rf = new RawFrame( RawFrame rf = new RawFrame(
ProtocolId.JT808, ProtocolId.JT808,
msg.header().messageId(), msg.header().messageId(),
@@ -173,63 +184,19 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
frame, frame,
meta, meta,
Instant.now()); Instant.now());
if (durableAck == null) { if (registerAck != null) {
byte[] ack = registerAck;
dispatcher.dispatchAndAwait(rf)
.thenRun(() -> ctx.writeAndFlush(Unpooled.wrappedBuffer(ack)))
.exceptionally(ex -> {
log.warn("[jt808] register ack withheld because dispatch durability failed peer={} phone={} seq={} error={}",
addr(ctx), phone, msg.header().serialNo(),
ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
return null;
});
return;
}
dispatcher.dispatch(rf); dispatcher.dispatch(rf);
return;
}
enqueueDurableAck(ctx, phone, msg.header().messageId(), dispatcher.dispatchAndAwait(rf), durableAck);
}
private void enqueueDurableAck(ChannelHandlerContext ctx,
String phone,
int messageId,
CompletableFuture<Void> dispatchFuture,
byte[] ackFrame) {
CompletableFuture<Void> previous = ctx.channel().attr(DURABLE_ACK_CHAIN_ATTR).get();
if (previous == null) {
previous = CompletableFuture.completedFuture(null);
}
CompletableFuture<Void> next = previous
.handle((ignored, previousFailure) -> null)
.thenCompose(ignored -> dispatchFuture.handle((ok, failure) -> failure))
.thenCompose(failure -> runOnEventLoop(ctx,
() -> handleDurableAckResult(ctx, phone, messageId, failure, ackFrame)));
ctx.channel().attr(DURABLE_ACK_CHAIN_ATTR).set(next);
}
private CompletableFuture<Void> runOnEventLoop(ChannelHandlerContext ctx, Runnable task) {
CompletableFuture<Void> done = new CompletableFuture<>();
Runnable wrapped = () -> {
try {
task.run();
done.complete(null);
} catch (Throwable t) {
done.completeExceptionally(t);
}
};
if (ctx.executor().inEventLoop()) {
wrapped.run();
} else {
ctx.executor().execute(wrapped);
}
return done;
}
private void handleDurableAckResult(ChannelHandlerContext ctx,
String phone,
int messageId,
Throwable failure,
byte[] ackFrame) {
if (!ctx.channel().isActive()) {
return;
}
if (failure != null) {
log.warn("[jt808] dispatch failed before ack peer={} phone={} msgId=0x{}",
addr(ctx), phone, Integer.toHexString(messageId), failure);
ctx.close();
return;
}
ctx.writeAndFlush(Unpooled.wrappedBuffer(ackFrame));
} }
private IdentityResolution resolveIdentity(Jt808Message msg) { private IdentityResolution resolveIdentity(Jt808Message msg) {
@@ -267,6 +234,28 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
return new VehicleIdentity("unknown", false, identity.source()); return new VehicleIdentity("unknown", false, identity.source());
} }
private IdentityResolution registerIdentity(Jt808Message msg,
Jt808Body.Register reg,
IdentityResolution current) {
VehicleIdentity currentIdentity = current.identity();
boolean alreadyResolved = currentIdentity != null && currentIdentity.resolved();
String vin = alreadyResolved ? currentIdentity.vin() : generatedInternalVin(msg.header().phone(), reg);
identityRegistry.bind(new VehicleIdentityBinding(
ProtocolId.JT808,
vin,
msg.header().phone(),
reg.deviceId(),
reg.plate(),
Map.of(
"province", Integer.toString(reg.province()),
"city", Integer.toString(reg.city()),
"maker", reg.maker(),
"deviceType", reg.deviceType(),
"plateColor", Integer.toString(reg.plateColor()))));
VehicleIdentitySource source = alreadyResolved ? currentIdentity.source() : VehicleIdentitySource.REGISTERED;
return new IdentityResolution(new VehicleIdentity(vin, true, source), current.errorMessage());
}
private void dispatchFrameError(ChannelHandlerContext ctx, Jt808MalformedFrame malformed) { private void dispatchFrameError(ChannelHandlerContext ctx, Jt808MalformedFrame malformed) {
String peer = malformed.peer() == null || malformed.peer().isBlank() ? addr(ctx) : malformed.peer(); String peer = malformed.peer() == null || malformed.peer().isBlank() ? addr(ctx) : malformed.peer();
dispatchMalformed(peer, malformed.rawBytes(), malformed.reason(), true); dispatchMalformed(peer, malformed.rawBytes(), malformed.reason(), true);
@@ -283,7 +272,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
new Jt808Header(0, frame == null ? 0 : frame.length, 0, false, new Jt808Header(0, frame == null ? 0 : frame.length, 0, false,
Jt808Header.ProtocolVersion.V2013, "unknown", 0, 0, 0), Jt808Header.ProtocolVersion.V2013, "unknown", 0, 0, 0),
new Jt808Body.Malformed(frame == null ? new byte[0] : frame, peer, message)); new Jt808Body.Malformed(frame == null ? new byte[0] : frame, peer, message));
Map<String, String> meta = new HashMap<>(4); Map<String, String> meta = new HashMap<>(8);
meta.put("vin", "unknown"); meta.put("vin", "unknown");
meta.put("peer", peer); meta.put("peer", peer);
meta.put("identityResolved", "false"); meta.put("identityResolved", "false");
@@ -295,6 +284,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
meta.put("parseError", "true"); meta.put("parseError", "true");
meta.put("parseErrorMessage", message); meta.put("parseErrorMessage", message);
} }
putBodyMetadata(meta, malformed.body());
dispatcher.dispatch(new RawFrame( dispatcher.dispatch(new RawFrame(
ProtocolId.JT808, ProtocolId.JT808,
0, 0,
@@ -312,15 +302,19 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
msg.header().version(), msg.header().phone(), msg.header().serialNo(), 0, 0), msg.header().version(), msg.header().phone(), msg.header().serialNo(), 0, 0),
new Jt808Body.Malformed(frame == null ? new byte[0] : frame, addr(ctx), new Jt808Body.Malformed(frame == null ? new byte[0] : frame, addr(ctx),
PROCESSING_ERROR_PREFIX + message)); PROCESSING_ERROR_PREFIX + message));
Map<String, String> meta = new HashMap<>(8); Map<String, String> meta = new HashMap<>(16);
meta.put("vin", "unknown"); meta.put("vin", "unknown");
meta.put("phone", msg.header().phone()); meta.put("phone", msg.header().phone());
meta.put("peer", addr(ctx)); meta.put("peer", addr(ctx));
meta.put("seq", Integer.toString(msg.header().serialNo())); meta.put("seq", Integer.toString(msg.header().serialNo()));
meta.put("messageId", "0x" + Jt808MessageId.hex(msg.header().messageId()));
meta.put("messageName", Jt808MessageId.nameOf(msg.header().messageId()));
meta.put("reportType", Jt808MessageId.reportTypeOf(msg.header().messageId()));
meta.put("identityResolved", "false"); meta.put("identityResolved", "false");
meta.put("identitySource", VehicleIdentitySource.UNKNOWN.name()); meta.put("identitySource", VehicleIdentitySource.UNKNOWN.name());
meta.put("processingError", "true"); meta.put("processingError", "true");
meta.put("processingErrorMessage", message); meta.put("processingErrorMessage", message);
putBodyMetadata(meta, failed.body());
dispatcher.dispatch(new RawFrame( dispatcher.dispatch(new RawFrame(
ProtocolId.JT808, ProtocolId.JT808,
0, 0,
@@ -343,25 +337,6 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
Instant.now(), Instant.now(), Instant.now(), Instant.now(),
Map.of("protocolVersion", msg.header().version().name()))); Map.of("protocolVersion", msg.header().version().name())));
session = session.withVin(identity.vin()); session = session.withVin(identity.vin());
if (msg.body() instanceof Jt808Body.Register reg) {
identityRegistry.register(new VehicleRegistrationBinding(
ProtocolId.JT808,
session.vin(),
phone,
reg.deviceId(),
reg.plate(),
reg.province(),
reg.city(),
reg.maker(),
reg.deviceType(),
reg.plateColor()));
if (identity.resolved() && session.vin() != null
&& !session.vin().isBlank()
&& !"unknown".equalsIgnoreCase(session.vin())) {
identityRegistry.bind(new VehicleIdentityBinding(
ProtocolId.JT808, session.vin(), phone, reg.deviceId(), reg.plate()));
}
}
sessions.put(session); sessions.put(session);
return session; return session;
} }
@@ -380,37 +355,175 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
return body instanceof Jt808Body.Register reg ? reg.plate() : ""; return body instanceof Jt808Body.Register reg ? reg.plate() : "";
} }
private static void addRegistrationMetadata(Map<String, String> meta, Jt808Body body) { private static void putBodyMetadata(Map<String, String> meta, Jt808Body body) {
if (!(body instanceof Jt808Body.Register reg)) { switch (body) {
return; case Jt808Body.Register reg -> {
put(meta, "body.province", reg.province());
put(meta, "body.city", reg.city());
put(meta, "body.maker", reg.maker());
put(meta, "body.deviceType", reg.deviceType());
put(meta, "body.deviceId", reg.deviceId());
put(meta, "body.plateColor", reg.plateColor());
put(meta, "body.plate", reg.plate());
}
case Jt808Body.Auth auth -> {
put(meta, "body.token", auth.token());
put(meta, "body.imei", auth.imei());
put(meta, "body.softwareVersion", auth.softwareVersion());
}
case Jt808Body.Location location -> putLocationMetadata(meta, "body.", location);
case Jt808Body.LocationBatch batch -> {
put(meta, "body.batchType", batch.batchType());
put(meta, "body.locationCount", batch.locations() == null ? 0 : batch.locations().size());
}
case Jt808Body.ParamsReport report -> {
put(meta, "body.responseSerialNo", report.responseSerialNo());
put(meta, "body.parameterCount", report.parameters() == null ? 0 : report.parameters().size());
if (report.parameters() != null) {
report.parameters().forEach((id, bytes) ->
put(meta, "body.param.0x" + String.format("%08X", id), toHex(bytes)));
}
}
case Jt808Body.TerminalAttrs attrs -> {
put(meta, "body.terminalType", attrs.terminalType());
put(meta, "body.maker", attrs.maker());
put(meta, "body.terminalModel", attrs.terminalModel());
put(meta, "body.terminalId", attrs.terminalId());
put(meta, "body.iccid", attrs.iccid());
put(meta, "body.hardwareVersion", attrs.hardwareVersion());
put(meta, "body.firmwareVersion", attrs.firmwareVersion());
put(meta, "body.gnssProperty", attrs.gnssProperty());
put(meta, "body.commProperty", attrs.commProperty());
}
case Jt808Body.MediaEvent event -> {
put(meta, "body.mediaId", event.mediaId());
put(meta, "body.mediaType", event.mediaType());
put(meta, "body.mediaFormat", event.mediaFormat());
put(meta, "body.eventCode", event.eventCode());
put(meta, "body.channelId", event.channelId());
}
case Jt808Body.MediaUpload upload -> {
put(meta, "body.mediaId", upload.mediaId());
put(meta, "body.mediaType", upload.mediaType());
put(meta, "body.mediaFormat", upload.mediaFormat());
put(meta, "body.eventCode", upload.eventCode());
put(meta, "body.channelId", upload.channelId());
put(meta, "body.dataSizeBytes", upload.data() == null ? 0 : upload.data().length);
if (upload.location() != null) {
putLocationMetadata(meta, "body.location.", upload.location());
}
}
case Jt808Body.MediaAttributes attrs -> {
put(meta, "body.inputAudioEncoding", attrs.inputAudioEncoding());
put(meta, "body.inputAudioChannels", attrs.inputAudioChannels());
put(meta, "body.inputAudioSampleRate", attrs.inputAudioSampleRate());
put(meta, "body.inputAudioBitDepth", attrs.inputAudioBitDepth());
put(meta, "body.audioFrameLength", attrs.audioFrameLength());
put(meta, "body.audioOutputSupported", attrs.audioOutputSupported());
put(meta, "body.videoEncoding", attrs.videoEncoding());
put(meta, "body.maxAudioChannels", attrs.maxAudioChannels());
put(meta, "body.maxVideoChannels", attrs.maxVideoChannels());
put(meta, "body.rawSizeBytes", attrs.raw() == null ? 0 : attrs.raw().length);
}
case Jt808Body.FileUploadComplete complete -> {
put(meta, "body.responseSerialNo", complete.responseSerialNo());
put(meta, "body.result", complete.result());
put(meta, "body.rawSizeBytes", complete.raw() == null ? 0 : complete.raw().length);
}
case Jt808Body.FileList files -> {
put(meta, "body.responseSerialNo", files.responseSerialNo());
put(meta, "body.fileCount", files.fileCount());
put(meta, "body.rawSizeBytes", files.raw() == null ? 0 : files.raw().length);
}
case Jt808Body.PassengerVolume volume -> {
put(meta, "body.channelId", volume.channelId());
put(meta, "body.startTime", volume.startTime());
put(meta, "body.endTime", volume.endTime());
put(meta, "body.passengerGetOn", volume.passengerGetOn());
put(meta, "body.passengerGetOff", volume.passengerGetOff());
put(meta, "body.rawSizeBytes", volume.raw() == null ? 0 : volume.raw().length);
}
case Jt808Body.Passthrough passthrough -> {
put(meta, "body.passthroughType", "0x" + String.format("%02X", passthrough.passthroughType()));
put(meta, "body.dataSizeBytes", passthrough.data() == null ? 0 : passthrough.data().length);
put(meta, "body.dataHex", toHex(passthrough.data()));
}
case Jt808Body.Raw raw -> {
put(meta, "body.rawMessageId", "0x" + Jt808MessageId.hex(raw.messageId()));
put(meta, "body.rawSizeBytes", raw.bytes() == null ? 0 : raw.bytes().length);
put(meta, "body.rawHex", toHex(raw.bytes()));
}
case Jt808Body.Malformed malformed -> {
put(meta, "body.peer", malformed.peer());
put(meta, "body.errorMessage", malformed.errorMessage());
put(meta, "body.rawSizeBytes", malformed.bytes() == null ? 0 : malformed.bytes().length);
}
case Jt808Body.Heartbeat ignored -> {
}
case Jt808Body.Unregister ignored -> {
}
} }
meta.put("jt808.register.province", Integer.toString(reg.province()));
meta.put("jt808.register.city", Integer.toString(reg.city()));
meta.put("jt808.register.maker", nullToEmpty(reg.maker()));
meta.put("jt808.register.deviceType", nullToEmpty(reg.deviceType()));
meta.put("jt808.register.deviceId", nullToEmpty(reg.deviceId()));
meta.put("jt808.register.plateColor", Integer.toString(reg.plateColor()));
meta.put("jt808.register.plate", nullToEmpty(reg.plate()));
} }
private static String nullToEmpty(String value) { private static void putLocationMetadata(Map<String, String> meta, String prefix, Jt808Body.Location location) {
return value == null ? "" : value; put(meta, prefix + "alarmFlag", location.alarmFlag());
put(meta, prefix + "statusFlag", location.statusFlag());
meta.putAll(Jt808LocationFlagInfo.decodeStatus(location.statusFlag(), prefix + "status."));
meta.putAll(Jt808LocationFlagInfo.decodeAlarm(location.alarmFlag(), prefix + "alarm."));
put(meta, prefix + "longitude", location.longitude());
put(meta, prefix + "latitude", location.latitude());
put(meta, prefix + "altitudeM", location.altitudeM());
put(meta, prefix + "speedKmh", location.speedKmh());
put(meta, prefix + "directionDeg", location.directionDeg());
put(meta, prefix + "deviceTime", location.deviceTime());
if (location.extensionItems() != null) {
location.extensionItems().forEach((id, bytes) ->
put(meta, prefix + "extra.0x" + String.format("%02X", id), toHex(bytes)));
meta.putAll(Jt808LocationAdditionalInfo.decode(location.extensionItems(), prefix + "extra."));
}
} }
private byte[] registerAckFrame(String phone, int ackSerial, String token) { private static void put(Map<String, String> meta, String key, Object value) {
var cmd = Jt808Commands.registerAck(ackSerial, 0, token); if (value != null) {
return Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body()); meta.put(key, String.valueOf(value));
}
} }
private byte[] terminalAckFrame(String phone, int ackSerial, int ackMsgId, int result) { private static String toHex(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return "";
}
StringBuilder out = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
out.append(String.format("%02X", b & 0xFF));
}
return out.toString();
}
private void ackTerminal(ChannelHandlerContext ctx, String phone, int ackSerial, int ackMsgId, int result) {
var cmd = Jt808Commands.platformAck(ackSerial, ackMsgId, result); var cmd = Jt808Commands.platformAck(ackSerial, ackMsgId, result);
return Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body()); byte[] frame = Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body());
ctx.writeAndFlush(Unpooled.wrappedBuffer(frame));
} }
private static String generateToken(String phone) { private static String generateToken(String phone) {
return UUID.nameUUIDFromBytes(("jt808:" + phone).getBytes()).toString().replace("-", "").substring(0, 16); return UUID.nameUUIDFromBytes(("jt808:" + phone).getBytes()).toString().replace("-", "").substring(0, 16);
} }
private static String generatedInternalVin(String phone, Jt808Body.Register reg) {
String material = String.join("|", phone, reg.deviceId(), reg.plate(), reg.maker(), reg.deviceType());
try {
byte[] digest = MessageDigest.getInstance("SHA-256").digest(material.getBytes(StandardCharsets.UTF_8));
StringBuilder suffix = new StringBuilder(12);
for (int i = 0; i < digest.length && suffix.length() < 12; i++) {
suffix.append(String.format("%02X", digest[i] & 0xFF));
}
return "JT808" + suffix.substring(0, 12);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 not available", e);
}
}
@Override @Override
public void channelInactive(ChannelHandlerContext ctx) { public void channelInactive(ChannelHandlerContext ctx) {
channelRegistry.unbind(ctx.channel()) channelRegistry.unbind(ctx.channel())

View File

@@ -10,6 +10,8 @@ import com.lingniu.ingest.identity.VehicleIdentityResolver;
import com.lingniu.ingest.identity.VehicleIdentitySource; import com.lingniu.ingest.identity.VehicleIdentitySource;
import com.lingniu.ingest.protocol.jt808.model.Jt808Body; import com.lingniu.ingest.protocol.jt808.model.Jt808Body;
import com.lingniu.ingest.protocol.jt808.model.Jt808Header; import com.lingniu.ingest.protocol.jt808.model.Jt808Header;
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationAdditionalInfo;
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationFlagInfo;
import com.lingniu.ingest.protocol.jt808.model.Jt808Message; import com.lingniu.ingest.protocol.jt808.model.Jt808Message;
import java.time.Instant; import java.time.Instant;
@@ -276,12 +278,14 @@ public final class Jt808EventMapper implements EventMapper<Jt808Message> {
} }
private static Map<String, String> locationMeta(Map<String, String> meta, Jt808Body.Location loc) { private static Map<String, String> locationMeta(Map<String, String> meta, Jt808Body.Location loc) {
if (loc.extensionItems().isEmpty()) {
return meta;
}
java.util.HashMap<String, String> copy = new java.util.HashMap<>(meta); java.util.HashMap<String, String> copy = new java.util.HashMap<>(meta);
copy.putAll(Jt808LocationFlagInfo.decodeStatus(loc.statusFlag(), "jt808.status."));
copy.putAll(Jt808LocationFlagInfo.decodeAlarm(loc.alarmFlag(), "jt808.alarm."));
if (!loc.extensionItems().isEmpty()) {
loc.extensionItems().forEach((id, bytes) -> loc.extensionItems().forEach((id, bytes) ->
copy.put("jt808.extra.0x" + String.format("%02X", id), hex(bytes))); copy.put("jt808.extra.0x" + String.format("%02X", id), hex(bytes)));
copy.putAll(Jt808LocationAdditionalInfo.decode(loc.extensionItems()));
}
return Map.copyOf(copy); return Map.copyOf(copy);
} }
@@ -304,15 +308,7 @@ public final class Jt808EventMapper implements EventMapper<Jt808Message> {
} }
private static Double totalMileageKm(Jt808Body.Location loc) { private static Double totalMileageKm(Jt808Body.Location loc) {
byte[] bytes = loc.extensionItems().get(0x01); return Jt808LocationAdditionalInfo.mileageKm(loc.extensionItems());
if (bytes == null || bytes.length != 4) {
return null;
}
long raw = ((long) bytes[0] & 0xFF) << 24
| ((long) bytes[1] & 0xFF) << 16
| ((long) bytes[2] & 0xFF) << 8
| ((long) bytes[3] & 0xFF);
return raw * 0.1;
} }
private static String hex(byte[] bytes) { private static String hex(byte[] bytes) {

View File

@@ -0,0 +1,247 @@
package com.lingniu.ingest.protocol.jt808.model;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public final class Jt808LocationAdditionalInfo {
public static final String DEFAULT_PREFIX = "jt808.extra.";
private static final String[] VEHICLE_SIGNAL_BITS = {
"low_beam",
"high_beam",
"right_turn",
"left_turn",
"brake",
"reverse",
"fog_lamp",
"position_lamp",
"horn",
"air_conditioner",
"neutral",
"retarder",
"abs",
"heater",
"clutch"
};
private static final String[] IO_STATUS_BITS = {
"deep_sleep",
"sleep"
};
private Jt808LocationAdditionalInfo() {
}
public static Map<String, String> decode(Map<Integer, byte[]> extensionItems) {
return decode(extensionItems, DEFAULT_PREFIX);
}
public static Map<String, String> decode(Map<Integer, byte[]> extensionItems, String prefix) {
if (extensionItems == null || extensionItems.isEmpty()) {
return Map.of();
}
String p = prefix == null ? "" : prefix;
Map<String, String> out = new LinkedHashMap<>();
extensionItems.forEach((id, bytes) -> decodeItem(out, p, id == null ? -1 : id, bytes));
return Map.copyOf(out);
}
public static Double mileageKm(Map<Integer, byte[]> extensionItems) {
if (extensionItems == null) {
return null;
}
byte[] bytes = extensionItems.get(0x01);
if (bytes == null || bytes.length != 4) {
return null;
}
return u32(bytes, 0) * 0.1;
}
private static void decodeItem(Map<String, String> out, String prefix, int id, byte[] bytes) {
if (bytes == null) {
return;
}
switch (id) {
case 0x01 -> put(out, prefix + "mileage_km", scaled(bytes, 4, 1));
case 0x02 -> put(out, prefix + "fuel_l", scaled(bytes, 2, 1));
case 0x03 -> put(out, prefix + "recorder_speed_kmh", scaled(bytes, 2, 1));
case 0x04 -> put(out, prefix + "manual_alarm_event_id", unsigned(bytes, 2));
case 0x05 -> decodeTirePressure(out, prefix, bytes);
case 0x06 -> put(out, prefix + "compartment_temp_c", signedHighBitWord(bytes));
case 0x11 -> decodeOverspeed(out, prefix, bytes);
case 0x12 -> decodeAreaRouteAlarm(out, prefix, bytes);
case 0x13 -> decodeRouteTime(out, prefix, bytes);
case 0x25 -> decodeVehicleSignal(out, prefix, bytes);
case 0x2A -> decodeIoStatus(out, prefix, bytes);
case 0x2B -> decodeAnalog(out, prefix, bytes);
case 0x30 -> put(out, prefix + "network_signal_strength", unsigned(bytes, 1));
case 0x31 -> put(out, prefix + "gnss_satellite_count", unsigned(bytes, 1));
default -> {
}
}
}
private static void decodeTirePressure(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 30) {
return;
}
List<String> values = new ArrayList<>();
List<Integer> validIndexes = new ArrayList<>();
List<Integer> invalidIndexes = new ArrayList<>();
for (int i = 0; i < bytes.length; i++) {
int value = u8(bytes, i);
if (value == 0xFF) {
invalidIndexes.add(i);
} else {
validIndexes.add(i);
values.add(Integer.toString(value));
}
}
put(out, prefix + "tire_pressure_pa_values", String.join(",", values));
put(out, prefix + "tire_pressure_valid_indexes", ranges(validIndexes));
put(out, prefix + "tire_pressure_invalid_indexes", ranges(invalidIndexes));
}
private static void decodeOverspeed(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 1 && bytes.length != 5) {
return;
}
put(out, prefix + "overspeed.position_type", Integer.toString(u8(bytes, 0)));
if (bytes.length == 5) {
put(out, prefix + "overspeed.area_route_id", Long.toString(u32(bytes, 1)));
}
}
private static void decodeAreaRouteAlarm(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 6) {
return;
}
put(out, prefix + "area_route_alarm.position_type", Integer.toString(u8(bytes, 0)));
put(out, prefix + "area_route_alarm.area_route_id", Long.toString(u32(bytes, 1)));
put(out, prefix + "area_route_alarm.direction", Integer.toString(u8(bytes, 5)));
}
private static void decodeRouteTime(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 7) {
return;
}
put(out, prefix + "route_time.segment_id", Long.toString(u32(bytes, 0)));
put(out, prefix + "route_time.duration_seconds", Integer.toString(u16(bytes, 4)));
put(out, prefix + "route_time.result", Integer.toString(u8(bytes, 6)));
}
private static void decodeVehicleSignal(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 4) {
return;
}
long raw = u32(bytes, 0);
put(out, prefix + "vehicle_signal_raw", Long.toString(raw));
for (int i = 0; i < VEHICLE_SIGNAL_BITS.length; i++) {
put(out, prefix + "vehicle_signal." + VEHICLE_SIGNAL_BITS[i], Boolean.toString(bit(raw, i)));
}
}
private static void decodeIoStatus(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 2) {
return;
}
int raw = u16(bytes, 0);
put(out, prefix + "io_status_raw", Integer.toString(raw));
for (int i = 0; i < IO_STATUS_BITS.length; i++) {
put(out, prefix + "io_status." + IO_STATUS_BITS[i], Boolean.toString(bit(raw, i)));
}
}
private static void decodeAnalog(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 4) {
return;
}
long raw = u32(bytes, 0);
put(out, prefix + "analog_raw", Long.toString(raw));
put(out, prefix + "analog_ad0", Integer.toString((int) (raw & 0xFFFF)));
put(out, prefix + "analog_ad1", Integer.toString((int) ((raw >>> 16) & 0xFFFF)));
}
private static String scaled(byte[] bytes, int expectedLength, int scale) {
if (bytes.length != expectedLength) {
return null;
}
long raw = expectedLength == 4 ? u32(bytes, 0) : u16(bytes, 0);
return BigDecimal.valueOf(raw, scale).stripTrailingZeros().toPlainString();
}
private static String unsigned(byte[] bytes, int expectedLength) {
if (bytes.length != expectedLength) {
return null;
}
return switch (expectedLength) {
case 1 -> Integer.toString(u8(bytes, 0));
case 2 -> Integer.toString(u16(bytes, 0));
case 4 -> Long.toString(u32(bytes, 0));
default -> null;
};
}
private static String signedHighBitWord(byte[] bytes) {
if (bytes.length != 2) {
return null;
}
int raw = u16(bytes, 0);
int value = (raw & 0x8000) == 0 ? raw : -(raw & 0x7FFF);
return Integer.toString(value);
}
private static int u8(byte[] bytes, int offset) {
return bytes[offset] & 0xFF;
}
private static int u16(byte[] bytes, int offset) {
return (u8(bytes, offset) << 8) | u8(bytes, offset + 1);
}
private static long u32(byte[] bytes, int offset) {
return ((long) u8(bytes, offset) << 24)
| ((long) u8(bytes, offset + 1) << 16)
| ((long) u8(bytes, offset + 2) << 8)
| u8(bytes, offset + 3);
}
private static boolean bit(long raw, int bit) {
return ((raw >>> bit) & 1L) == 1L;
}
private static void put(Map<String, String> out, String key, String value) {
if (value != null) {
out.put(key, value);
}
}
private static String ranges(List<Integer> indexes) {
if (indexes.isEmpty()) {
return "";
}
List<String> ranges = new ArrayList<>();
int start = indexes.getFirst();
int previous = start;
for (int i = 1; i < indexes.size(); i++) {
int current = indexes.get(i);
if (current == previous + 1) {
previous = current;
continue;
}
ranges.add(formatRange(start, previous));
start = current;
previous = current;
}
ranges.add(formatRange(start, previous));
return String.join(",", ranges);
}
private static String formatRange(int start, int end) {
return start == end ? Integer.toString(start) : start + "-" + end;
}
}

View File

@@ -0,0 +1,99 @@
package com.lingniu.ingest.protocol.jt808.model;
import java.util.LinkedHashMap;
import java.util.Map;
public final class Jt808LocationFlagInfo {
private static final String[] ALARM_BITS = {
"emergency",
"overspeed",
"fatigue_driving",
"warning",
"gnss_module_fault",
"gnss_antenna_disconnected",
"gnss_antenna_short_circuit",
"terminal_main_power_undervoltage",
"terminal_main_power_lost",
"display_fault",
"tts_fault",
"camera_fault",
"transport_ic_card_fault",
"overspeed_warning",
"fatigue_driving_warning",
"prohibited_driving",
null,
null,
"daily_driving_timeout",
"overtime_parking",
"area_in_out",
"route_in_out",
"route_time_abnormal",
"route_deviation",
"vss_fault",
"fuel_abnormal",
"stolen",
"illegal_ignition",
"illegal_displacement",
"collision_rollover",
"rollover_warning"
};
private Jt808LocationFlagInfo() {
}
public static Map<String, String> decodeAlarm(long alarmFlag, String prefix) {
Map<String, String> out = new LinkedHashMap<>();
String p = prefix == null ? "" : prefix;
for (int i = 0; i < ALARM_BITS.length; i++) {
String name = ALARM_BITS[i];
if (name != null) {
out.put(p + name, Boolean.toString(bit(alarmFlag, i)));
}
}
return Map.copyOf(out);
}
public static Map<String, String> decodeStatus(long statusFlag, String prefix) {
Map<String, String> out = new LinkedHashMap<>();
String p = prefix == null ? "" : prefix;
out.put(p + "acc_on", Boolean.toString(bit(statusFlag, 0)));
out.put(p + "positioned", Boolean.toString(bit(statusFlag, 1)));
out.put(p + "south_latitude", Boolean.toString(bit(statusFlag, 2)));
out.put(p + "west_longitude", Boolean.toString(bit(statusFlag, 3)));
out.put(p + "out_of_service", Boolean.toString(bit(statusFlag, 4)));
out.put(p + "encrypted", Boolean.toString(bit(statusFlag, 5)));
out.put(p + "forward_collision_warning", Boolean.toString(bit(statusFlag, 6)));
out.put(p + "lane_departure_warning", Boolean.toString(bit(statusFlag, 7)));
out.put(p + "load_status", loadStatus(statusFlag));
out.put(p + "fuel_circuit_cut", Boolean.toString(bit(statusFlag, 10)));
out.put(p + "electric_circuit_cut", Boolean.toString(bit(statusFlag, 11)));
out.put(p + "door_locked", Boolean.toString(bit(statusFlag, 12)));
out.put(p + "door1_open", Boolean.toString(bit(statusFlag, 13)));
out.put(p + "door2_open", Boolean.toString(bit(statusFlag, 14)));
out.put(p + "door3_open", Boolean.toString(bit(statusFlag, 15)));
out.put(p + "door4_open", Boolean.toString(bit(statusFlag, 16)));
out.put(p + "door5_open", Boolean.toString(bit(statusFlag, 17)));
out.put(p + "gps_used", Boolean.toString(bit(statusFlag, 18)));
out.put(p + "beidou_used", Boolean.toString(bit(statusFlag, 19)));
out.put(p + "glonass_used", Boolean.toString(bit(statusFlag, 20)));
out.put(p + "galileo_used", Boolean.toString(bit(statusFlag, 21)));
out.put(p + "moving", Boolean.toString(bit(statusFlag, 22)));
return Map.copyOf(out);
}
private static String loadStatus(long statusFlag) {
int code = (int) ((statusFlag >>> 8) & 0x03);
return switch (code) {
case 0 -> "EMPTY";
case 1 -> "HALF_LOADED";
case 2 -> "RESERVED";
case 3 -> "FULL";
default -> "UNKNOWN";
};
}
private static boolean bit(long raw, int bit) {
return ((raw >>> bit) & 1L) == 1L;
}
}

View File

@@ -36,4 +36,56 @@ public final class Jt808MessageId {
public static final int PLATFORM_TRACK_CONTROL = 0x8202; public static final int PLATFORM_TRACK_CONTROL = 0x8202;
public static final int PLATFORM_ALARM_ACK = 0x8203; public static final int PLATFORM_ALARM_ACK = 0x8203;
public static final int PLATFORM_DELETE_CIRCLE_AREA = 0x8601; public static final int PLATFORM_DELETE_CIRCLE_AREA = 0x8601;
public static String nameOf(int messageId) {
return switch (messageId) {
case TERMINAL_GENERAL_RESPONSE -> "TERMINAL_GENERAL_RESPONSE";
case TERMINAL_HEARTBEAT -> "TERMINAL_HEARTBEAT";
case TERMINAL_UNREGISTER -> "TERMINAL_UNREGISTER";
case TERMINAL_REGISTER -> "TERMINAL_REGISTER";
case TERMINAL_AUTH -> "TERMINAL_AUTH";
case TERMINAL_PARAMS_REPORT -> "TERMINAL_PARAMS_REPORT";
case TERMINAL_ATTRS_REPORT -> "TERMINAL_ATTRS_REPORT";
case TERMINAL_LOCATION -> "TERMINAL_LOCATION";
case TERMINAL_LOCATION_BATCH -> "TERMINAL_LOCATION_BATCH";
case TERMINAL_EVENT_REPORT -> "TERMINAL_EVENT_REPORT";
case TERMINAL_CAN_DATA_REPORT -> "TERMINAL_CAN_DATA_REPORT";
case TERMINAL_MEDIA_EVENT -> "TERMINAL_MEDIA_EVENT";
case TERMINAL_MEDIA_UPLOAD -> "TERMINAL_MEDIA_UPLOAD";
case TERMINAL_PASSTHROUGH -> "TERMINAL_PASSTHROUGH";
case PLATFORM_GENERAL_RESPONSE -> "PLATFORM_GENERAL_RESPONSE";
case PLATFORM_REGISTER_ACK -> "PLATFORM_REGISTER_ACK";
case PLATFORM_SET_PARAMS -> "PLATFORM_SET_PARAMS";
case PLATFORM_QUERY_PARAMS -> "PLATFORM_QUERY_PARAMS";
case PLATFORM_TERMINAL_CONTROL -> "PLATFORM_TERMINAL_CONTROL";
case PLATFORM_QUERY_ATTRS -> "PLATFORM_QUERY_ATTRS";
case PLATFORM_QUERY_LOCATION -> "PLATFORM_QUERY_LOCATION";
case PLATFORM_TRACK_CONTROL -> "PLATFORM_TRACK_CONTROL";
case PLATFORM_ALARM_ACK -> "PLATFORM_ALARM_ACK";
case PLATFORM_DELETE_CIRCLE_AREA -> "PLATFORM_DELETE_CIRCLE_AREA";
default -> "UNKNOWN_0x" + hex(messageId);
};
}
public static String reportTypeOf(int messageId) {
return switch (messageId) {
case TERMINAL_LOCATION -> "LOCATION";
case TERMINAL_LOCATION_BATCH -> "LOCATION_BATCH";
case TERMINAL_HEARTBEAT -> "HEARTBEAT";
case TERMINAL_AUTH -> "AUTH";
case TERMINAL_REGISTER -> "REGISTER";
case TERMINAL_UNREGISTER -> "UNREGISTER";
case TERMINAL_GENERAL_RESPONSE -> "TERMINAL_RESPONSE";
case TERMINAL_PARAMS_REPORT -> "PARAMS_REPORT";
case TERMINAL_ATTRS_REPORT -> "ATTRS_REPORT";
case TERMINAL_MEDIA_EVENT -> "MEDIA_EVENT";
case TERMINAL_MEDIA_UPLOAD -> "MEDIA_UPLOAD";
case TERMINAL_PASSTHROUGH -> "PASSTHROUGH";
default -> "RAW";
};
}
public static String hex(int messageId) {
return String.format("%04X", messageId & 0xFFFF);
}
} }

View File

@@ -41,6 +41,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.within;
class Jt808ChannelHandlerTest { class Jt808ChannelHandlerTest {
@@ -86,7 +87,7 @@ class Jt808ChannelHandlerTest {
} }
@Test @Test
void unresolvedRegisterDoesNotBindUnknownVinToExternalIdentifiers() { void unresolvedRegisterBindsGeneratedInternalVinToExternalIdentifiers() {
InMemorySessionStore sessions = new InMemorySessionStore(); InMemorySessionStore sessions = new InMemorySessionStore();
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService(); InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(new ImmediateKafkaSink())); DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(new ImmediateKafkaSink()));
@@ -115,9 +116,9 @@ class Jt808ChannelHandlerTest {
assertThat(identity.resolve(new VehicleIdentityLookup( assertThat(identity.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "123456789012", "", ""))) ProtocolId.JT808, "", "123456789012", "", "")))
.satisfies(resolved -> { .satisfies(resolved -> {
assertThat(resolved.vin()).isEqualTo("123456789012"); assertThat(resolved.vin()).startsWith("JT808");
assertThat(resolved.resolved()).isFalse(); assertThat(resolved.resolved()).isTrue();
assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.FALLBACK_PHONE); assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
}); });
batchExecutor.close(); batchExecutor.close();
@@ -269,14 +270,143 @@ class Jt808ChannelHandlerTest {
assertThat(archive.metadata()) assertThat(archive.metadata())
.containsEntry("vin", "LNVIN000000000808") .containsEntry("vin", "LNVIN000000000808")
.containsEntry("phone", "123456789012") .containsEntry("phone", "123456789012")
.containsEntry("messageId", "0x0200")
.containsEntry("messageName", "TERMINAL_LOCATION")
.containsEntry("reportType", "LOCATION")
.containsEntry("body.alarmFlag", "0")
.containsEntry("body.statusFlag", "196608")
.containsEntry("body.status.acc_on", "false")
.containsEntry("body.status.load_status", "EMPTY")
.containsEntry("body.status.door4_open", "true")
.containsEntry("body.status.door5_open", "true")
.containsEntry("body.alarm.overspeed", "false")
.containsEntry("body.longitude", "116.397128")
.containsEntry("body.latitude", "39.916527")
.containsEntry("body.altitudeM", "50")
.containsEntry("body.directionDeg", "90")
.containsEntry("body.deviceTime", "2024-01-01T19:04:05Z")
.containsEntry("body.extra.0x01", "00003039")
.containsEntry("body.extra.mileage_km", "1234.5")
.containsEntry("body.extra.0x30", "58")
.containsEntry("body.extra.network_signal_strength", "88")
.containsEntry("identityResolved", "true") .containsEntry("identityResolved", "true")
.containsEntry("identitySource", "BOUND_PHONE"); .containsEntry("identitySource", "BOUND_PHONE");
assertThat(Double.parseDouble(archive.metadata().get("body.speedKmh"))).isCloseTo(52.3, within(0.001));
}); });
batchExecutor.close(); batchExecutor.close();
eventBus.close(); eventBus.close();
} }
@Test
void registerArchiveIncludesParsedBodyFieldsForRawQuery() throws Exception {
RecordingSink sink = new RecordingSink(1);
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish);
Dispatcher dispatcher = new Dispatcher(
new HandlerRegistry(),
new InterceptorChain(List.of()),
new HandlerInvoker(),
eventBus,
batchExecutor);
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
identity.bind(new VehicleIdentityBinding(
ProtocolId.JT808, "LNVIN000000000808", "123456789012", "DEV808", "B80808"));
Jt808ChannelHandler handler = new Jt808ChannelHandler(
new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser()))),
dispatcher,
new InMemorySessionStore(),
identity,
new Jt808ChannelRegistry(),
new Jt808PendingRequests());
EmbeddedChannel channel = new EmbeddedChannel(handler);
byte[] frame = buildFrame(
Jt808MessageId.TERMINAL_REGISTER,
"123456789012",
3,
buildRegisterBody("DEV808", "B80808"));
channel.writeInbound(frame);
assertThat(sink.await()).isTrue();
assertThat(sink.events).singleElement().satisfies(event -> {
assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class);
VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event;
assertThat(archive.metadata())
.containsEntry("messageId", "0x0100")
.containsEntry("messageName", "TERMINAL_REGISTER")
.containsEntry("reportType", "REGISTER")
.containsEntry("body.province", "44")
.containsEntry("body.city", "4401")
.containsEntry("body.maker", "MAKER")
.containsEntry("body.deviceType", "TYPE-A")
.containsEntry("body.deviceId", "DEV808")
.containsEntry("body.plateColor", "1")
.containsEntry("body.plate", "B80808");
});
batchExecutor.close();
eventBus.close();
}
@Test
void unboundRegisterCreatesVinBindingForRegisterAndFollowingRawFrames() throws Exception {
RecordingSink sink = new RecordingSink(2);
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish);
Dispatcher dispatcher = new Dispatcher(
new HandlerRegistry(),
new InterceptorChain(List.of()),
new HandlerInvoker(),
eventBus,
batchExecutor);
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
Jt808ChannelHandler handler = new Jt808ChannelHandler(
new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser(), new LocationBodyParser()))),
dispatcher,
new InMemorySessionStore(),
identity,
new Jt808ChannelRegistry(),
new Jt808PendingRequests());
EmbeddedChannel channel = new EmbeddedChannel(handler);
channel.writeInbound(buildFrame(
Jt808MessageId.TERMINAL_REGISTER,
"13079963289",
3,
buildRegisterBody("9963289", "HUA00113F")));
channel.writeInbound(buildFrame(
Jt808MessageId.TERMINAL_LOCATION,
"13079963289",
4,
buildLocationBody()));
assertThat(sink.await()).isTrue();
VehicleEvent.RawArchive register = (VehicleEvent.RawArchive) sink.events.get(0);
VehicleEvent.RawArchive location = (VehicleEvent.RawArchive) sink.events.get(1);
assertThat(register.metadata())
.containsEntry("reportType", "REGISTER")
.containsEntry("body.deviceId", "9963289")
.containsEntry("body.plate", "HUA00113F")
.containsEntry("identityResolved", "true")
.containsEntry("identitySource", "REGISTERED");
assertThat(register.vin())
.startsWith("JT808")
.hasSize(17);
assertThat(location.metadata())
.containsEntry("reportType", "LOCATION")
.containsEntry("vin", register.vin())
.containsEntry("identityResolved", "true")
.containsEntry("identitySource", "BOUND_PHONE");
assertThat(location.vin()).isEqualTo(register.vin());
assertThat(identity.resolve(new com.lingniu.ingest.identity.VehicleIdentityLookup(
ProtocolId.JT808, "", "13079963289", "", "")).vin())
.isEqualTo(register.vin());
batchExecutor.close();
eventBus.close();
}
@Test @Test
void unboundLocationArchiveUsesUnknownVinInsteadOfPhoneFallback() throws Exception { void unboundLocationArchiveUsesUnknownVinInsteadOfPhoneFallback() throws Exception {
RecordingSink sink = new RecordingSink(1); RecordingSink sink = new RecordingSink(1);
@@ -309,6 +439,9 @@ class Jt808ChannelHandlerTest {
assertThat(archive.metadata()) assertThat(archive.metadata())
.containsEntry("vin", "unknown") .containsEntry("vin", "unknown")
.containsEntry("phone", "123456789012") .containsEntry("phone", "123456789012")
.containsEntry("messageId", "0x0200")
.containsEntry("messageName", "TERMINAL_LOCATION")
.containsEntry("reportType", "LOCATION")
.containsEntry("identityResolved", "false") .containsEntry("identityResolved", "false")
.containsEntry("identitySource", "FALLBACK_PHONE"); .containsEntry("identitySource", "FALLBACK_PHONE");
}); });
@@ -351,13 +484,13 @@ class Jt808ChannelHandlerTest {
VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event; VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event;
assertThat(archive.command()).isEqualTo(Jt808MessageId.TERMINAL_REGISTER); assertThat(archive.command()).isEqualTo(Jt808MessageId.TERMINAL_REGISTER);
assertThat(archive.metadata()) assertThat(archive.metadata())
.containsEntry("jt808.register.province", "44") .containsEntry("body.province", "44")
.containsEntry("jt808.register.city", "4401") .containsEntry("body.city", "4401")
.containsEntry("jt808.register.maker", "MAKER") .containsEntry("body.maker", "MAKER")
.containsEntry("jt808.register.deviceType", "TYPE-A") .containsEntry("body.deviceType", "TYPE-A")
.containsEntry("jt808.register.deviceId", "DEV808") .containsEntry("body.deviceId", "DEV808")
.containsEntry("jt808.register.plateColor", "1") .containsEntry("body.plateColor", "1")
.containsEntry("jt808.register.plate", "B80808"); .containsEntry("body.plate", "B80808");
}); });
batchExecutor.close(); batchExecutor.close();
@@ -581,6 +714,12 @@ class Jt808ChannelHandlerTest {
writeU16(os, 90); writeU16(os, 90);
byte[] bcd = BcdCodec.encode("240102030405"); byte[] bcd = BcdCodec.encode("240102030405");
os.write(bcd, 0, 6); os.write(bcd, 0, 6);
os.write(0x01);
os.write(4);
writeU32(os, 12345);
os.write(0x30);
os.write(1);
os.write(0x58);
return os.toByteArray(); return os.toByteArray();
} }

View File

@@ -38,6 +38,10 @@ class Jt808EventMapperTest {
assertThat(events.get(0).metadata()) assertThat(events.get(0).metadata())
.containsEntry("vin", "unknown") .containsEntry("vin", "unknown")
.containsEntry("phone", "123456789012") .containsEntry("phone", "123456789012")
.containsEntry("jt808.status.acc_on", "false")
.containsEntry("jt808.status.positioned", "false")
.containsEntry("jt808.status.load_status", "EMPTY")
.containsEntry("jt808.alarm.overspeed", "false")
.containsEntry("identityResolved", "false") .containsEntry("identityResolved", "false")
.containsEntry("identitySource", "FALLBACK_PHONE"); .containsEntry("identitySource", "FALLBACK_PHONE");
} }
@@ -58,7 +62,66 @@ class Jt808EventMapperTest {
.extracting(VehicleEvent::metadata) .extracting(VehicleEvent::metadata)
.satisfies(meta -> assertThat(meta) .satisfies(meta -> assertThat(meta)
.containsEntry("jt808.extra.0x01", "00003039") .containsEntry("jt808.extra.0x01", "00003039")
.containsEntry("jt808.extra.0x30", "58")); .containsEntry("jt808.extra.0x30", "58")
.containsEntry("jt808.extra.mileage_km", "1234.5")
.containsEntry("jt808.extra.network_signal_strength", "88"));
}
@Test
void locationAdditionalItemsAreDecodedAsTable27Fields() {
var header = new Jt808Header(
Jt808MessageId.TERMINAL_LOCATION, 64, 0, false,
Jt808Header.ProtocolVersion.V2013, "123456789012", 1, 0, 0);
var body = new Jt808Body.Location(
0, 0, 116.397128, 39.916527, 50, 52.3, 90,
Instant.parse("2024-01-02T03:04:05Z"),
java.util.Map.ofEntries(
java.util.Map.entry(0x01, new byte[]{0x00, 0x00, 0x30, 0x39}),
java.util.Map.entry(0x02, new byte[]{0x00, 0x64}),
java.util.Map.entry(0x03, new byte[]{0x01, (byte) 0xF4}),
java.util.Map.entry(0x04, new byte[]{0x00, 0x02}),
java.util.Map.entry(0x05, tirePressureBytes()),
java.util.Map.entry(0x06, new byte[]{(byte) 0x80, 0x05}),
java.util.Map.entry(0x11, new byte[]{0x01, 0x00, 0x00, 0x00, 0x2A}),
java.util.Map.entry(0x12, new byte[]{0x04, 0x00, 0x00, 0x00, 0x2B, 0x01}),
java.util.Map.entry(0x13, new byte[]{0x00, 0x00, 0x00, 0x2C, 0x00, 0x78, 0x01}),
java.util.Map.entry(0x25, new byte[]{0x00, 0x00, 0x00, 0x05}),
java.util.Map.entry(0x2A, new byte[]{0x00, 0x03}),
java.util.Map.entry(0x2B, new byte[]{0x12, 0x34, 0x56, 0x78}),
java.util.Map.entry(0x30, new byte[]{0x1F}),
java.util.Map.entry(0x31, new byte[]{0x0C})));
VehicleEvent.Location event = (VehicleEvent.Location) mapper.toEvents(new Jt808Message(header, body)).getFirst();
assertThat(event.payload().totalMileageKm()).isEqualTo(1234.5);
assertThat(event.metadata())
.containsEntry("jt808.extra.mileage_km", "1234.5")
.containsEntry("jt808.extra.fuel_l", "10")
.containsEntry("jt808.extra.recorder_speed_kmh", "50")
.containsEntry("jt808.extra.manual_alarm_event_id", "2")
.containsEntry("jt808.extra.tire_pressure_pa_values", "32,33")
.containsEntry("jt808.extra.tire_pressure_invalid_indexes", "2-29")
.containsEntry("jt808.extra.compartment_temp_c", "-5")
.containsEntry("jt808.extra.overspeed.position_type", "1")
.containsEntry("jt808.extra.overspeed.area_route_id", "42")
.containsEntry("jt808.extra.area_route_alarm.position_type", "4")
.containsEntry("jt808.extra.area_route_alarm.area_route_id", "43")
.containsEntry("jt808.extra.area_route_alarm.direction", "1")
.containsEntry("jt808.extra.route_time.segment_id", "44")
.containsEntry("jt808.extra.route_time.duration_seconds", "120")
.containsEntry("jt808.extra.route_time.result", "1")
.containsEntry("jt808.extra.vehicle_signal_raw", "5")
.containsEntry("jt808.extra.vehicle_signal.low_beam", "true")
.containsEntry("jt808.extra.vehicle_signal.high_beam", "false")
.containsEntry("jt808.extra.vehicle_signal.right_turn", "true")
.containsEntry("jt808.extra.io_status_raw", "3")
.containsEntry("jt808.extra.io_status.deep_sleep", "true")
.containsEntry("jt808.extra.io_status.sleep", "true")
.containsEntry("jt808.extra.analog_raw", "305419896")
.containsEntry("jt808.extra.analog_ad0", "22136")
.containsEntry("jt808.extra.analog_ad1", "4660")
.containsEntry("jt808.extra.network_signal_strength", "31")
.containsEntry("jt808.extra.gnss_satellite_count", "12");
} }
@Test @Test
@@ -249,4 +312,12 @@ class Jt808EventMapperTest {
.containsEntry("parseError", "true"); .containsEntry("parseError", "true");
}); });
} }
private static byte[] tirePressureBytes() {
byte[] bytes = new byte[30];
java.util.Arrays.fill(bytes, (byte) 0xFF);
bytes[0] = 32;
bytes[1] = 33;
return bytes;
}
} }

View File

@@ -36,6 +36,10 @@
<groupId>org.apache.kafka</groupId> <groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId> <artifactId>kafka-clients</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>

View File

@@ -12,6 +12,10 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import java.io.IOException; import java.io.IOException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -27,6 +31,8 @@ import java.util.Map;
@RequestMapping("/api/event-history") @RequestMapping("/api/event-history")
public class EventHistoryController { public class EventHistoryController {
private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai");
private final EventFileStore store; private final EventFileStore store;
public EventHistoryController(EventFileStore store) { public EventHistoryController(EventFileStore store) {
@@ -40,8 +46,13 @@ public class EventHistoryController {
@RequestParam String dateTo, @RequestParam String dateTo,
@RequestParam(defaultValue = "ASC") EventFileQuery.Order order, @RequestParam(defaultValue = "ASC") EventFileQuery.Order order,
@RequestParam(defaultValue = "100") int limit, @RequestParam(defaultValue = "100") int limit,
@RequestParam(required = false) String vin) throws IOException { @RequestParam(required = false) String vin,
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin).stream() @RequestParam(required = false) String eventType,
@RequestParam(required = false) String cursorEventTime,
@RequestParam(required = false) String cursorIngestTime,
@RequestParam(required = false) String cursorEventId) throws IOException {
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin, eventType,
cursorEventTime, cursorIngestTime, cursorEventId).stream()
.map(RecordResponse::from) .map(RecordResponse::from)
.toList(); .toList();
} }
@@ -51,7 +62,11 @@ public class EventHistoryController {
String dateTo, String dateTo,
EventFileQuery.Order order, EventFileQuery.Order order,
int limit, int limit,
String vin) throws IOException { String vin,
String eventType,
String cursorEventTime,
String cursorIngestTime,
String cursorEventId) throws IOException {
QueryTimeRange range = QueryTimeRange.parse(dateFrom, dateTo); QueryTimeRange range = QueryTimeRange.parse(dateFrom, dateTo);
return store.query(new EventFileQuery( return store.query(new EventFileQuery(
protocol, protocol,
@@ -62,7 +77,22 @@ public class EventHistoryController {
order, order,
limit, limit,
vin, vin,
null)); eventType,
instantParam(cursorEventTime),
instantParam(cursorIngestTime),
cursorEventId));
}
private static Instant instantParam(String raw) {
if (raw == null || raw.isBlank()) {
return null;
}
String value = raw.trim();
try {
return OffsetDateTime.parse(value).toInstant();
} catch (RuntimeException ignored) {
return LocalDateTime.parse(value).atZone(DEFAULT_ZONE).toInstant();
}
} }
public record RecordResponse( public record RecordResponse(

View File

@@ -1,8 +1,8 @@
package com.lingniu.ingest.eventhistory; package com.lingniu.ingest.eventhistory;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
import com.lingniu.ingest.eventfilestore.EventFileRecord; import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore; import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
@@ -20,7 +20,7 @@ import java.util.List;
* <p>当前 32960 本机运行配置没有启用 Kafka consumer这个类保留给“其他服务把 envelope * <p>当前 32960 本机运行配置没有启用 Kafka consumer这个类保留给“其他服务把 envelope
* 写回历史库”的解耦部署方式。失败时返回结构化结果,由 Kafka worker 决定是否进 DLQ。 * 写回历史库”的解耦部署方式。失败时返回结构化结果,由 Kafka worker 决定是否进 DLQ。
*/ */
public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor { public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor {
private final EventFileStore store; private final EventFileStore store;
private final TelemetryEnvelopeRecordMapper mapper; private final TelemetryEnvelopeRecordMapper mapper;
@@ -33,7 +33,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper, public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter) { TdengineHistoryWriter tdengineWriter) {
this(mapper, tdengineWriter, true); this(mapper, tdengineWriter, false);
} }
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper, public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
@@ -45,7 +45,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
public EventHistoryEnvelopeIngestor(EventFileStore store, public EventHistoryEnvelopeIngestor(EventFileStore store,
TelemetryEnvelopeRecordMapper mapper, TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter) { TdengineHistoryWriter tdengineWriter) {
this(store, mapper, tdengineWriter, true, true); this(store, mapper, tdengineWriter, true, false);
} }
public EventHistoryEnvelopeIngestor(EventFileStore store, public EventHistoryEnvelopeIngestor(EventFileStore store,

View File

@@ -1,7 +1,5 @@
package com.lingniu.ingest.eventhistory; package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader; import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery; import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow; import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
@@ -32,8 +30,6 @@ import java.util.List;
@Tag(name = "jt-808-location-controller", description = "JT808 位置历史分页查询接口。") @Tag(name = "jt-808-location-controller", description = "JT808 位置历史分页查询接口。")
public final class Jt808LocationHistoryController { public final class Jt808LocationHistoryController {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final TdengineHistoryReader reader; private final TdengineHistoryReader reader;
public Jt808LocationHistoryController(TdengineHistoryReader reader) { public Jt808LocationHistoryController(TdengineHistoryReader reader) {
@@ -127,8 +123,7 @@ public final class Jt808LocationHistoryController {
long alarmFlag, long alarmFlag,
long statusFlag, long statusFlag,
Double totalMileageKm, Double totalMileageKm,
String rawUri, String rawUri) {
String metadataJson) {
private static LocationResponse from(TdengineLocationRow row) { private static LocationResponse from(TdengineLocationRow row) {
return new LocationResponse( return new LocationResponse(
@@ -147,23 +142,7 @@ public final class Jt808LocationHistoryController {
row.alarmFlag(), row.alarmFlag(),
row.statusFlag(), row.statusFlag(),
row.totalMileageKm(), row.totalMileageKm(),
rawUri(row), row.rawUri());
row.metadataJson());
}
private static String rawUri(TdengineLocationRow row) {
if (row.rawUri() != null && !row.rawUri().isBlank()) {
return row.rawUri();
}
String metadataJson = row.metadataJson();
if (metadataJson == null || metadataJson.isBlank()) {
return row.rawUri();
}
try {
return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
} catch (JsonProcessingException ignored) {
return row.rawUri();
}
} }
} }
} }

View File

@@ -1,7 +1,5 @@
package com.lingniu.ingest.eventhistory; package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader; import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery; import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow; import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
@@ -32,8 +30,6 @@ import java.util.Locale;
@Tag(name = "location-history-controller", description = "通用位置历史分页查询接口。") @Tag(name = "location-history-controller", description = "通用位置历史分页查询接口。")
public final class LocationHistoryController { public final class LocationHistoryController {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final TdengineHistoryReader reader; private final TdengineHistoryReader reader;
public LocationHistoryController(TdengineHistoryReader reader) { public LocationHistoryController(TdengineHistoryReader reader) {
@@ -161,7 +157,6 @@ public final class LocationHistoryController {
long statusFlag, long statusFlag,
Double totalMileageKm, Double totalMileageKm,
String rawUri, String rawUri,
String metadataJson,
String protocol) { String protocol) {
private static LocationResponse from(TdengineLocationRow row) { private static LocationResponse from(TdengineLocationRow row) {
@@ -181,24 +176,8 @@ public final class LocationHistoryController {
row.alarmFlag(), row.alarmFlag(),
row.statusFlag(), row.statusFlag(),
row.totalMileageKm(), row.totalMileageKm(),
rawUri(row), row.rawUri(),
row.metadataJson(),
row.protocol()); row.protocol());
} }
private static String rawUri(TdengineLocationRow row) {
if (row.rawUri() != null && !row.rawUri().isBlank()) {
return row.rawUri();
}
String metadataJson = row.metadataJson();
if (metadataJson == null || metadataJson.isBlank()) {
return row.rawUri();
}
try {
return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
} catch (JsonProcessingException ignored) {
return row.rawUri();
}
}
} }
} }

View File

@@ -184,6 +184,7 @@ public final class RawFrameHistoryController {
String parseError, String parseError,
String peer, String peer,
String metadataJson, String metadataJson,
String parsedJson,
Map<String, Object> metadata, Map<String, Object> metadata,
Map<String, Object> parsedFields, Map<String, Object> parsedFields,
String protocol, String protocol,
@@ -209,8 +210,9 @@ public final class RawFrameHistoryController {
row.parseError(), row.parseError(),
row.peer(), row.peer(),
row.metadataJson(), row.metadataJson(),
row.parsedJson(),
metadata, metadata,
parsedFields(reader, row, metadata, includeParsedFields), parsedFields(reader, row, metadata, parsed(row.parsedJson()), includeParsedFields),
row.protocol(), row.protocol(),
row.vehicleKey(), row.vehicleKey(),
row.vin(), row.vin(),
@@ -231,8 +233,10 @@ public final class RawFrameHistoryController {
private static Map<String, Object> parsedFields(TdengineHistoryReader reader, private static Map<String, Object> parsedFields(TdengineHistoryReader reader,
TdengineRawFrameRow row, TdengineRawFrameRow row,
Map<String, Object> metadata, Map<String, Object> metadata,
Map<String, Object> parsed,
boolean includeParsedFields) throws IOException { boolean includeParsedFields) throws IOException {
Map<String, Object> out = new LinkedHashMap<>(); Map<String, Object> out = new LinkedHashMap<>();
if (parsed.isEmpty()) {
for (Map.Entry<String, Object> entry : metadata.entrySet()) { for (Map.Entry<String, Object> entry : metadata.entrySet()) {
String key = entry.getKey(); String key = entry.getKey();
if (key == null || key.isBlank()) { if (key == null || key.isBlank()) {
@@ -244,6 +248,9 @@ public final class RawFrameHistoryController {
out.put("raw." + key, entry.getValue()); out.put("raw." + key, entry.getValue());
} }
} }
} else {
out.putAll(parsed);
}
if (!includeParsedFields) { if (!includeParsedFields) {
return Map.copyOf(out); return Map.copyOf(out);
} }
@@ -262,6 +269,17 @@ public final class RawFrameHistoryController {
return Map.copyOf(out); return Map.copyOf(out);
} }
private static Map<String, Object> parsed(String parsedJson) {
if (parsedJson == null || parsedJson.isBlank()) {
return Map.of();
}
try {
return OBJECT_MAPPER.readValue(parsedJson, MAP_TYPE);
} catch (Exception ignored) {
return Map.of("_raw", parsedJson);
}
}
private static void addLocation(Map<String, Object> out, TdengineLocationRow row) { private static void addLocation(Map<String, Object> out, TdengineLocationRow row) {
out.put("location.longitude", row.longitude()); out.put("location.longitude", row.longitude());
out.put("location.latitude", row.latitude()); out.put("location.latitude", row.latitude());

View File

@@ -150,6 +150,7 @@ public final class TelemetryEnvelopeRecordMapper {
payload.put("infoType", metadata.get("infoType")); payload.put("infoType", metadata.get("infoType"));
} }
payload.put("rawSizeBytes", rawArchive.getSizeBytes()); payload.put("rawSizeBytes", rawArchive.getSizeBytes());
putParsedJson(payload, rawArchive.getParsedJson());
payload.put("metadata", metadata); payload.put("metadata", metadata);
try { try {
return OBJECT_MAPPER.writeValueAsString(payload); return OBJECT_MAPPER.writeValueAsString(payload);
@@ -157,4 +158,16 @@ public final class TelemetryEnvelopeRecordMapper {
throw new IllegalArgumentException("failed to serialize raw_archive", e); throw new IllegalArgumentException("failed to serialize raw_archive", e);
} }
} }
private static void putParsedJson(Map<String, Object> payload, String parsedJson) {
if (parsedJson == null || parsedJson.isBlank()) {
return;
}
try {
payload.put("parsed", OBJECT_MAPPER.readTree(parsedJson));
} catch (JsonProcessingException ex) {
payload.put("parsed", parsedJson);
payload.put("parsedJsonError", ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
}
}
} }

View File

@@ -26,7 +26,11 @@ class EventHistoryControllerTest {
"2026-06-22", "2026-06-22",
EventFileQuery.Order.DESC, EventFileQuery.Order.DESC,
50, 50,
"VIN001"); "VIN001",
"RAW_ARCHIVE",
"2026-06-22T08:00:00Z",
"2026-06-22T08:00:01Z",
"event-0");
assertThat(records).extracting(EventHistoryController.RecordResponse::eventId).containsExactly("event-1"); assertThat(records).extracting(EventHistoryController.RecordResponse::eventId).containsExactly("event-1");
assertThat(records.getFirst().eventTime()).isEqualTo("2026-06-22T08:00:00Z"); assertThat(records.getFirst().eventTime()).isEqualTo("2026-06-22T08:00:00Z");
@@ -37,6 +41,10 @@ class EventHistoryControllerTest {
assertThat(store.lastQuery.order()).isEqualTo(EventFileQuery.Order.DESC); assertThat(store.lastQuery.order()).isEqualTo(EventFileQuery.Order.DESC);
assertThat(store.lastQuery.limit()).isEqualTo(50); assertThat(store.lastQuery.limit()).isEqualTo(50);
assertThat(store.lastQuery.vin()).isEqualTo("VIN001"); assertThat(store.lastQuery.vin()).isEqualTo("VIN001");
assertThat(store.lastQuery.eventType()).isEqualTo("RAW_ARCHIVE");
assertThat(store.lastQuery.cursorEventTime()).isEqualTo(Instant.parse("2026-06-22T08:00:00Z"));
assertThat(store.lastQuery.cursorIngestTime()).isEqualTo(Instant.parse("2026-06-22T08:00:01Z"));
assertThat(store.lastQuery.cursorEventId()).isEqualTo("event-0");
} }
private static EventFileRecord record(String eventId) { private static EventFileRecord record(String eventId) {

View File

@@ -126,7 +126,7 @@ class EventHistoryEnvelopeIngestorTest {
} }
@Test @Test
void tryIngestWritesRawLocationAndTelemetryFactsToTdengineWhenWriterExists() throws Exception { void tryIngestWritesRawAndLocationFactsToTdengineWhenWriterExists() throws Exception {
CapturingStore store = new CapturingStore(); CapturingStore store = new CapturingStore();
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter(); CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor( EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
@@ -180,10 +180,7 @@ class EventHistoryEnvelopeIngestorTest {
.extracting(TdengineLocationRow::factId) .extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1"); .containsExactly("jt808-location-1");
assertThat(tdengineWriter.locations.getFirst().longitude()).isEqualTo(113.12); assertThat(tdengineWriter.locations.getFirst().longitude()).isEqualTo(113.12);
assertThat(tdengineWriter.telemetryFields) assertThat(tdengineWriter.telemetryFields).isEmpty();
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
assertThat(tdengineWriter.telemetryFields.getFirst().valueDouble()).isEqualTo(42.5);
} }
@Test @Test
@@ -200,16 +197,14 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1"); .containsExactly("frame-jt808-1");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId) assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1"); .containsExactly("jt808-location-1");
assertThat(tdengineWriter.telemetryFields) assertThat(tdengineWriter.telemetryFields).isEmpty();
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
} }
@Test @Test
void tryIngestCanSkipTelemetryFieldFactsForHighThroughputRuntime() { void tryIngestCanWriteTelemetryFieldFactsWhenExplicitlyEnabled() {
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter(); CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor( EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
new TelemetryEnvelopeRecordMapper(), tdengineWriter, false); new TelemetryEnvelopeRecordMapper(), tdengineWriter, true);
EnvelopeIngestResult result = ingestor.tryIngest( EnvelopeIngestResult result = ingestor.tryIngest(
jt808LocationEnvelope("jt808-location-1", "frame-jt808-1", "013800000001").toByteArray()); jt808LocationEnvelope("jt808-location-1", "frame-jt808-1", "013800000001").toByteArray());
@@ -219,7 +214,9 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1"); .containsExactly("frame-jt808-1");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId) assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1"); .containsExactly("jt808-location-1");
assertThat(tdengineWriter.telemetryFields).isEmpty(); assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
} }
@Test @Test
@@ -249,9 +246,7 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1", "frame-jt808-2"); .containsExactly("frame-jt808-1", "frame-jt808-2");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId) assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1", "jt808-location-2"); .containsExactly("jt808-location-1", "jt808-location-2");
assertThat(tdengineWriter.telemetryFields) assertThat(tdengineWriter.telemetryFields).isEmpty();
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh", "location.speedKmh");
} }
private static VehicleEnvelope envelope(String eventId) { private static VehicleEnvelope envelope(String eventId) {

View File

@@ -314,6 +314,7 @@ class Gb32960DecodedFrameServiceTest {
"", "",
"127.0.0.1:32960", "127.0.0.1:32960",
"{\"platformAccount\":\"Hyundai\"}", "{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960", "GB32960",
vin, vin,
vin, vin,
@@ -360,6 +361,7 @@ class Gb32960DecodedFrameServiceTest {
"", "",
"8.134.95.166:37302", "8.134.95.166:37302",
"{\"platformAccount\":\"Hyundai\"}", "{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960", "GB32960",
"unknown:GB32960:platform-login", "unknown:GB32960:platform-login",
"", "",
@@ -409,6 +411,7 @@ class Gb32960DecodedFrameServiceTest {
"", "",
"127.0.0.1:32960", "127.0.0.1:32960",
"{\"platformAccount\":\"Hyundai\"}", "{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960", "GB32960",
vin, vin,
vin, vin,

View File

@@ -51,13 +51,12 @@ class Jt808LocationHistoryControllerTest {
} }
@Test @Test
void fallsBackToRawArchiveUriInMetadataWhenLocationRawUriIsBlank() throws Exception { void returnsLocationRawUriFromCoreColumn() throws Exception {
CapturingReader reader = new CapturingReader(new TdenginePage<>( CapturingReader reader = new CapturingReader(new TdenginePage<>(
List.of(row( List.of(row(
"fact-blank-raw", "fact-blank-raw",
"2026-06-29T05:00:01Z", "2026-06-29T05:00:01Z",
"", "archive://jt808/core-frame.bin")),
"{\"rawArchiveUri\":\"archive://jt808/metadata-frame.bin\"}")),
Optional.empty())); Optional.empty()));
Jt808LocationHistoryController controller = new Jt808LocationHistoryController(reader); Jt808LocationHistoryController controller = new Jt808LocationHistoryController(reader);
@@ -71,7 +70,7 @@ class Jt808LocationHistoryControllerTest {
null); null);
assertThat(response.items().getFirst().rawUri()) assertThat(response.items().getFirst().rawUri())
.isEqualTo("archive://jt808/metadata-frame.bin"); .isEqualTo("archive://jt808/core-frame.bin");
} }
@Test @Test
@@ -92,10 +91,10 @@ class Jt808LocationHistoryControllerTest {
} }
private static TdengineLocationRow row(String factId, String ts) { private static TdengineLocationRow row(String factId, String ts) {
return row(factId, ts, "archive://jt808/frame-1.bin", "{\"messageId\":\"512\"}"); return row(factId, ts, "archive://jt808/frame-1.bin");
} }
private static TdengineLocationRow row(String factId, String ts, String rawUri, String metadataJson) { private static TdengineLocationRow row(String factId, String ts, String rawUri) {
Instant instant = Instant.parse(ts); Instant instant = Instant.parse(ts);
return new TdengineLocationRow( return new TdengineLocationRow(
instant, instant,
@@ -111,7 +110,6 @@ class Jt808LocationHistoryControllerTest {
3L, 3L,
null, null,
rawUri, rawUri,
metadataJson,
"JT808", "JT808",
"jt808:g7gps", "jt808:g7gps",
"", "",

View File

@@ -96,6 +96,7 @@ class Jt808RawFrameHistoryControllerTest {
"", "",
"222.66.200.68:41000", "222.66.200.68:41000",
"{\"jt808.register.deviceId\":\"9963320\",\"jt808.register.plate\":\"沪A61559F\"}", "{\"jt808.register.deviceId\":\"9963320\",\"jt808.register.plate\":\"沪A61559F\"}",
"",
"JT808", "JT808",
"jt808:13079963320", "jt808:13079963320",
"unknown", "unknown",

View File

@@ -82,7 +82,6 @@ class LocationHistoryControllerTest {
1, 1,
null, null,
"", "",
"{\"rawArchiveUri\":\"archive://raw.bin\"}",
"MQTT_YUTONG", "MQTT_YUTONG",
"LMRKH9AC2R1004087", "LMRKH9AC2R1004087",
"LMRKH9AC2R1004087", "LMRKH9AC2R1004087",

View File

@@ -91,7 +91,6 @@ class RawFrameHistoryControllerTest {
524290, 524290,
14504.446, 14504.446,
raw.rawUri(), raw.rawUri(),
"{\"rawArchiveUri\":\"" + raw.rawUri() + "\"}",
"JT808", "JT808",
"VIN-JT808-1", "VIN-JT808-1",
"VIN-JT808-1", "VIN-JT808-1",
@@ -183,6 +182,7 @@ class RawFrameHistoryControllerTest {
"xinda", "xinda",
"{\"rawArchiveUri\":\"archive://2026/06/29/XINDA_PUSH/unknown/" + frameId "{\"rawArchiveUri\":\"archive://2026/06/29/XINDA_PUSH/unknown/" + frameId
+ ".bin\",\"plateNo\":\"粤BD12345\",\"rawJson\":\"{\\\"carName\\\":\\\"粤BD12345\\\"}\"}", + ".bin\",\"plateNo\":\"粤BD12345\",\"rawJson\":\"{\\\"carName\\\":\\\"粤BD12345\\\"}\"}",
"",
"XINDA_PUSH", "XINDA_PUSH",
"LB9A32A24P0LS1270", "LB9A32A24P0LS1270",
"LB9A32A24P0LS1270", "LB9A32A24P0LS1270",

View File

@@ -2,9 +2,11 @@ package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.eventfilestore.EventFileRecord; import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.mq.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -12,6 +14,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
class TelemetryEnvelopeRecordMapperTest { class TelemetryEnvelopeRecordMapperTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test @Test
void mapsFullFieldEnvelopeToEventFileRecord() { void mapsFullFieldEnvelopeToEventFileRecord() {
VehicleEnvelope envelope = VehicleEnvelope.newBuilder() VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
@@ -91,4 +95,33 @@ class TelemetryEnvelopeRecordMapperTest {
.containsEntry("tenant", "ln") .containsEntry("tenant", "ln")
.containsEntry("originalSource", "FUTURE_OEM"); .containsEntry("originalSource", "FUTURE_OEM");
} }
@Test
void rawArchiveRecordStoresParsedJsonInPayloadOnly() throws Exception {
VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
.setSchemaVersion("1.0")
.setEventId("raw-808-1")
.setTraceId("trace-raw-1")
.setVin("VIN808")
.setSource("JT808")
.setEventTimeMs(1_782_112_400_000L)
.setIngestTimeMs(1_782_112_401_000L)
.putMetadata("phone", "13307795425")
.setRawArchive(RawArchiveRef.newBuilder()
.setUri("archive://2026/06/30/JT808/VIN808/raw-808-1.bin")
.setSizeBytes(64)
.setParsedJson("{\"messageId\":\"0x0200\",\"body\":{\"speedKmh\":60.5}}")
.build())
.build();
EventFileRecord record = new TelemetryEnvelopeRecordMapper().toRecord(envelope);
assertThat(record.eventType()).isEqualTo("RAW_ARCHIVE");
assertThat(record.metadata())
.containsEntry("phone", "13307795425")
.doesNotContainKey("rawArchiveParsedJson");
var payload = objectMapper.readTree(record.payloadJson());
assertThat(payload.path("parsed").path("messageId").asText()).isEqualTo("0x0200");
assertThat(payload.path("parsed").path("body").path("speedKmh").asDouble()).isEqualTo(60.5);
}
} }

View File

@@ -35,11 +35,14 @@ class EventHistoryAutoConfigurationTest {
@Test @Test
void createsHistoryBeansWhenEnabled() { void createsHistoryBeansWhenEnabled() {
contextRunner contextRunner
.withPropertyValues("lingniu.ingest.event-history.enabled=true") .withPropertyValues(
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=true")
.run(context -> { .run(context -> {
assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class); assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class);
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class); assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context).hasSingleBean(EnvelopeConsumerProcessor.class); assertThat(context.getBeansOfType(EnvelopeConsumerProcessor.class))
.containsOnlyKeys("eventHistoryEnvelopeConsumerProcessor");
assertThat(context).hasSingleBean(EventHistoryController.class); assertThat(context).hasSingleBean(EventHistoryController.class);
assertThat(context).doesNotHaveBean(Gb32960DecodedFrameService.class); assertThat(context).doesNotHaveBean(Gb32960DecodedFrameService.class);
assertThat(context).doesNotHaveBean(Gb32960FrameController.class); assertThat(context).doesNotHaveBean(Gb32960FrameController.class);

View File

@@ -23,6 +23,9 @@ import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore; import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -36,7 +39,11 @@ import java.nio.file.Path;
import java.time.Duration; import java.time.Duration;
import java.time.ZoneId; import java.time.ZoneId;
@AutoConfiguration @AutoConfiguration(after = {
DataSourceAutoConfiguration.class,
JdbcTemplateAutoConfiguration.class,
RedisAutoConfiguration.class
})
@EnableConfigurationProperties(VehicleStatProperties.class) @EnableConfigurationProperties(VehicleStatProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.vehicle-stat", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "lingniu.ingest.vehicle-stat", name = "enabled", havingValue = "true")
public class VehicleStatAutoConfiguration { public class VehicleStatAutoConfiguration {

View File

@@ -4,13 +4,16 @@ import org.springframework.jdbc.core.JdbcTemplate;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.time.Instant; import java.time.Instant;
import java.util.Locale;
import java.util.Objects;
public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageRepository { public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageRepository {
private final JdbcTemplate jdbc; private final JdbcTemplate jdbc;
public JdbcJt808DailyMileageRepository(JdbcTemplate jdbc) { public JdbcJt808DailyMileageRepository(JdbcTemplate jdbc) {
this.jdbc = jdbc; this.jdbc = Objects.requireNonNull(jdbc, "jdbc must not be null");
initializeSchema();
} }
@Override @Override
@@ -18,18 +21,23 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
jdbc.update(""" jdbc.update("""
insert into vehicle_daily_mileage_jt808 ( insert into vehicle_daily_mileage_jt808 (
stat_date, vehicle_key, vin, phone, first_event_time, last_event_time, stat_date, vehicle_key, vin, phone, first_event_time, last_event_time,
gps_mileage_km, speed_integral_km, odometer_mileage_km, daily_mileage_km, mileage_source, gps_mileage_km, speed_integral_km,
odometer_mileage_km, first_total_mileage_km, last_total_mileage_km,
accepted_points, bad_jump_segments, long_gap_segments, out_of_order_points, accepted_points, bad_jump_segments, long_gap_segments, out_of_order_points,
odometer_anomalies, data_quality odometer_anomalies, data_quality
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on duplicate key update on duplicate key update
vin = values(vin), vin = values(vin),
phone = values(phone), phone = values(phone),
first_event_time = values(first_event_time), first_event_time = values(first_event_time),
last_event_time = values(last_event_time), last_event_time = values(last_event_time),
daily_mileage_km = values(daily_mileage_km),
mileage_source = values(mileage_source),
gps_mileage_km = values(gps_mileage_km), gps_mileage_km = values(gps_mileage_km),
speed_integral_km = values(speed_integral_km), speed_integral_km = values(speed_integral_km),
odometer_mileage_km = values(odometer_mileage_km), odometer_mileage_km = values(odometer_mileage_km),
first_total_mileage_km = values(first_total_mileage_km),
last_total_mileage_km = values(last_total_mileage_km),
accepted_points = values(accepted_points), accepted_points = values(accepted_points),
bad_jump_segments = values(bad_jump_segments), bad_jump_segments = values(bad_jump_segments),
long_gap_segments = values(long_gap_segments), long_gap_segments = values(long_gap_segments),
@@ -44,9 +52,13 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
result.phone(), result.phone(),
timestamp(result.firstEventTime()), timestamp(result.firstEventTime()),
timestamp(result.lastEventTime()), timestamp(result.lastEventTime()),
result.dailyMileageKm(),
result.mileageSource(),
result.gpsMileageKm(), result.gpsMileageKm(),
result.speedIntegralKm(), result.speedIntegralKm(),
result.odometerMileageKm(), result.odometerMileageKm(),
result.firstTotalMileageKm(),
result.lastTotalMileageKm(),
result.acceptedPoints(), result.acceptedPoints(),
result.badJumpSegments(), result.badJumpSegments(),
result.longGapSegments(), result.longGapSegments(),
@@ -55,6 +67,59 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
result.dataQuality()); result.dataQuality());
} }
private void initializeSchema() {
jdbc.execute("""
create table if not exists vehicle_daily_mileage_jt808 (
stat_date date not null,
vehicle_key varchar(64) not null,
vin varchar(32) null,
phone varchar(32) null,
first_event_time timestamp null,
last_event_time timestamp null,
daily_mileage_km decimal(12,3) not null default 0,
mileage_source varchar(32) not null default 'INSUFFICIENT',
gps_mileage_km decimal(12,3) not null,
speed_integral_km decimal(12,3) not null,
odometer_mileage_km decimal(12,3) null,
first_total_mileage_km decimal(12,3) null,
last_total_mileage_km decimal(12,3) null,
accepted_points int not null,
bad_jump_segments int not null,
long_gap_segments int not null,
out_of_order_points int not null,
odometer_anomalies int not null,
data_quality varchar(16) not null,
updated_at timestamp not null default current_timestamp on update current_timestamp,
primary key (stat_date, vehicle_key),
key idx_vehicle_daily_mileage_jt808_vehicle_date (vehicle_key, stat_date),
key idx_vehicle_daily_mileage_jt808_vin_date (vin, stat_date),
key idx_vehicle_daily_mileage_jt808_phone_date (phone, stat_date)
)
""");
ensureColumn("daily_mileage_km", "daily_mileage_km decimal(12,3) not null default 0");
ensureColumn("mileage_source", "mileage_source varchar(32) not null default 'INSUFFICIENT'");
ensureColumn("first_total_mileage_km", "first_total_mileage_km decimal(12,3) null");
ensureColumn("last_total_mileage_km", "last_total_mileage_km decimal(12,3) null");
}
private void ensureColumn(String columnName, String definition) {
if (!columnExists(columnName)) {
jdbc.execute("alter table vehicle_daily_mileage_jt808 add column " + definition);
}
}
private boolean columnExists(String columnName) {
Integer count = jdbc.queryForObject("""
select count(*)
from information_schema.columns
where lower(table_name) = ?
and lower(column_name) = ?
""", Integer.class,
"vehicle_daily_mileage_jt808",
columnName.toLowerCase(Locale.ROOT));
return count != null && count > 0;
}
private static Timestamp timestamp(Instant instant) { private static Timestamp timestamp(Instant instant) {
return instant == null ? null : Timestamp.from(instant); return instant == null ? null : Timestamp.from(instant);
} }

View File

@@ -10,9 +10,13 @@ public record Jt808DailyMileageResult(
String phone, String phone,
Instant firstEventTime, Instant firstEventTime,
Instant lastEventTime, Instant lastEventTime,
double dailyMileageKm,
String mileageSource,
double gpsMileageKm, double gpsMileageKm,
double speedIntegralKm, double speedIntegralKm,
Double odometerMileageKm, Double odometerMileageKm,
Double firstTotalMileageKm,
Double lastTotalMileageKm,
int acceptedPoints, int acceptedPoints,
int badJumpSegments, int badJumpSegments,
int longGapSegments, int longGapSegments,

View File

@@ -12,6 +12,7 @@ public final class Jt808DailyMileageState {
private Instant firstEventTime; private Instant firstEventTime;
private Instant lastEventTime; private Instant lastEventTime;
private Jt808LocationPoint lastPoint; private Jt808LocationPoint lastPoint;
private double firstTotalMileageKm = Double.NaN;
private double lastTotalMileageKm = Double.NaN; private double lastTotalMileageKm = Double.NaN;
private double gpsMileageKm; private double gpsMileageKm;
private double speedIntegralKm; private double speedIntegralKm;
@@ -47,6 +48,7 @@ public final class Jt808DailyMileageState {
Instant firstEventTime, Instant firstEventTime,
Instant lastEventTime, Instant lastEventTime,
Jt808LocationPoint lastPoint, Jt808LocationPoint lastPoint,
double firstTotalMileageKm,
double lastTotalMileageKm, double lastTotalMileageKm,
double gpsMileageKm, double gpsMileageKm,
double speedIntegralKm, double speedIntegralKm,
@@ -60,6 +62,7 @@ public final class Jt808DailyMileageState {
state.firstEventTime = firstEventTime; state.firstEventTime = firstEventTime;
state.lastEventTime = lastEventTime; state.lastEventTime = lastEventTime;
state.lastPoint = lastPoint; state.lastPoint = lastPoint;
state.firstTotalMileageKm = firstTotalMileageKm;
state.lastTotalMileageKm = lastTotalMileageKm; state.lastTotalMileageKm = lastTotalMileageKm;
state.gpsMileageKm = gpsMileageKm; state.gpsMileageKm = gpsMileageKm;
state.speedIntegralKm = speedIntegralKm; state.speedIntegralKm = speedIntegralKm;
@@ -82,7 +85,7 @@ public final class Jt808DailyMileageState {
} }
if (lastPoint == null) { if (lastPoint == null) {
firstEventTime = point.eventTime(); firstEventTime = point.eventTime();
updateOdometer(point); updateOdometer(point, calculator);
accept(point); accept(point);
return; return;
} }
@@ -100,7 +103,7 @@ public final class Jt808DailyMileageState {
gpsMileageKm += segment.gpsKm(); gpsMileageKm += segment.gpsKm();
speedIntegralKm += segment.speedIntegralKm(); speedIntegralKm += segment.speedIntegralKm();
} }
updateOdometer(point); updateOdometer(point, calculator);
accept(point); accept(point);
} }
@@ -112,9 +115,13 @@ public final class Jt808DailyMileageState {
phone, phone,
firstEventTime, firstEventTime,
lastEventTime, lastEventTime,
dailyMileageKm(),
mileageSource(),
gpsMileageKm, gpsMileageKm,
speedIntegralKm, speedIntegralKm,
odometerMileageKm > 0.0 ? odometerMileageKm : null, odometerMileageKm > 0.0 ? odometerMileageKm : null,
nullable(firstTotalMileageKm),
nullable(lastTotalMileageKm),
acceptedPoints, acceptedPoints,
badJumpSegments, badJumpSegments,
longGapSegments, longGapSegments,
@@ -129,11 +136,14 @@ public final class Jt808DailyMileageState {
acceptedPoints++; acceptedPoints++;
} }
private void updateOdometer(Jt808LocationPoint point) { private void updateOdometer(Jt808LocationPoint point, Jt808GpsMileageCalculator calculator) {
if (point.totalMileageKm() == null || !Double.isFinite(point.totalMileageKm())) { if (point.totalMileageKm() == null || !Double.isFinite(point.totalMileageKm())) {
return; return;
} }
double current = point.totalMileageKm(); double current = point.totalMileageKm();
if (!Double.isFinite(firstTotalMileageKm)) {
firstTotalMileageKm = current;
}
if (!Double.isFinite(lastTotalMileageKm)) { if (!Double.isFinite(lastTotalMileageKm)) {
lastTotalMileageKm = current; lastTotalMileageKm = current;
return; return;
@@ -143,15 +153,56 @@ public final class Jt808DailyMileageState {
lastTotalMileageKm = current; lastTotalMileageKm = current;
return; return;
} }
odometerMileageKm += current - lastTotalMileageKm; double delta = current - lastTotalMileageKm;
if (lastPoint != null && !calculator.plausibleOdometerDelta(lastPoint, point, delta)) {
odometerAnomalies++;
lastTotalMileageKm = current; lastTotalMileageKm = current;
return;
}
odometerMileageKm += delta;
lastTotalMileageKm = current;
}
private double dailyMileageKm() {
if (hasUsableTotalMileage()) {
return odometerMileageKm;
}
if (gpsMileageKm > 0.0) {
return gpsMileageKm;
}
return 0.0;
}
private String mileageSource() {
if (hasUsableTotalMileage()) {
return odometerAnomalies > 0 ? "JT808_TOTAL_MILEAGE_PARTIAL" : "JT808_TOTAL_MILEAGE";
}
if (gpsMileageKm > 0.0) {
return "GPS_DISTANCE";
}
return "INSUFFICIENT";
}
private boolean hasUsableTotalMileage() {
if (odometerMileageKm > 0.0) {
return true;
}
return acceptedPoints >= 2
&& odometerAnomalies == 0
&& Double.isFinite(firstTotalMileageKm)
&& Double.isFinite(lastTotalMileageKm)
&& lastTotalMileageKm >= firstTotalMileageKm;
}
private static Double nullable(double value) {
return Double.isFinite(value) ? value : null;
} }
private String dataQuality() { private String dataQuality() {
if (acceptedPoints < 2) { if (acceptedPoints < 2) {
return "PARTIAL"; return "PARTIAL";
} }
if (badJumpSegments > 0 || longGapSegments > 0 || vehicleKey.startsWith("jt808:")) { if (badJumpSegments > 0 || longGapSegments > 0 || odometerAnomalies > 0 || vehicleKey.startsWith("jt808:")) {
return "PARTIAL"; return "PARTIAL";
} }
return "GOOD"; return "GOOD";
@@ -201,6 +252,10 @@ public final class Jt808DailyMileageState {
return lastTotalMileageKm; return lastTotalMileageKm;
} }
public double firstTotalMileageKm() {
return firstTotalMileageKm;
}
public int acceptedPoints() { public int acceptedPoints() {
return acceptedPoints; return acceptedPoints;
} }

View File

@@ -5,6 +5,7 @@ import java.time.Duration;
public final class Jt808GpsMileageCalculator { public final class Jt808GpsMileageCalculator {
private static final double EARTH_RADIUS_KM = 6371.0088; private static final double EARTH_RADIUS_KM = 6371.0088;
private static final double ODOMETER_DELTA_TOLERANCE_KM = 0.1;
private final Duration maxSegmentGap; private final Duration maxSegmentGap;
private final double maxImpliedSpeedKmh; private final double maxImpliedSpeedKmh;
@@ -61,6 +62,20 @@ public final class Jt808GpsMileageCalculator {
return new SegmentResult(gpsKm, speedIntegralKm, true, false, false, false); return new SegmentResult(gpsKm, speedIntegralKm, true, false, false, false);
} }
public boolean plausibleOdometerDelta(Jt808LocationPoint previous,
Jt808LocationPoint current,
double deltaKm) {
if (previous == null || current == null || !Double.isFinite(deltaKm) || deltaKm < 0.0) {
return false;
}
Duration elapsed = Duration.between(previous.eventTime(), current.eventTime());
if (elapsed.isZero() || elapsed.isNegative()) {
return false;
}
double hours = elapsed.toMillis() / 3_600_000.0;
return deltaKm <= maxImpliedSpeedKmh * hours + ODOMETER_DELTA_TOLERANCE_KM;
}
private static double haversineKm(double lon1, double lat1, double lon2, double lat2) { private static double haversineKm(double lon1, double lat1, double lon2, double lat2) {
double latRad1 = Math.toRadians(lat1); double latRad1 = Math.toRadians(lat1);
double latRad2 = Math.toRadians(lat2); double latRad2 = Math.toRadians(lat2);

View File

@@ -13,7 +13,8 @@ public final class Jt808LocationPointExtractor {
public Optional<Jt808LocationPoint> extract(VehicleEnvelope envelope) { public Optional<Jt808LocationPoint> extract(VehicleEnvelope envelope) {
if (envelope == null if (envelope == null
|| !"JT808".equalsIgnoreCase(envelope.getSource()) || !"JT808".equalsIgnoreCase(envelope.getSource())
|| !envelope.hasTelemetrySnapshot()) { || !envelope.hasTelemetrySnapshot()
|| !"LOCATION".equalsIgnoreCase(envelope.getTelemetrySnapshot().getEventType())) {
return Optional.empty(); return Optional.empty();
} }
Map<String, String> fields = fields(envelope); Map<String, String> fields = fields(envelope);

View File

@@ -64,7 +64,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
String firstEventTime, String firstEventTime,
String lastEventTime, String lastEventTime,
PointSnapshot lastPoint, PointSnapshot lastPoint,
double lastTotalMileageKm, Double firstTotalMileageKm,
Double lastTotalMileageKm,
double gpsMileageKm, double gpsMileageKm,
double speedIntegralKm, double speedIntegralKm,
double odometerMileageKm, double odometerMileageKm,
@@ -83,7 +84,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
format(state.firstEventTime()), format(state.firstEventTime()),
format(state.lastEventTime()), format(state.lastEventTime()),
PointSnapshot.from(state.lastPoint()), PointSnapshot.from(state.lastPoint()),
state.lastTotalMileageKm(), finiteOrNull(state.firstTotalMileageKm()),
finiteOrNull(state.lastTotalMileageKm()),
state.gpsMileageKm(), state.gpsMileageKm(),
state.speedIntegralKm(), state.speedIntegralKm(),
state.odometerMileageKm(), state.odometerMileageKm(),
@@ -103,7 +105,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
parseInstant(firstEventTime), parseInstant(firstEventTime),
parseInstant(lastEventTime), parseInstant(lastEventTime),
lastPoint == null ? null : lastPoint.toPoint(), lastPoint == null ? null : lastPoint.toPoint(),
lastTotalMileageKm, nanIfNull(firstTotalMileageKm),
nanIfNull(lastTotalMileageKm),
gpsMileageKm, gpsMileageKm,
speedIntegralKm, speedIntegralKm,
odometerMileageKm, odometerMileageKm,
@@ -164,4 +167,12 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
private static Instant parseInstant(String value) { private static Instant parseInstant(String value) {
return value == null || value.isBlank() ? null : Instant.parse(value); return value == null || value.isBlank() ? null : Instant.parse(value);
} }
private static Double finiteOrNull(double value) {
return Double.isFinite(value) ? value : null;
}
private static double nanIfNull(Double value) {
return value == null ? Double.NaN : value;
}
} }

View File

@@ -21,10 +21,13 @@ import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration;
import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource;
import java.time.LocalDate; import java.time.LocalDate;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
@@ -117,6 +120,29 @@ class VehicleStatAutoConfigurationTest {
}); });
} }
@Test
void createsJt808MileageBeansAfterJdbcAutoConfiguration() {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(
JdbcTemplateAutoConfiguration.class,
VehicleStatAutoConfiguration.class))
.withBean(EnvelopeDeadLetterSink.class, () -> record -> {})
.withBean(DataSource.class, () -> new DriverManagerDataSource(
"jdbc:h2:mem:vehicle_stat_auto;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1",
"sa",
""))
.withPropertyValues(
"lingniu.ingest.vehicle-stat.enabled=true",
"lingniu.ingest.vehicle-stat.jt808.enabled=true",
"lingniu.ingest.vehicle-stat.jt808.state-store=memory")
.run(context -> {
assertThat(context).hasSingleBean(JdbcTemplate.class);
assertThat(context).hasSingleBean(Jt808DailyMileageRepository.class);
assertThat(context).hasSingleBean(Jt808MileageStreamProcessor.class);
assertThat(context).hasSingleBean(VehicleStatEnvelopeIngestor.class);
});
}
private static final class InMemoryVehicleStatRepository implements VehicleStatRepository { private static final class InMemoryVehicleStatRepository implements VehicleStatRepository {
@Override public void appendMileagePoint(String vin, MileagePoint point) {} @Override public void appendMileagePoint(String vin, MileagePoint point) {}
@Override public List<MileagePoint> mileagePoints(String vin, LocalDate statDate) { return List.of(); } @Override public List<MileagePoint> mileagePoints(String vin, LocalDate statDate) { return List.of(); }

View File

@@ -24,30 +24,20 @@ class JdbcJt808DailyMileageRepositoryTest {
""); "");
jdbc = new JdbcTemplate(dataSource); jdbc = new JdbcTemplate(dataSource);
jdbc.execute("drop table if exists vehicle_daily_mileage_jt808"); jdbc.execute("drop table if exists vehicle_daily_mileage_jt808");
jdbc.execute("""
create table vehicle_daily_mileage_jt808 (
stat_date date not null,
vehicle_key varchar(64) not null,
vin varchar(32) null,
phone varchar(32) null,
first_event_time timestamp null,
last_event_time timestamp null,
gps_mileage_km decimal(12,3) not null,
speed_integral_km decimal(12,3) not null,
odometer_mileage_km decimal(12,3) null,
accepted_points int not null,
bad_jump_segments int not null,
long_gap_segments int not null,
out_of_order_points int not null,
odometer_anomalies int not null,
data_quality varchar(16) not null,
updated_at timestamp not null default current_timestamp,
primary key (stat_date, vehicle_key)
)
""");
repository = new JdbcJt808DailyMileageRepository(jdbc); repository = new JdbcJt808DailyMileageRepository(jdbc);
} }
@Test
void initializesTableWhenMissing() {
Integer count = jdbc.queryForObject("""
select count(*)
from information_schema.tables
where table_name = 'vehicle_daily_mileage_jt808'
""", Integer.class);
assertThat(count).isEqualTo(1);
}
@Test @Test
void upsertsDailyMileageResult() { void upsertsDailyMileageResult() {
Jt808DailyMileageResult first = new Jt808DailyMileageResult( Jt808DailyMileageResult first = new Jt808DailyMileageResult(
@@ -58,8 +48,12 @@ class JdbcJt808DailyMileageRepositoryTest {
Instant.parse("2026-06-30T00:00:00Z"), Instant.parse("2026-06-30T00:00:00Z"),
Instant.parse("2026-06-30T00:01:00Z"), Instant.parse("2026-06-30T00:01:00Z"),
1.23456, 1.23456,
"GPS_DISTANCE",
1.23456,
1.2, 1.2,
null, null,
null,
null,
2, 2,
0, 0,
0, 0,
@@ -73,9 +67,13 @@ class JdbcJt808DailyMileageRepositoryTest {
"13900000001", "13900000001",
Instant.parse("2026-06-30T00:00:00Z"), Instant.parse("2026-06-30T00:00:00Z"),
Instant.parse("2026-06-30T00:02:00Z"), Instant.parse("2026-06-30T00:02:00Z"),
2.0,
"JT808_TOTAL_MILEAGE",
2.34567, 2.34567,
2.3, 2.3,
2.0, 2.0,
100.0,
102.0,
3, 3,
1, 1,
0, 0,
@@ -87,14 +85,19 @@ class JdbcJt808DailyMileageRepositoryTest {
repository.upsert(updated); repository.upsert(updated);
Map<String, Object> row = jdbc.queryForMap(""" Map<String, Object> row = jdbc.queryForMap("""
select vehicle_key, gps_mileage_km, odometer_mileage_km, accepted_points, select vehicle_key, daily_mileage_km, mileage_source, gps_mileage_km,
bad_jump_segments, data_quality odometer_mileage_km, first_total_mileage_km, last_total_mileage_km,
accepted_points, bad_jump_segments, data_quality
from vehicle_daily_mileage_jt808 from vehicle_daily_mileage_jt808
where stat_date = '2026-06-30' and vehicle_key = 'VIN001' where stat_date = '2026-06-30' and vehicle_key = 'VIN001'
"""); """);
assertThat(row.get("vehicle_key")).isEqualTo("VIN001"); assertThat(row.get("vehicle_key")).isEqualTo("VIN001");
assertThat(row.get("daily_mileage_km").toString()).isEqualTo("2.000");
assertThat(row.get("mileage_source")).isEqualTo("JT808_TOTAL_MILEAGE");
assertThat(row.get("gps_mileage_km").toString()).isEqualTo("2.346"); assertThat(row.get("gps_mileage_km").toString()).isEqualTo("2.346");
assertThat(row.get("odometer_mileage_km").toString()).isEqualTo("2.000"); assertThat(row.get("odometer_mileage_km").toString()).isEqualTo("2.000");
assertThat(row.get("first_total_mileage_km").toString()).isEqualTo("100.000");
assertThat(row.get("last_total_mileage_km").toString()).isEqualTo("102.000");
assertThat(row.get("accepted_points")).isEqualTo(3); assertThat(row.get("accepted_points")).isEqualTo(3);
assertThat(row.get("bad_jump_segments")).isEqualTo(1); assertThat(row.get("bad_jump_segments")).isEqualTo(1);
assertThat(row.get("data_quality")).isEqualTo("PARTIAL"); assertThat(row.get("data_quality")).isEqualTo("PARTIAL");

View File

@@ -25,6 +25,10 @@ class Jt808DailyMileageStateTest {
assertThat(state.gpsMileageKm()).isZero(); assertThat(state.gpsMileageKm()).isZero();
assertThat(state.speedIntegralKm()).isZero(); assertThat(state.speedIntegralKm()).isZero();
assertThat(state.odometerMileageKm()).isZero(); assertThat(state.odometerMileageKm()).isZero();
assertThat(state.firstTotalMileageKm()).isEqualTo(100.0);
assertThat(state.lastTotalMileageKm()).isEqualTo(100.0);
assertThat(state.toResult().dailyMileageKm()).isZero();
assertThat(state.toResult().mileageSource()).isEqualTo("INSUFFICIENT");
assertThat(state.firstEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z")); assertThat(state.firstEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z"));
assertThat(state.lastEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z")); assertThat(state.lastEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z"));
} }
@@ -41,6 +45,12 @@ class Jt808DailyMileageStateTest {
assertThat(state.gpsMileageKm()).isBetween(0.99, 1.01); assertThat(state.gpsMileageKm()).isBetween(0.99, 1.01);
assertThat(state.speedIntegralKm()).isBetween(0.99, 1.01); assertThat(state.speedIntegralKm()).isBetween(0.99, 1.01);
assertThat(state.odometerMileageKm()).isCloseTo(1.2, offset(0.000001)); assertThat(state.odometerMileageKm()).isCloseTo(1.2, offset(0.000001));
assertThat(state.firstTotalMileageKm()).isEqualTo(100.0);
assertThat(state.lastTotalMileageKm()).isEqualTo(101.2);
assertThat(state.toResult().dailyMileageKm()).isCloseTo(1.2, offset(0.000001));
assertThat(state.toResult().mileageSource()).isEqualTo("JT808_TOTAL_MILEAGE");
assertThat(state.toResult().firstTotalMileageKm()).isEqualTo(100.0);
assertThat(state.toResult().lastTotalMileageKm()).isEqualTo(101.2);
assertThat(state.badJumpSegments()).isZero(); assertThat(state.badJumpSegments()).isZero();
assertThat(state.longGapSegments()).isZero(); assertThat(state.longGapSegments()).isZero();
} }
@@ -68,6 +78,36 @@ class Jt808DailyMileageStateTest {
assertThat(state.odometerMileageKm()).isZero(); assertThat(state.odometerMileageKm()).isZero();
assertThat(state.odometerAnomalies()).isEqualTo(1); assertThat(state.odometerAnomalies()).isEqualTo(1);
assertThat(state.toResult().dailyMileageKm()).isGreaterThan(0.9);
assertThat(state.toResult().mileageSource()).isEqualTo("GPS_DISTANCE");
}
@Test
void odometerJumpAboveConfiguredSpeedIsIgnoredAndCounted() {
Jt808DailyMileageState state = Jt808DailyMileageState.empty("VIN001", "VIN001", "13900000001",
LocalDate.of(2026, 6, 30));
state.apply(point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0, 100.0), calculator);
state.apply(point("2026-06-30T00:01:00Z", 120.000100, 30.000000, 36.0, 200.0), calculator);
assertThat(state.odometerMileageKm()).isZero();
assertThat(state.odometerAnomalies()).isEqualTo(1);
assertThat(state.toResult().dataQuality()).isEqualTo("PARTIAL");
}
@Test
void gpsDistanceIsFallbackWhenTotalMileageIsMissing() {
Jt808DailyMileageState state = Jt808DailyMileageState.empty("VIN001", "VIN001", "13900000001",
LocalDate.of(2026, 6, 30));
state.apply(point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0, null), calculator);
state.apply(point("2026-06-30T00:01:40Z", 120.010370, 30.000000, 36.0, null), calculator);
assertThat(state.toResult().dailyMileageKm()).isBetween(0.99, 1.01);
assertThat(state.toResult().mileageSource()).isEqualTo("GPS_DISTANCE");
assertThat(state.toResult().odometerMileageKm()).isNull();
assertThat(state.toResult().firstTotalMileageKm()).isNull();
assertThat(state.toResult().lastTotalMileageKm()).isNull();
} }
private static Jt808LocationPoint point(String time, private static Jt808LocationPoint point(String time,

View File

@@ -71,6 +71,18 @@ class Jt808GpsMileageCalculatorTest {
assertThat(result.gpsKm()).isZero(); assertThat(result.gpsKm()).isZero();
} }
@Test
void validatesPlausibleOdometerDeltaByConfiguredSpeed() {
assertThat(calculator.plausibleOdometerDelta(
point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0),
point("2026-06-30T00:01:00Z", 120.001000, 30.000000, 36.0),
2.0)).isTrue();
assertThat(calculator.plausibleOdometerDelta(
point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0),
point("2026-06-30T00:01:00Z", 120.001000, 30.000000, 36.0),
20.0)).isFalse();
}
private static Jt808LocationPoint point(String time, double longitude, double latitude, Double speedKmh) { private static Jt808LocationPoint point(String time, double longitude, double latitude, Double speedKmh) {
return new Jt808LocationPoint( return new Jt808LocationPoint(
"VIN001", "VIN001",

View File

@@ -57,6 +57,21 @@ class Jt808LocationPointExtractorTest {
field("longitude", "118.901034")))).isEmpty(); field("longitude", "118.901034")))).isEmpty();
} }
@Test
void ignoresJt808NonLocationSnapshot() {
VehicleEnvelope envelope = envelope("JT808", "VIN001", "13900000001",
field("longitude", "118.901034"),
field("latitude", "31.946325"))
.toBuilder()
.setTelemetrySnapshot(TelemetrySnapshot.newBuilder()
.setEventType("REGISTER")
.addFields(field("longitude", "118.901034"))
.addFields(field("latitude", "31.946325")))
.build();
assertThat(extractor.extract(envelope)).isEmpty();
}
private static VehicleEnvelope envelope(String source, String vin, String phone, TelemetryField... fields) { private static VehicleEnvelope envelope(String source, String vin, String phone, TelemetryField... fields) {
TelemetrySnapshot.Builder snapshot = TelemetrySnapshot.newBuilder().setEventType("LOCATION"); TelemetrySnapshot.Builder snapshot = TelemetrySnapshot.newBuilder().setEventType("LOCATION");
for (TelemetryField field : fields) { for (TelemetryField field : fields) {

View File

@@ -30,6 +30,8 @@ class Jt808MileageStreamProcessorTest {
assertThat(state.gpsMileageKm()).isGreaterThan(0.9); assertThat(state.gpsMileageKm()).isGreaterThan(0.9);
assertThat(repository.results).hasSize(2); assertThat(repository.results).hasSize(2);
assertThat(repository.results.getLast().vehicleKey()).isEqualTo("VIN001"); assertThat(repository.results.getLast().vehicleKey()).isEqualTo("VIN001");
assertThat(repository.results.getLast().dailyMileageKm()).isEqualTo(1.0);
assertThat(repository.results.getLast().mileageSource()).isEqualTo("JT808_TOTAL_MILEAGE");
assertThat(repository.results.getLast().odometerMileageKm()).isEqualTo(1.0); assertThat(repository.results.getLast().odometerMileageKm()).isEqualTo(1.0);
} }

View File

@@ -36,11 +36,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
private static final TypeReference<Map<String, String>> STRING_MAP = private static final TypeReference<Map<String, String>> STRING_MAP =
new TypeReference<>() {}; new TypeReference<>() {};
private static final String HEADER_VERSION = "event-records-v1"; private static final String HEADER_VERSION = "event-records-v1";
private static final int VEHICLE_BUCKETS = 64;
private final Path root; private final Path root;
private final Path indexPath; private final Path indexPath;
private final ZoneId partitionZone; private final ZoneId partitionZone;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final boolean writeParquetEnabled;
private volatile boolean indexInitialized; private volatile boolean indexInitialized;
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) { public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) {
@@ -48,6 +50,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
} }
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) { public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) {
this(root, partitionZone, objectMapper, true);
}
public DuckDbParquetEventFileStore(Path root,
ZoneId partitionZone,
ObjectMapper objectMapper,
boolean writeParquetEnabled) {
if (root == null) { if (root == null) {
throw new IllegalArgumentException("root must not be null"); throw new IllegalArgumentException("root must not be null");
} }
@@ -55,6 +64,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
this.indexPath = this.root.resolve("events.duckdb"); this.indexPath = this.root.resolve("events.duckdb");
this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone; this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone;
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper; this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
this.writeParquetEnabled = writeParquetEnabled;
} }
@Override @Override
@@ -67,16 +77,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
Partition partition = partition(record); Partition partition = partition(record);
byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record); byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
} }
for (Map.Entry<Partition, List<EventFileRecord>> entry : byPartition.entrySet()) { if (writeParquetEnabled) {
writePartition(entry.getKey(), entry.getValue()); writePartitions(byPartition);
appendIndex(entry.getKey(), entry.getValue());
} }
appendIndex(byPartition);
} }
@Override @Override
public List<EventFileRecord> query(EventFileQuery query) throws IOException { public List<EventFileRecord> query(EventFileQuery query) throws IOException {
// 有 VIN 时直接扫该车分区 parquet避免先查总索引再和车辆文件 join // 有 VIN 且启用 Parquet 副本时,可直接扫分区文件;索引-only 模式走 DuckDB 主索引
if (query.vin() != null) { if (query.vin() != null && writeParquetEnabled) {
return queryParquet(query); return queryParquet(query);
} }
// 无 VIN 的管理类查询走 DuckDB 索引,代价更高但不影响高频单车查询路径。 // 无 VIN 的管理类查询走 DuckDB 索引,代价更高但不影响高频单车查询路径。
@@ -130,6 +140,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) { if (query.eventTimeTo() != null) {
predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli()); predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli());
} }
if (query.cursorEventTime() != null) {
predicates.add(cursorPredicateSql(query));
}
// 这里的 SQL 只拼接经过 escape 的内部值;外部 rawArchiveUri 查询使用 PreparedStatement。 // 这里的 SQL 只拼接经过 escape 的内部值;外部 rawArchiveUri 查询使用 PreparedStatement。
String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n"; String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n";
String sql = """ String sql = """
@@ -182,6 +195,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) { if (query.eventTimeTo() != null) {
where.append(" AND event_time_ms <= ?\n"); where.append(" AND event_time_ms <= ?\n");
} }
if (query.cursorEventTime() != null) {
where.append(cursorPredicatePrepared(query.order()));
}
String sql = """ String sql = """
SELECT event_id, protocol, event_type, vin, SELECT event_id, protocol, event_type, vin,
event_time_ms, ingest_time_ms, event_time_ms, ingest_time_ms,
@@ -209,6 +225,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) { if (query.eventTimeTo() != null) {
ps.setLong(index++, query.eventTimeTo().toEpochMilli()); ps.setLong(index++, query.eventTimeTo().toEpochMilli());
} }
if (query.cursorEventTime() != null) {
long eventTimeMs = query.cursorEventTime().toEpochMilli();
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, ingestTimeMs);
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, ingestTimeMs);
ps.setString(index++, query.cursorEventId());
}
ps.setInt(index, query.limit()); ps.setInt(index, query.limit());
try (ResultSet rs = ps.executeQuery()) { try (ResultSet rs = ps.executeQuery()) {
List<EventFileRecord> result = new ArrayList<>(); List<EventFileRecord> result = new ArrayList<>();
@@ -222,16 +248,43 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
} }
} }
private void writePartition(Partition partition, List<EventFileRecord> records) throws IOException { private static String cursorPredicateSql(EventFileQuery query) {
Path dir = partition.dir(root); long eventTimeMs = query.cursorEventTime().toEpochMilli();
Files.createDirectories(dir); long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
Path partFile = dir.resolve("events.parquet"); String eventId = escapeSql(query.cursorEventId());
Path tempFile = dir.resolve("events-" + UUID.randomUUID() + ".tmp.parquet"); String cmp = query.order() == EventFileQuery.Order.DESC ? "<" : ">";
return """
(
event_time_ms %s %d
OR (event_time_ms = %d AND ingest_time_ms %s %d)
OR (event_time_ms = %d AND ingest_time_ms = %d AND event_id %s '%s')
)
""".formatted(cmp, eventTimeMs, eventTimeMs, cmp, ingestTimeMs,
eventTimeMs, ingestTimeMs, cmp, eventId);
}
private static String cursorPredicatePrepared(EventFileQuery.Order order) {
String cmp = order == EventFileQuery.Order.DESC ? "<" : ">";
return """
AND (
event_time_ms %s ?
OR (event_time_ms = ? AND ingest_time_ms %s ?)
OR (event_time_ms = ? AND ingest_time_ms = ? AND event_id %s ?)
)
""".formatted(cmp, cmp, cmp);
}
private void writePartitions(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
if (recordsByPartition.isEmpty()) {
return;
}
List<Path> tempFiles = new ArrayList<>();
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:"); try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
Statement statement = connection.createStatement()) { Statement statement = connection.createStatement()) {
statement.execute(""" statement.execute("""
CREATE TEMPORARY TABLE event_records ( CREATE TEMPORARY TABLE event_records (
partition_id INTEGER,
event_id VARCHAR, event_id VARCHAR,
protocol VARCHAR, protocol VARCHAR,
event_type VARCHAR, event_type VARCHAR,
@@ -246,54 +299,62 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
) )
"""); """);
try (PreparedStatement ps = connection.prepareStatement(""" try (PreparedStatement ps = connection.prepareStatement("""
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) { """)) {
int partitionId = 0;
for (List<EventFileRecord> records : recordsByPartition.values()) {
for (EventFileRecord record : records) { for (EventFileRecord record : records) {
ps.setString(1, record.eventId()); ps.setInt(1, partitionId);
ps.setString(2, record.protocol().name()); ps.setString(2, record.eventId());
ps.setString(3, record.eventType()); ps.setString(3, record.protocol().name());
ps.setString(4, record.vin()); ps.setString(4, record.eventType());
ps.setLong(5, record.eventTime().toEpochMilli()); ps.setString(5, record.vin());
ps.setString(6, record.eventTime().toString()); ps.setLong(6, record.eventTime().toEpochMilli());
ps.setLong(7, record.ingestTime().toEpochMilli()); ps.setString(7, record.eventTime().toString());
ps.setString(8, record.ingestTime().toString()); ps.setLong(8, record.ingestTime().toEpochMilli());
ps.setString(9, record.rawArchiveUri()); ps.setString(9, record.ingestTime().toString());
ps.setString(10, objectMapper.writeValueAsString(record.metadata())); ps.setString(10, record.rawArchiveUri());
ps.setString(11, record.payloadJson()); ps.setString(11, objectMapper.writeValueAsString(record.metadata()));
ps.setString(12, record.payloadJson());
ps.addBatch(); ps.addBatch();
} }
partitionId++;
}
ps.executeBatch(); ps.executeBatch();
} }
if (Files.exists(partFile)) { int partitionId = 0;
// DuckDB 不能原地追加 parquet先 UNION 旧文件和本批数据,再原子替换分区文件。 for (Partition partition : recordsByPartition.keySet()) {
Path dir = partition.dir(root);
Files.createDirectories(dir);
String fragmentId = UUID.randomUUID().toString();
Path partFile = dir.resolve("events-" + fragmentId + ".parquet");
Path tempFile = dir.resolve("events-" + fragmentId + ".tmp.parquet");
tempFiles.add(tempFile);
statement.execute(""" statement.execute("""
CREATE TEMPORARY TABLE combined_records AS COPY (
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM read_parquet('%s')
UNION ALL
SELECT event_id, protocol, event_type, vin, SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time, event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json raw_archive_uri, metadata_json, payload_json
FROM event_records FROM event_records
""".formatted(escapeSql(partFile))); WHERE partition_id = %d
statement.execute("COPY combined_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)"); ) TO '%s' (FORMAT parquet)
} else { """.formatted(partitionId, escapeSql(tempFile)));
statement.execute("COPY event_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
}
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
partitionId++;
}
} catch (SQLException e) { } catch (SQLException e) {
throw new IOException("write parquet event store failed: " + partFile, e); throw new IOException("write parquet event store failed", e);
} finally { } finally {
for (Path tempFile : tempFiles) {
Files.deleteIfExists(tempFile); Files.deleteIfExists(tempFile);
} }
} }
}
private void appendIndex(Partition partition, List<EventFileRecord> records) throws IOException { private void appendIndex(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
ensureIndexInitialized(); ensureIndexInitialized();
try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) { try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) {
insertIndex(connection, partition, records); insertIndex(connection, recordsByPartition);
} catch (SQLException e) { } catch (SQLException e) {
throw new IOException("write duckdb event index failed", e); throw new IOException("write duckdb event index failed", e);
} }
@@ -364,12 +425,14 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
""".formatted(fileList)); """.formatted(fileList));
} }
private void insertIndex(Connection connection, Partition partition, List<EventFileRecord> records) private void insertIndex(Connection connection, Map<Partition, List<EventFileRecord>> recordsByPartition)
throws SQLException, IOException { throws SQLException, IOException {
try (PreparedStatement ps = connection.prepareStatement(""" try (PreparedStatement ps = connection.prepareStatement("""
INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) { """)) {
for (EventFileRecord record : records) { for (Map.Entry<Partition, List<EventFileRecord>> entry : recordsByPartition.entrySet()) {
Partition partition = entry.getKey();
for (EventFileRecord record : entry.getValue()) {
ps.setString(1, record.eventId()); ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name()); ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType()); ps.setString(3, record.eventType());
@@ -382,6 +445,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
ps.setString(10, record.payloadJson()); ps.setString(10, record.payloadJson());
ps.addBatch(); ps.addBatch();
} }
}
ps.executeBatch(); ps.executeBatch();
} }
} }
@@ -390,10 +454,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
List<Path> files = new ArrayList<>(); List<Path> files = new ArrayList<>();
LocalDate date = query.dateFrom(); LocalDate date = query.dateFrom();
while (!date.isAfter(query.dateTo())) { while (!date.isAfter(query.dateTo())) {
Path dir = query.vin() == null Path dateDir = partitionDir(query.protocol(), date);
? partitionDir(query.protocol(), date) List<Path> candidateDirs = query.vin() == null
: partitionDir(query.protocol(), date).resolve("vehicle=" + storageName(query.vin())); ? List.of(dateDir)
// 目录结构把 VIN 放在 date 下,单车多日查询只需要遍历目标日期内的目标车辆目录。 : List.of(
// 新布局:高写入路径按协议/日期写一个批次文件VIN 由 Parquet/索引字段过滤。
dateDir.resolve("header=" + HEADER_VERSION),
// 兼容已落盘的旧布局。
dateDir.resolve("bucket=" + bucketName(query.vin())),
dateDir.resolve("vehicle=" + storageName(query.vin())));
for (Path dir : candidateDirs) {
if (Files.isDirectory(dir)) { if (Files.isDirectory(dir)) {
try (Stream<Path> stream = Files.walk(dir)) { try (Stream<Path> stream = Files.walk(dir)) {
stream.filter(path -> path.getFileName().toString().endsWith(".parquet")) stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
@@ -401,6 +471,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
.forEach(files::add); .forEach(files::add);
} }
} }
}
date = date.plusDays(1); date = date.plusDays(1);
} }
return files; return files;
@@ -434,8 +505,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
private Partition partition(EventFileRecord record) { private Partition partition(EventFileRecord record) {
return new Partition( return new Partition(
record.protocol(), record.protocol(),
LocalDate.ofInstant(record.eventTime(), partitionZone), LocalDate.ofInstant(record.eventTime(), partitionZone));
storageName(record.vin()));
} }
private Path partitionDir(ProtocolId protocol, LocalDate date) { private Path partitionDir(ProtocolId protocol, LocalDate date) {
@@ -488,11 +558,18 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
return vin.replaceAll("[^A-Za-z0-9._-]", "_"); return vin.replaceAll("[^A-Za-z0-9._-]", "_");
} }
private record Partition(ProtocolId protocol, LocalDate date, String vehicle) { private static String bucketName(String vin) {
String storageName = storageName(vin);
if ("_unknown".equals(storageName)) {
return "_unknown";
}
return "%02d".formatted(Math.floorMod(storageName.hashCode(), VEHICLE_BUCKETS));
}
private record Partition(ProtocolId protocol, LocalDate date) {
private Path dir(Path root) { private Path dir(Path root) {
return root.resolve("protocol=" + protocol.name()) return root.resolve("protocol=" + protocol.name())
.resolve("date=" + date) .resolve("date=" + date)
.resolve("vehicle=" + vehicle)
.resolve("header=" + HEADER_VERSION); .resolve("header=" + HEADER_VERSION);
} }
} }

View File

@@ -20,7 +20,10 @@ public record EventFileQuery(
Order order, Order order,
int limit, int limit,
String vin, String vin,
String eventType String eventType,
Instant cursorEventTime,
Instant cursorIngestTime,
String cursorEventId
) { ) {
public enum Order { public enum Order {
ASC, ASC,
@@ -45,6 +48,24 @@ public record EventFileQuery(
limit = limit <= 0 ? 100 : limit; limit = limit <= 0 ? 100 : limit;
vin = vin == null || vin.isBlank() ? null : vin.trim(); vin = vin == null || vin.isBlank() ? null : vin.trim();
eventType = eventType == null || eventType.isBlank() ? null : eventType.trim(); eventType = eventType == null || eventType.isBlank() ? null : eventType.trim();
cursorEventId = cursorEventId == null || cursorEventId.isBlank() ? null : cursorEventId.trim();
if ((cursorEventTime == null || cursorIngestTime == null || cursorEventId == null)
&& !(cursorEventTime == null && cursorIngestTime == null && cursorEventId == null)) {
throw new IllegalArgumentException("cursor eventTime, ingestTime and eventId must be provided together");
}
}
public EventFileQuery(ProtocolId protocol,
LocalDate dateFrom,
LocalDate dateTo,
Instant eventTimeFrom,
Instant eventTimeTo,
Order order,
int limit,
String vin,
String eventType) {
this(protocol, dateFrom, dateTo, eventTimeFrom, eventTimeTo, order, limit,
vin, eventType, null, null, null);
} }
public EventFileQuery(ProtocolId protocol, public EventFileQuery(ProtocolId protocol,

View File

@@ -226,9 +226,9 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
return payload; return payload;
} }
private static Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw, private Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
String rawArchiveKey, String rawArchiveKey,
String rawArchiveUri) { String rawArchiveUri) throws IOException {
Map<String, Object> payload = new LinkedHashMap<>(); Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventId", raw.eventId()); payload.put("eventId", raw.eventId());
payload.put("vin", raw.vin()); payload.put("vin", raw.vin());
@@ -241,10 +241,18 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
payload.put("command", hex(raw.command())); payload.put("command", hex(raw.command()));
payload.put("infoType", hex(raw.infoType())); payload.put("infoType", hex(raw.infoType()));
payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length); payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length);
putParsedJson(payload, raw.parsedJson());
payload.put("metadata", raw.metadata()); payload.put("metadata", raw.metadata());
return payload; return payload;
} }
private void putParsedJson(Map<String, Object> payload, String parsedJson) throws IOException {
if (parsedJson == null || parsedJson.isBlank()) {
return;
}
payload.put("parsed", objectMapper.readTree(parsedJson));
}
private static String hex(int value) { private static String hex(int value) {
return "0x" + String.format("%04X", value); return "0x" + String.format("%04X", value);
} }

View File

@@ -44,7 +44,8 @@ public class EventFileStoreAutoConfiguration {
return new DuckDbParquetEventFileStore( return new DuckDbParquetEventFileStore(
Path.of(properties.getPath()), Path.of(properties.getPath()),
ZoneId.of(properties.getZoneId()), ZoneId.of(properties.getZoneId()),
mapper); mapper,
properties.isWriteParquetEnabled());
} }
@Bean @Bean

View File

@@ -11,10 +11,10 @@ public class EventFileStoreProperties {
private boolean enabled = false; private boolean enabled = false;
/** /**
* Parquet 文件库根路径。 * 历史库根路径。
* *
* <p>实际结构为 {@code protocol=GB32960/date=yyyy-MM-dd/vehicle=VIN/header=event-records-v1/events.parquet} * <p>{@code events.duckdb} 是可查询主索引;开启 Parquet 副本时,还会在该根目录下按
* 另有 {@code events.duckdb} sidecar index 放在根路径下 * {@code protocol/date/header} 保存追加文件
*/ */
private String path = "./event-store"; private String path = "./event-store";
@@ -39,6 +39,14 @@ public class EventFileStoreProperties {
*/ */
private long flushIntervalMillis = 1000; private long flushIntervalMillis = 1000;
/**
* 是否同步写 Parquet 副本。
*
* <p>高写入生产链路可以关闭,只保留 DuckDB JSON 主索引和 raw archive 引用;需要离线副本或
* 索引重建能力时再开启。
*/
private boolean writeParquetEnabled = true;
public boolean isEnabled() { public boolean isEnabled() {
return enabled; return enabled;
} }
@@ -78,4 +86,12 @@ public class EventFileStoreProperties {
public void setFlushIntervalMillis(long flushIntervalMillis) { public void setFlushIntervalMillis(long flushIntervalMillis) {
this.flushIntervalMillis = flushIntervalMillis; this.flushIntervalMillis = flushIntervalMillis;
} }
public boolean isWriteParquetEnabled() {
return writeParquetEnabled;
}
public void setWriteParquetEnabled(boolean writeParquetEnabled) {
this.writeParquetEnabled = writeParquetEnabled;
}
} }

View File

@@ -1,6 +1,7 @@
package com.lingniu.ingest.eventfilestore; package com.lingniu.ingest.eventfilestore;
import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.api.ProtocolId;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.api.io.TempDir;
@@ -11,6 +12,7 @@ import java.time.LocalDate;
import java.time.ZoneId; import java.time.ZoneId;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.IntStream;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -28,7 +30,7 @@ class DuckDbParquetEventFileStoreTest {
store.appendAll(List.of(first, second)); store.appendAll(List.of(first, second));
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22"); Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(2); assertThat(parquetFiles(dateDir)).hasSize(1);
EventFileQuery ascending = new EventFileQuery( EventFileQuery ascending = new EventFileQuery(
ProtocolId.GB32960, ProtocolId.GB32960,
@@ -135,16 +137,13 @@ class DuckDbParquetEventFileStoreTest {
} }
@Test @Test
void appendsSameVehicleDayHeaderToOneParquetFile() throws Exception { void appendsSameVehicleDayHeaderAsAppendOnlyParquetFragments() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai")); EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME")); store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME"));
store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME")); store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960") Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
.resolve("date=2026-06-22") assertThat(parquetFiles(dateDir)).hasSize(2);
.resolve("vehicle=VIN-SAME")
.resolve("header=event-records-v1");
assertThat(parquetFiles(vehicleDayDir)).containsExactly(vehicleDayDir.resolve("events.parquet"));
EventFileQuery query = new EventFileQuery( EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960, ProtocolId.GB32960,
@@ -159,20 +158,55 @@ class DuckDbParquetEventFileStoreTest {
.containsExactly("first", "second"); .containsExactly("first", "second");
} }
@Test
void writesManyVehiclesIntoBoundedDailyLayoutInsteadOfVehicleDirectories() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
List<EventFileRecord> records = IntStream.range(0, 80)
.mapToObj(index -> rawRecord(
"raw-" + index,
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
"VIN-" + index))
.toList();
store.appendAll(records);
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
try (var files = Files.list(dateDir)) {
List<String> partitionDirs = files
.filter(Files::isDirectory)
.map(path -> path.getFileName().toString())
.sorted()
.toList();
assertThat(partitionDirs).noneMatch(name -> name.startsWith("vehicle="));
assertThat(partitionDirs).hasSizeLessThan(records.size());
}
}
@Test
void writesOneDailyParquetFragmentForOneBatchAcrossManyVehicles() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
List<EventFileRecord> records = IntStream.range(0, 80)
.mapToObj(index -> rawRecord(
"daily-raw-" + index,
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
"VIN-DAILY-" + index))
.toList();
store.appendAll(records);
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(1);
}
@Test @Test
void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception { void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai")); EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED")); store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960") Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
.resolve("date=2026-06-22") for (Path file : parquetFiles(dateDir)) {
.resolve("vehicle=VIN-INDEXED")
.resolve("header=event-records-v1");
try (var files = Files.list(vehicleDayDir)) {
for (Path file : files.filter(p -> p.getFileName().toString().endsWith(".parquet")).toList()) {
Files.delete(file); Files.delete(file);
} }
}
EventFileQuery query = new EventFileQuery( EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960, ProtocolId.GB32960,
@@ -219,6 +253,31 @@ class DuckDbParquetEventFileStoreTest {
assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri()); assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri());
} }
@Test
void canQueryVinRecordsFromIndexWhenParquetWritingIsDisabled() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(
tempDir,
ZoneId.of("Asia/Shanghai"),
new ObjectMapper(),
false);
EventFileRecord record = rawRecord("index-only", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEX-ONLY");
store.append(record);
assertThat(parquetFiles(tempDir)).isEmpty();
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
10,
"VIN-INDEX-ONLY",
"RAW_ARCHIVE");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("index-only");
}
private static List<Path> parquetFiles(Path root) throws Exception { private static List<Path> parquetFiles(Path root) throws Exception {
try (var files = Files.walk(root)) { try (var files = Files.walk(root)) {
return files.filter(p -> p.getFileName().toString().endsWith(".parquet")) return files.filter(p -> p.getFileName().toString().endsWith(".parquet"))

View File

@@ -20,12 +20,15 @@ class EventFileStoreAutoConfigurationTest {
contextRunner contextRunner
.withPropertyValues( .withPropertyValues(
"lingniu.ingest.event-file-store.enabled=true", "lingniu.ingest.event-file-store.enabled=true",
"lingniu.ingest.event-file-store.path=target/test-event-store") "lingniu.ingest.event-file-store.path=target/test-event-store",
"lingniu.ingest.event-file-store.write-parquet-enabled=false")
.run(context -> { .run(context -> {
assertThat(context).hasSingleBean(EventFileStore.class); assertThat(context).hasSingleBean(EventFileStore.class);
assertThat(context).hasSingleBean(EventFileStoreSink.class); assertThat(context).hasSingleBean(EventFileStoreSink.class);
assertThat(context.getBean(EventFileStoreProperties.class).getPath()) assertThat(context.getBean(EventFileStoreProperties.class).getPath())
.isEqualTo(Path.of("target/test-event-store").toString()); .isEqualTo(Path.of("target/test-event-store").toString());
assertThat(context.getBean(EventFileStoreProperties.class).isWriteParquetEnabled())
.isFalse();
}); });
} }

View File

@@ -92,6 +92,7 @@ public final class EnvelopeMapper {
.setUri(uri) .setUri(uri)
.setChecksum(checksum) .setChecksum(checksum)
.setSizeBytes(size) .setSizeBytes(size)
.setParsedJson(nullToEmpty(ra.parsedJson()))
.build()); .build());
b.setRawFrameFact(com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload.newBuilder() b.setRawFrameFact(com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload.newBuilder()
.setFrameId(frameId) .setFrameId(frameId)

View File

@@ -10,17 +10,19 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCloseable { public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(KafkaEnvelopeConsumerRunner.class); private static final Logger log = LoggerFactory.getLogger(KafkaEnvelopeConsumerRunner.class);
private final List<KafkaEnvelopeConsumerWorker> workers; private final Supplier<List<KafkaEnvelopeConsumerWorker>> workersSupplier;
private final Duration pollTimeout; private final Duration pollTimeout;
private final Duration loopBackoff; private final Duration loopBackoff;
private final boolean autoStartup; private final boolean autoStartup;
private final AtomicBoolean running = new AtomicBoolean(false); private final AtomicBoolean running = new AtomicBoolean(false);
private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicBoolean closed = new AtomicBoolean(false);
private volatile List<KafkaEnvelopeConsumerWorker> workers;
private ExecutorService executor; private ExecutorService executor;
public KafkaEnvelopeConsumerRunner(List<KafkaEnvelopeConsumerWorker> workers, public KafkaEnvelopeConsumerRunner(List<KafkaEnvelopeConsumerWorker> workers,
@@ -30,14 +32,28 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (workers == null || workers.isEmpty()) { if (workers == null || workers.isEmpty()) {
throw new IllegalArgumentException("workers must not be empty"); throw new IllegalArgumentException("workers must not be empty");
} }
this.workersSupplier = () -> List.copyOf(workers);
this.workers = List.copyOf(workers); this.workers = List.copyOf(workers);
this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout; this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout;
this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff; this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff;
this.autoStartup = autoStartup; this.autoStartup = autoStartup;
} }
public KafkaEnvelopeConsumerRunner(Supplier<List<KafkaEnvelopeConsumerWorker>> workersSupplier,
Duration pollTimeout,
Duration loopBackoff,
boolean autoStartup) {
if (workersSupplier == null) {
throw new IllegalArgumentException("workersSupplier must not be null");
}
this.workersSupplier = workersSupplier;
this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout;
this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff;
this.autoStartup = autoStartup;
}
public List<KafkaEnvelopeConsumerWorker> workers() { public List<KafkaEnvelopeConsumerWorker> workers() {
return workers; return workersOrCreate();
} }
@Override @Override
@@ -45,17 +61,35 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (!running.compareAndSet(false, true)) { if (!running.compareAndSet(false, true)) {
return; return;
} }
List<KafkaEnvelopeConsumerWorker> activeWorkers = workersOrCreate();
// 每个 worker 一条后台线程,避免某个处理器阻塞时拖慢其他消费组。 // 每个 worker 一条后台线程,避免某个处理器阻塞时拖慢其他消费组。
executor = Executors.newFixedThreadPool(workers.size(), r -> { executor = Executors.newFixedThreadPool(activeWorkers.size(), r -> {
Thread thread = new Thread(r, "kafka-envelope-consumer"); Thread thread = new Thread(r, "kafka-envelope-consumer");
thread.setDaemon(true); thread.setDaemon(true);
return thread; return thread;
}); });
for (KafkaEnvelopeConsumerWorker worker : workers) { for (KafkaEnvelopeConsumerWorker worker : activeWorkers) {
executor.submit(() -> pollLoop(worker)); executor.submit(() -> pollLoop(worker));
} }
} }
private List<KafkaEnvelopeConsumerWorker> workersOrCreate() {
List<KafkaEnvelopeConsumerWorker> current = workers;
if (current != null) {
return current;
}
synchronized (this) {
if (workers == null) {
List<KafkaEnvelopeConsumerWorker> created = workersSupplier.get();
if (created == null || created.isEmpty()) {
throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
}
workers = List.copyOf(created);
}
return workers;
}
}
private void pollLoop(KafkaEnvelopeConsumerWorker worker) { private void pollLoop(KafkaEnvelopeConsumerWorker worker) {
while (running.get()) { while (running.get()) {
try { try {
@@ -121,7 +155,11 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (!closed.compareAndSet(false, true)) { if (!closed.compareAndSet(false, true)) {
return; return;
} }
for (KafkaEnvelopeConsumerWorker worker : workers) { List<KafkaEnvelopeConsumerWorker> current = workers;
if (current == null) {
return;
}
for (KafkaEnvelopeConsumerWorker worker : current) {
worker.close(); worker.close();
} }
} }

View File

@@ -36,26 +36,26 @@ public final class KafkaEnvelopeConsumerWorker implements AutoCloseable {
public int pollOnce(Duration timeout) { public int pollOnce(Duration timeout) {
ConsumerRecords<String, byte[]> records = consumer.poll(timeout == null ? Duration.ZERO : timeout); ConsumerRecords<String, byte[]> records = consumer.poll(timeout == null ? Duration.ZERO : timeout);
Map<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> batches = new LinkedHashMap<>(); Map<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> byProcessor = new LinkedHashMap<>();
int processed = 0;
for (ConsumerRecord<String, byte[]> record : records) { for (ConsumerRecord<String, byte[]> record : records) {
EnvelopeConsumerProcessor processor = processorsByTopic.get(record.topic()); EnvelopeConsumerProcessor processor = processorsByTopic.get(record.topic());
if (processor == null) { if (processor == null) {
// worker 可能订阅多个 topic没有显式绑定处理器的 topic 不参与提交语义。 // worker 可能订阅多个 topic没有显式绑定处理器的 topic 不参与提交语义。
continue; continue;
} }
batches.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord( byProcessor.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord(
record.topic(), record.topic(),
record.partition(), record.partition(),
record.offset(), record.offset(),
record.key(), record.key(),
record.value())); record.value()));
processed++;
} }
int processed = 0; for (Map.Entry<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> entry : byProcessor.entrySet()) {
for (Map.Entry<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> entry : batches.entrySet()) {
// EnvelopeConsumerProcessor 内部会把解析或业务错误转成 DLQ 记录, // EnvelopeConsumerProcessor 内部会把解析或业务错误转成 DLQ 记录,
// 这里保持 Kafka worker 的职责单一:轮询、批量分发、成功后提交 offset。 // 这里保持 Kafka worker 的职责单一:轮询、分发、成功后提交 offset。
entry.getKey().processAll(entry.getValue()); entry.getKey().processBatch(entry.getValue());
processed += entry.getValue().size();
} }
if (processed > 0) { if (processed > 0) {
// commitSync 放在批次末尾,保证同一个 poll 批次内的消息按 Kafka offset 一起确认。 // commitSync 放在批次末尾,保证同一个 poll 批次内的消息按 Kafka offset 一起确认。

View File

@@ -1,6 +1,5 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import io.github.resilience4j.circuitbreaker.CircuitBreaker; import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig; import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
@@ -8,15 +7,12 @@ import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.serialization.ByteArraySerializer;
import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import java.time.Duration; import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
/** /**
@@ -100,21 +96,4 @@ public class SinkMqAutoConfiguration {
return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq()); return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq());
} }
@Bean
@ConditionalOnMissingBean
@ConditionalOnBean(EnvelopeConsumerProcessor.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(Map<String, EnvelopeConsumerProcessor> processors,
SinkMqProperties props) {
// Consumer runner 根据 processor bean 名和配置 binding 生成 worker没有 binding 时直接失败,避免静默不消费。
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
if (workers.isEmpty()) {
throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
}
return new KafkaEnvelopeConsumerRunner(
workers,
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
props.getConsumer().isAutoStartup());
}
} }

View File

@@ -0,0 +1,42 @@
package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import java.time.Duration;
import java.util.List;
import java.util.Map;
@AutoConfiguration(after = SinkMqAutoConfiguration.class)
@AutoConfigureAfter(name = {
"com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration",
"com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration",
"com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration"
})
@EnableConfigurationProperties(SinkMqProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
public class SinkMqConsumerAutoConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory,
SinkMqProperties props) {
return new KafkaEnvelopeConsumerRunner(
() -> createWorkers(beanFactory, props),
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
props.getConsumer().isAutoStartup());
}
private List<KafkaEnvelopeConsumerWorker> createWorkers(ListableBeanFactory beanFactory, SinkMqProperties props) {
Map<String, EnvelopeConsumerProcessor> processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class);
return new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
}
}

View File

@@ -45,6 +45,7 @@ message RawArchiveRef {
string uri = 1; string uri = 1;
string checksum = 2; string checksum = 2;
int64 size_bytes = 3; int64 size_bytes = 3;
string parsed_json = 4;
} }
message TelemetrySnapshot { message TelemetrySnapshot {

View File

@@ -1 +1,2 @@
com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration
com.lingniu.ingest.sink.mq.SinkMqConsumerAutoConfiguration

View File

@@ -1,6 +1,7 @@
package com.lingniu.ingest.sink.mq; package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord; import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord; import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
@@ -14,6 +15,7 @@ import java.time.Duration;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -44,4 +46,39 @@ class KafkaEnvelopeConsumerWorkerTest {
assertThat(record.payload()).containsExactly(0x01, 0x02); assertThat(record.payload()).containsExactly(0x01, 0x02);
}); });
} }
@Test
void pollsKafkaRecordsAndDispatchesBatchWhenProcessorSupportsIt() {
MockConsumer<String, byte[]> consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST);
TopicPartition partition = new TopicPartition("vehicle.raw", 0);
consumer.assign(List.of(partition));
consumer.updateBeginningOffsets(Map.of(partition, 0L));
consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 12L, "VIN001", new byte[]{0x01}));
consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 13L, "VIN002", new byte[]{0x02}));
AtomicInteger batchCalls = new AtomicInteger();
EnvelopeConsumerProcessor processor = new EnvelopeConsumerProcessor(
"event-history",
new EnvelopeBatchIngestor() {
@Override
public EnvelopeIngestResult tryIngest(byte[] kafkaValue) {
throw new AssertionError("single-record ingest should not be used for a batch-capable processor");
}
@Override
public List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues) {
batchCalls.incrementAndGet();
assertThat(kafkaValues).hasSize(2);
return List.of(
EnvelopeIngestResult.stored("event-1", "VIN001"),
EnvelopeIngestResult.invalid("bad envelope"));
}
},
record -> {});
int processed = new KafkaEnvelopeConsumerWorker(
consumer, Map.of("vehicle.raw", processor)).pollOnce(Duration.ZERO);
assertThat(processed).isEqualTo(2);
assertThat(batchCalls).hasValue(1);
}
} }

View File

@@ -10,7 +10,7 @@ import static org.assertj.core.api.Assertions.assertThat;
class SinkMqConsumerAutoConfigurationTest { class SinkMqConsumerAutoConfigurationTest {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(SinkMqAutoConfiguration.class) .withUserConfiguration(SinkMqAutoConfiguration.class, SinkMqConsumerAutoConfiguration.class)
.withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class, .withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
() -> new EnvelopeConsumerProcessor( () -> new EnvelopeConsumerProcessor(
"vehicle-state", "vehicle-state",

View File

@@ -43,6 +43,7 @@ public final class TdengineEnvelopeRows {
raw.getParseError(), raw.getParseError(),
raw.getPeer(), raw.getPeer(),
json(metadata), json(metadata),
parsedJson(envelope, metadata),
protocol(envelope), protocol(envelope),
vehicleKey(envelope, raw.getVehicleKey(), raw.getPhone()), vehicleKey(envelope, raw.getVehicleKey(), raw.getPhone()),
firstNonBlank(raw.getVin(), envelope.getVin()), firstNonBlank(raw.getVin(), envelope.getVin()),
@@ -73,7 +74,6 @@ public final class TdengineEnvelopeRows {
location.getStatusFlag(), location.getStatusFlag(),
totalMileage(envelope), totalMileage(envelope),
rawUri, rawUri,
json(envelope.getMetadataMap()),
protocol(envelope), protocol(envelope),
vehicleKey(envelope, envelope.getMetadataOrDefault("vehicle_key", ""), vehicleKey(envelope, envelope.getMetadataOrDefault("vehicle_key", ""),
envelope.getMetadataOrDefault("phone", "")), envelope.getMetadataOrDefault("phone", "")),
@@ -188,6 +188,13 @@ public final class TdengineEnvelopeRows {
return Double.parseDouble(value); return Double.parseDouble(value);
} }
private static String parsedJson(VehicleEnvelope envelope, Map<String, String> metadata) {
if (envelope.hasRawArchive() && !envelope.getRawArchive().getParsedJson().isBlank()) {
return envelope.getRawArchive().getParsedJson();
}
return firstNonBlank(metadata.get("parsedJson"), metadata.get("parsed_json"));
}
private static Double valueDouble(TelemetryField field) { private static Double valueDouble(TelemetryField field) {
String valueType = field.getValueType(); String valueType = field.getValueType();
if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) { if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) {

View File

@@ -6,11 +6,11 @@ import java.util.List;
public final class TdengineHistoryQueries { public final class TdengineHistoryQueries {
private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, " private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, " + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json, "
+ "protocol, vehicle_key, vin, phone"; + "protocol, vehicle_key, vin, phone";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, " private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, " + "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone"; + "protocol, vehicle_key, vin, phone";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, " private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, "
+ "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, " + "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone"; + "metadata_json, protocol, vehicle_key, vin, phone";

View File

@@ -64,7 +64,8 @@ public final class TdengineHistorySchema {
parse_status NCHAR(16), parse_status NCHAR(16),
parse_error NCHAR(512), parse_error NCHAR(512),
peer NCHAR(128), peer NCHAR(128),
metadata_json NCHAR(4096) metadata_json NCHAR(4096),
parsed_json NCHAR(16374)
) TAGS ( ) TAGS (
protocol NCHAR(16), protocol NCHAR(16),
vehicle_key NCHAR(128), vehicle_key NCHAR(128),
@@ -88,8 +89,7 @@ public final class TdengineHistorySchema {
alarm_flag BIGINT, alarm_flag BIGINT,
status_flag BIGINT, status_flag BIGINT,
total_mileage_km DOUBLE, total_mileage_km DOUBLE,
raw_uri NCHAR(512), raw_uri NCHAR(512)
metadata_json NCHAR(4096)
) TAGS ( ) TAGS (
protocol NCHAR(16), protocol NCHAR(16),
vehicle_key NCHAR(128), vehicle_key NCHAR(128),

View File

@@ -6,12 +6,12 @@ import java.util.Arrays;
public final class TdengineHistoryStatements { public final class TdengineHistoryStatements {
private static final String RAW_FRAME_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, " private static final String RAW_FRAME_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json"; + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json";
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?"; private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, " private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, metadata_json"; + "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri";
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?"; private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, " private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, "
+ "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json"; + "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json";
@@ -35,7 +35,7 @@ public final class TdengineHistoryStatements {
values( values(
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(), row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(), row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
row.parseError(), row.peer(), row.metadataJson() row.parseError(), row.peer(), row.metadataJson(), row.parsedJson()
) )
); );
} }
@@ -49,7 +49,7 @@ public final class TdengineHistoryStatements {
values( values(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(), row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(), row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
row.totalMileageKm(), row.rawUri(), row.metadataJson() row.totalMileageKm(), row.rawUri()
) )
); );
} }

View File

@@ -157,6 +157,7 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
rs.getString("parse_error"), rs.getString("parse_error"),
rs.getString("peer"), rs.getString("peer"),
rs.getString("metadata_json"), rs.getString("metadata_json"),
rs.getString("parsed_json"),
rs.getString("protocol"), rs.getString("protocol"),
rs.getString("vehicle_key"), rs.getString("vehicle_key"),
rs.getString("vin"), rs.getString("vin"),
@@ -179,7 +180,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
rs.getLong("status_flag"), rs.getLong("status_flag"),
nullableDouble(rs, "total_mileage_km"), nullableDouble(rs, "total_mileage_km"),
rs.getString("raw_uri"), rs.getString("raw_uri"),
rs.getString("metadata_json"),
rs.getString("protocol"), rs.getString("protocol"),
rs.getString("vehicle_key"), rs.getString("vehicle_key"),
rs.getString("vin"), rs.getString("vin"),

View File

@@ -16,7 +16,6 @@ public record TdengineLocationRow(
long statusFlag, long statusFlag,
Double totalMileageKm, Double totalMileageKm,
String rawUri, String rawUri,
String metadataJson,
String protocol, String protocol,
String vehicleKey, String vehicleKey,
String vin, String vin,

View File

@@ -16,6 +16,7 @@ public record TdengineRawFrameRow(
String parseError, String parseError,
String peer, String peer,
String metadataJson, String metadataJson,
String parsedJson,
String protocol, String protocol,
String vehicleKey, String vehicleKey,
String vin, String vin,

View File

@@ -27,6 +27,7 @@ class TdengineHistoryStatementsTest {
"", "",
"10.0.0.1:808", "10.0.0.1:808",
"{\"auth\":\"passed\"}", "{\"auth\":\"passed\"}",
"{\"body\":{\"speedKmh\":42.0}}",
"JT808", "JT808",
"jt808:g7gps", "jt808:g7gps",
"VIN123", "VIN123",
@@ -39,13 +40,13 @@ class TdengineHistoryStatementsTest {
.contains(" USING raw_frames TAGS ('JT808', 'jt808:g7gps', 'VIN123', '013800000000')"); .contains(" USING raw_frames TAGS ('JT808', 'jt808:g7gps', 'VIN123', '013800000000')");
assertThat(batch.insertSql()) assertThat(batch.insertSql())
.startsWith("INSERT INTO raw_jt808_") .startsWith("INSERT INTO raw_jt808_")
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json)") .contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json)")
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); .endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
assertThat(batch.values()) assertThat(batch.values())
.containsExactly( .containsExactly(
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(), row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(), row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
row.parseError(), row.peer(), row.metadataJson()); row.parseError(), row.peer(), row.metadataJson(), row.parsedJson());
} }
@Test @Test
@@ -64,7 +65,6 @@ class TdengineHistoryStatementsTest {
3, 3,
null, null,
"archive://gb32960/frame-1.bin", "archive://gb32960/frame-1.bin",
"{}",
"GB32960", "GB32960",
"vin:O'HARE", "vin:O'HARE",
"O'HARE", "O'HARE",
@@ -79,7 +79,7 @@ class TdengineHistoryStatementsTest {
.containsExactly( .containsExactly(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(), row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(), row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
row.totalMileageKm(), row.rawUri(), row.metadataJson()); row.totalMileageKm(), row.rawUri());
} }
@Test @Test

View File

@@ -74,7 +74,6 @@ class TdengineJdbcHistoryWriterTest {
3, 3,
null, null,
"archive://gb32960/frame-1.bin", "archive://gb32960/frame-1.bin",
"{}",
"GB32960", "GB32960",
"vin:VIN123", "vin:VIN123",
"VIN123", "VIN123",
@@ -217,6 +216,7 @@ class TdengineJdbcHistoryWriterTest {
parseError, parseError,
"10.0.0.1:808", "10.0.0.1:808",
"{}", "{}",
"{\"frame\":\"" + frameId + "\"}",
"JT808", "JT808",
"jt808:g7gps", "jt808:g7gps",
"VIN123", "VIN123",

View File

@@ -65,6 +65,7 @@
<resilience4j.version>2.2.0</resilience4j.version> <resilience4j.version>2.2.0</resilience4j.version>
<micrometer.version>1.13.6</micrometer.version> <micrometer.version>1.13.6</micrometer.version>
<otel.version>1.42.1</otel.version> <otel.version>1.42.1</otel.version>
<paho-mqtt.version>1.2.5</paho-mqtt.version>
<fusesource-mqtt.version>1.12</fusesource-mqtt.version> <fusesource-mqtt.version>1.12</fusesource-mqtt.version>
<duckdb.version>1.1.3</duckdb.version> <duckdb.version>1.1.3</duckdb.version>
<taos-jdbcdriver.version>3.8.4</taos-jdbcdriver.version> <taos-jdbcdriver.version>3.8.4</taos-jdbcdriver.version>
@@ -306,6 +307,11 @@
<artifactId>kafka-clients</artifactId> <artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version> <version>${kafka.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.eclipse.paho</groupId>
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
<version>${paho-mqtt.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.fusesource.mqtt-client</groupId> <groupId>org.fusesource.mqtt-client</groupId>
<artifactId>mqtt-client</artifactId> <artifactId>mqtt-client</artifactId>