feat: productionize raw history ingestion
This commit is contained in:
@@ -10,6 +10,6 @@ ENV JAVA_OPTS=""
|
||||
WORKDIR /app
|
||||
COPY modules/apps/${APP_NAME}/target/${APP_NAME}.jar /app/app.jar
|
||||
|
||||
EXPOSE 20100 20200 20300 20400 32960 10808
|
||||
EXPOSE 20100 20200 20300 20400 20500 20600 32960 10808
|
||||
|
||||
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app/app.jar"]
|
||||
|
||||
@@ -4,7 +4,7 @@ version: "3.8"
|
||||
# main-0.1.0-SNAPSHOT-22468e7
|
||||
x-common-env: &common-env
|
||||
TZ: Asia/Shanghai
|
||||
JAVA_OPTS: ${JAVA_OPTS:--Xms128m -Xmx768m -Duser.timezone=Asia/Shanghai}
|
||||
JAVA_OPTS: ${JAVA_OPTS:--XX:+UseZGC -XX:MaxRAMPercentage=75}
|
||||
NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true}
|
||||
NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848}
|
||||
NACOS_NAMESPACE: ${NACOS_NAMESPACE:-}
|
||||
@@ -13,32 +13,33 @@ x-common-env: &common-env
|
||||
NACOS_REFRESH_ENABLED: ${NACOS_REFRESH_ENABLED:-true}
|
||||
NACOS_USERNAME: ${NACOS_USERNAME:-}
|
||||
NACOS_PASSWORD: ${NACOS_PASSWORD:-}
|
||||
KAFKA_BROKERS: ${KAFKA_BROKERS:-114.55.58.251:9092}
|
||||
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
|
||||
KAFKA_ENABLED: ${KAFKA_ENABLED:-true}
|
||||
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true}
|
||||
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:-900000}
|
||||
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
|
||||
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
|
||||
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
|
||||
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_XINDA_PUSH_EVENT: ${KAFKA_TOPIC_XINDA_PUSH_EVENT:-vehicle.event.xinda-push.v1}
|
||||
KAFKA_TOPIC_XINDA_PUSH_RAW: ${KAFKA_TOPIC_XINDA_PUSH_RAW:-vehicle.raw.xinda-push.v1}
|
||||
KAFKA_TOPIC_XINDA_PUSH_DLQ: ${KAFKA_TOPIC_XINDA_PUSH_DLQ:-vehicle.dlq.xinda-push.v1}
|
||||
KAFKA_TOPIC_MQTT_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_MQTT_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_MQTT_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_XINDA_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1}
|
||||
KAFKA_TOPIC_XINDA_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1}
|
||||
KAFKA_TOPIC_XINDA_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1}
|
||||
MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
|
||||
services:
|
||||
gb32960-ingest-app:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: gb32960-ingest-app
|
||||
restart: unless-stopped
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
environment:
|
||||
<<: *common-env
|
||||
JAVA_OPTS: ${JAVA_OPTS_GB32960:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
|
||||
HTTP_PORT: 20100
|
||||
GB32960_PORT: 32960
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
@@ -48,13 +49,11 @@ services:
|
||||
GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}
|
||||
GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}
|
||||
GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false}
|
||||
SINK_ARCHIVE_PATH: /archive/
|
||||
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
|
||||
ports:
|
||||
- "${GB32960_HTTP_PORT:-20100}:20100"
|
||||
- "${GB32960_TCP_PORT:-32960}:32960"
|
||||
volumes:
|
||||
- vehicle-history-archive:/archive
|
||||
- gb32960-ingest-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
@@ -63,109 +62,82 @@ services:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: jt808-ingest-app
|
||||
restart: unless-stopped
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
environment:
|
||||
<<: *common-env
|
||||
JAVA_OPTS: ${JAVA_OPTS_JT808:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
|
||||
HTTP_PORT: 20482
|
||||
HTTP_PORT: 20400
|
||||
JT808_PORT: 808
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer}
|
||||
SINK_ARCHIVE_PATH: /archive/
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
|
||||
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
|
||||
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
|
||||
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
ports:
|
||||
- "${JT808_HTTP_PORT:-20482}:20482"
|
||||
- "${JT808_HTTP_PORT:-20400}:20400"
|
||||
- "${JT808_TCP_PORT:-808}:808"
|
||||
volumes:
|
||||
- vehicle-history-archive:/archive
|
||||
- jt808-ingest-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
|
||||
yutong-mqtt-app:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: yutong-mqtt-app
|
||||
restart: unless-stopped
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
environment:
|
||||
<<: *common-env
|
||||
JAVA_OPTS: ${JAVA_OPTS_YUTONG_MQTT:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
|
||||
HTTP_PORT: 20500
|
||||
MQTT_ENABLED: "true"
|
||||
XINDA_PUSH_ENABLED: "false"
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_YUTONG_MQTT:-yutong-mqtt-portainer}
|
||||
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
|
||||
YUTONG_MQTT_ENDPOINT_NAME: ${YUTONG_MQTT_ENDPOINT_NAME:-yutong}
|
||||
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
|
||||
YUTONG_MQTT_TOPIC: ${YUTONG_MQTT_TOPIC:-#}
|
||||
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-1}
|
||||
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-yutong-mqtt}
|
||||
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
|
||||
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
|
||||
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
|
||||
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
|
||||
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
|
||||
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
|
||||
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
|
||||
SINK_ARCHIVE_PATH: /archive/
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
|
||||
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
|
||||
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
|
||||
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_MQTT:-yutong-mqtt-portainer}
|
||||
KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
|
||||
KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
MQTT_ENDPOINT_NAME: ${MQTT_ENDPOINT_NAME:-yutong}
|
||||
MQTT_URI: ${MQTT_URI:-tcp://127.0.0.1:1883}
|
||||
MQTT_TOPIC: "${MQTT_TOPIC:-#}"
|
||||
MQTT_QOS: ${MQTT_QOS:-1}
|
||||
MQTT_CLIENT_ID: ${MQTT_CLIENT_ID:-lingniu-yutong-mqtt}
|
||||
MQTT_USERNAME: ${MQTT_USERNAME:-}
|
||||
MQTT_PASSWORD: ${MQTT_PASSWORD:-}
|
||||
MQTT_CLEAN_SESSION: ${MQTT_CLEAN_SESSION:-true}
|
||||
MQTT_PROFILE: ${MQTT_PROFILE:-yutong}
|
||||
ports:
|
||||
- "${YUTONG_MQTT_HTTP_PORT:-20500}:20500"
|
||||
volumes:
|
||||
- vehicle-history-archive:/archive
|
||||
- yutong-mqtt-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
|
||||
xinda-push-app:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/xinda-push-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: xinda-push-app
|
||||
restart: unless-stopped
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
environment:
|
||||
<<: *common-env
|
||||
JAVA_OPTS: ${JAVA_OPTS_XINDA_PUSH:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
|
||||
HTTP_PORT: 20600
|
||||
MQTT_ENABLED: "false"
|
||||
XINDA_PUSH_ENABLED: "true"
|
||||
KAFKA_CONSUMER_ENABLED: "false"
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA_PUSH:-xinda-push-portainer}
|
||||
XINDA_PUSH_ENABLED: ${XINDA_PUSH_ENABLED:-false}
|
||||
KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA:-xinda-push-portainer}
|
||||
KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1}
|
||||
KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1}
|
||||
KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1}
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
|
||||
XINDA_PUSH_HOST: ${XINDA_PUSH_HOST:-}
|
||||
XINDA_PUSH_PORT: ${XINDA_PUSH_PORT:-10100}
|
||||
XINDA_PUSH_USERNAME: ${XINDA_PUSH_USERNAME:-}
|
||||
XINDA_PUSH_PASSWORD: ${XINDA_PUSH_PASSWORD:-}
|
||||
XINDA_PUSH_CLIENT_DESC: ${XINDA_PUSH_CLIENT_DESC:-lingniu-ingest}
|
||||
XINDA_PUSH_MSG_ID_LOCATION: ${XINDA_PUSH_MSG_ID_LOCATION:-0200}
|
||||
XINDA_PUSH_MSG_ID_ALARM: ${XINDA_PUSH_MSG_ID_ALARM:-0300}
|
||||
XINDA_PUSH_MSG_ID_PASSTHROUGH: ${XINDA_PUSH_MSG_ID_PASSTHROUGH:-0401}
|
||||
XINDA_PUSH_RECONNECT_INTERVAL_SEC: ${XINDA_PUSH_RECONNECT_INTERVAL_SEC:-5}
|
||||
SINK_ARCHIVE_PATH: /archive/
|
||||
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
|
||||
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
|
||||
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
|
||||
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
|
||||
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
|
||||
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
|
||||
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
|
||||
XINDA_PUSH_SUBSCRIBE_MSG_IDS: "${XINDA_PUSH_SUBSCRIBE_MSG_IDS:-0200,0300,0401}"
|
||||
ports:
|
||||
- "${XINDA_PUSH_HTTP_PORT:-20600}:20600"
|
||||
volumes:
|
||||
- vehicle-history-archive:/archive
|
||||
- xinda-push-data:/data
|
||||
networks:
|
||||
- vehicle-ingest
|
||||
|
||||
@@ -173,8 +145,6 @@ services:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: vehicle-history-app
|
||||
restart: unless-stopped
|
||||
mem_limit: 2560m
|
||||
memswap_limit: 2560m
|
||||
depends_on:
|
||||
- gb32960-ingest-app
|
||||
- jt808-ingest-app
|
||||
@@ -182,29 +152,28 @@ services:
|
||||
- xinda-push-app
|
||||
environment:
|
||||
<<: *common-env
|
||||
JAVA_OPTS: ${JAVA_OPTS_HISTORY:--Xms512m -Xmx1536m -Duser.timezone=Asia/Shanghai}
|
||||
HTTP_PORT: 20200
|
||||
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history}
|
||||
KAFKA_CONSUMER_MAX_POLL_RECORDS: ${KAFKA_CONSUMER_MAX_POLL_RECORDS_HISTORY:-2000}
|
||||
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
|
||||
KAFKA_CONSUMER_CONCURRENCY: ${KAFKA_CONSUMER_CONCURRENCY:-3}
|
||||
KAFKA_GROUP_HISTORY_GB32960: ${KAFKA_GROUP_HISTORY_GB32960:-vehicle-history}
|
||||
KAFKA_GROUP_HISTORY_JT808: ${KAFKA_GROUP_HISTORY_JT808:-vehicle-history}
|
||||
KAFKA_GROUP_HISTORY_MQTT_YUTONG: ${KAFKA_GROUP_HISTORY_MQTT_YUTONG:-vehicle-history}
|
||||
KAFKA_GROUP_HISTORY_XINDA: ${KAFKA_GROUP_HISTORY_XINDA:-vehicle-history}
|
||||
SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true}
|
||||
SINK_ARCHIVE_PATH: /archive/
|
||||
# 高吞吐生产热路径默认只写 TDengine。需要 GB32960 snapshot/fields 旧查询时再显式开启。
|
||||
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-false}
|
||||
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true}
|
||||
EVENT_FILE_STORE_PATH: /event-store/
|
||||
EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai}
|
||||
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}
|
||||
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}
|
||||
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://tdengine:6041/vehicle_ts}
|
||||
TDENGINE_DRIVER_CLASS_NAME: ${TDENGINE_DRIVER_CLASS_NAME:-com.taosdata.jdbc.ws.WebSocketDriver}
|
||||
EVENT_FILE_STORE_BATCH_SIZE: ${EVENT_FILE_STORE_BATCH_SIZE:-500}
|
||||
EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS: ${EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS:-1000}
|
||||
EVENT_FILE_STORE_WRITE_PARQUET_ENABLED: ${EVENT_FILE_STORE_WRITE_PARQUET_ENABLED:-false}
|
||||
TDENGINE_ENABLED: ${TDENGINE_ENABLED:-true}
|
||||
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts}
|
||||
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-vehicle_ts}
|
||||
TDENGINE_JT808_LOCATION_STABLE: ${TDENGINE_JT808_LOCATION_STABLE:-jt808_locations}
|
||||
TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root}
|
||||
TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata}
|
||||
TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32}
|
||||
TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-0}
|
||||
TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-5000}
|
||||
TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:--1}
|
||||
# 默认关闭逐字段宽表,避免 telemetry_fields 写放大;需要字段趋势查询时显式打开。
|
||||
TDENGINE_TELEMETRY_FIELDS_ENABLED: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:-false}
|
||||
ports:
|
||||
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
|
||||
volumes:
|
||||
@@ -217,16 +186,11 @@ services:
|
||||
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
|
||||
container_name: vehicle-analytics-app
|
||||
restart: unless-stopped
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
depends_on:
|
||||
- gb32960-ingest-app
|
||||
- jt808-ingest-app
|
||||
- yutong-mqtt-app
|
||||
- xinda-push-app
|
||||
environment:
|
||||
<<: *common-env
|
||||
JAVA_OPTS: ${JAVA_OPTS_ANALYTICS:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
|
||||
HTTP_PORT: 20300
|
||||
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics}
|
||||
KAFKA_GROUP_STATE: ${KAFKA_GROUP_STATE:-vehicle-state}
|
||||
@@ -244,6 +208,7 @@ services:
|
||||
REDIS_HOST: ${REDIS_HOST:-}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DATABASE: ${REDIS_DATABASE:-50}
|
||||
REDIS_USERNAME: ${REDIS_USERNAME:-}
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
|
||||
ports:
|
||||
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300"
|
||||
@@ -259,8 +224,6 @@ networks:
|
||||
volumes:
|
||||
gb32960-ingest-data:
|
||||
jt808-ingest-data:
|
||||
yutong-mqtt-data:
|
||||
xinda-push-data:
|
||||
vehicle-history-archive:
|
||||
vehicle-history-event-store:
|
||||
vehicle-analytics-stat:
|
||||
|
||||
92
deploy/tools/EnsureKafkaTopics.java
Normal file
92
deploy/tools/EnsureKafkaTopics.java
Normal file
@@ -0,0 +1,92 @@
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.clients.admin.AdminClientConfig;
|
||||
import org.apache.kafka.clients.admin.NewPartitions;
|
||||
import org.apache.kafka.clients.admin.NewTopic;
|
||||
import org.apache.kafka.clients.admin.TopicDescription;
|
||||
|
||||
public final class EnsureKafkaTopics {
|
||||
private static final short REPLICATION_FACTOR = 1;
|
||||
|
||||
private record TopicSpec(String name, int partitions) {}
|
||||
|
||||
private EnsureKafkaTopics() {}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String brokers = args.length > 0 ? args[0] : "172.17.111.56:9092";
|
||||
List<TopicSpec> specs =
|
||||
List.of(
|
||||
topic("vehicle.event.gb32960.v1", 12),
|
||||
topic("vehicle.raw.gb32960.v1", 12),
|
||||
topic("vehicle.dlq.gb32960.v1", 3),
|
||||
topic("vehicle.event.jt808.v1", 12),
|
||||
topic("vehicle.raw.jt808.v1", 12),
|
||||
topic("vehicle.dlq.jt808.v1", 3),
|
||||
topic("vehicle.event.mqtt-yutong.v1", 12),
|
||||
topic("vehicle.raw.mqtt-yutong.v1", 12),
|
||||
topic("vehicle.dlq.mqtt-yutong.v1", 3),
|
||||
topic("vehicle.event.xinda.v1", 12),
|
||||
topic("vehicle.raw.xinda.v1", 12),
|
||||
topic("vehicle.dlq.xinda.v1", 3),
|
||||
topic("vehicle.event.xinda-push.v1", 12),
|
||||
topic("vehicle.raw.xinda-push.v1", 12),
|
||||
topic("vehicle.dlq.xinda-push.v1", 3));
|
||||
|
||||
Properties props = new Properties();
|
||||
props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);
|
||||
props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "15000");
|
||||
props.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "30000");
|
||||
|
||||
try (AdminClient admin = AdminClient.create(props)) {
|
||||
Set<String> existing = admin.listTopics().names().get();
|
||||
List<NewTopic> missing =
|
||||
specs.stream()
|
||||
.filter(spec -> !existing.contains(spec.name()))
|
||||
.map(spec -> new NewTopic(spec.name(), spec.partitions(), REPLICATION_FACTOR))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
try {
|
||||
admin.createTopics(missing).all().get();
|
||||
} catch (ExecutionException e) {
|
||||
System.out.println("createTopics result=" + e.getCause());
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, TopicSpec> byName =
|
||||
specs.stream().collect(Collectors.toMap(TopicSpec::name, spec -> spec));
|
||||
Map<String, TopicDescription> descriptions =
|
||||
admin.describeTopics(byName.keySet()).allTopicNames().get();
|
||||
|
||||
Map<String, NewPartitions> increases =
|
||||
descriptions.entrySet().stream()
|
||||
.filter(entry -> entry.getValue().partitions().size() < byName.get(entry.getKey()).partitions())
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Map.Entry::getKey,
|
||||
entry -> NewPartitions.increaseTo(byName.get(entry.getKey()).partitions())));
|
||||
if (!increases.isEmpty()) {
|
||||
admin.createPartitions(increases).all().get();
|
||||
Thread.sleep(Duration.ofSeconds(2).toMillis());
|
||||
descriptions = admin.describeTopics(byName.keySet()).allTopicNames().get();
|
||||
}
|
||||
|
||||
List<String> lines = new ArrayList<>();
|
||||
descriptions.entrySet().stream()
|
||||
.sorted(Map.Entry.comparingByKey())
|
||||
.forEach(entry -> lines.add(entry.getKey() + " partitions=" + entry.getValue().partitions().size()));
|
||||
System.out.println("bootstrap=" + brokers);
|
||||
lines.forEach(System.out::println);
|
||||
}
|
||||
}
|
||||
|
||||
private static TopicSpec topic(String name, int partitions) {
|
||||
return new TopicSpec(name, partitions);
|
||||
}
|
||||
}
|
||||
@@ -52,12 +52,11 @@ lingniu:
|
||||
file:
|
||||
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
|
||||
mysql:
|
||||
table: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}
|
||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}
|
||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:}
|
||||
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
|
||||
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
|
||||
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
|
||||
driver-class-name: ${VEHICLE_IDENTITY_MYSQL_DRIVER_CLASS_NAME:com.mysql.cj.jdbc.Driver}
|
||||
refresh-interval: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}
|
||||
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
|
||||
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
|
||||
sink:
|
||||
mq:
|
||||
enabled: ${KAFKA_ENABLED:true}
|
||||
@@ -79,10 +78,6 @@ lingniu:
|
||||
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
|
||||
consumer:
|
||||
enabled: false
|
||||
archive:
|
||||
enabled: ${SINK_ARCHIVE_ENABLED:true}
|
||||
type: local
|
||||
path: ${SINK_ARCHIVE_PATH:./archive/}
|
||||
event-file-store:
|
||||
enabled: false
|
||||
event-history:
|
||||
|
||||
@@ -29,12 +29,14 @@ class Jt808IngestAppDefaultsTest {
|
||||
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
|
||||
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.table", "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url", "${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.refresh-interval", "${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}")
|
||||
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.table-name",
|
||||
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url",
|
||||
"${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:root}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.password", "${VEHICLE_IDENTITY_MYSQL_PASSWORD:}")
|
||||
.containsEntry("lingniu.ingest.identity.mysql.initialize-schema",
|
||||
"${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}")
|
||||
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
|
||||
.containsEntry("lingniu.ingest.event-history.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
|
||||
|
||||
@@ -33,6 +33,10 @@
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>vehicle-stat-service</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
|
||||
@@ -26,6 +26,7 @@ spring:
|
||||
redis:
|
||||
host: ${REDIS_HOST:}
|
||||
port: ${REDIS_PORT:6379}
|
||||
username: ${REDIS_USERNAME:}
|
||||
password: ${REDIS_PASSWORD:}
|
||||
database: ${REDIS_DATABASE:50}
|
||||
|
||||
@@ -55,6 +56,7 @@ lingniu:
|
||||
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics}
|
||||
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
|
||||
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:500}
|
||||
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:900000}
|
||||
bindings:
|
||||
vehicleStateEnvelopeConsumerProcessor:
|
||||
enabled: ${VEHICLE_STATE_ENABLED:false}
|
||||
|
||||
@@ -46,7 +46,7 @@ lingniu:
|
||||
enabled: ${KAFKA_CONSUMER_ENABLED:true}
|
||||
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history}
|
||||
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
|
||||
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}
|
||||
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}
|
||||
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}
|
||||
concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3}
|
||||
bindings:
|
||||
|
||||
@@ -43,7 +43,7 @@ class VehicleHistoryAppDefaultsTest {
|
||||
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
|
||||
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
|
||||
.containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
|
||||
.containsEntry(
|
||||
"lingniu.ingest.sink.mq.consumer.max-poll-interval-millis",
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.lingniu.ingest.api.consumer;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Optional extension for consumers that can persist a Kafka poll batch in one
|
||||
* storage operation.
|
||||
*/
|
||||
public interface EnvelopeBatchIngestor extends EnvelopeIngestor {
|
||||
|
||||
List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues);
|
||||
}
|
||||
@@ -70,6 +70,32 @@ public final class EnvelopeConsumerProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
public List<EnvelopeIngestResult> processBatch(List<EnvelopeConsumerRecord> records) {
|
||||
if (records == null || records.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
if (!(ingestor instanceof EnvelopeBatchIngestor batchIngestor)) {
|
||||
return records.stream().map(this::process).toList();
|
||||
}
|
||||
List<byte[]> payloads = records.stream()
|
||||
.map(EnvelopeConsumerRecord::payload)
|
||||
.toList();
|
||||
List<EnvelopeIngestResult> results = batchIngestor.tryIngestAll(payloads);
|
||||
if (results.size() != records.size()) {
|
||||
throw new IllegalStateException("batch ingestor returned " + results.size()
|
||||
+ " results for " + records.size() + " records");
|
||||
}
|
||||
List<EnvelopeIngestResult> copy = new ArrayList<>(results.size());
|
||||
for (int i = 0; i < records.size(); i++) {
|
||||
EnvelopeIngestResult result = results.get(i);
|
||||
if (DEAD_LETTER_STATUSES.contains(result.status())) {
|
||||
deadLetterSink.publish(toDeadLetter(records.get(i), result));
|
||||
}
|
||||
copy.add(result);
|
||||
}
|
||||
return List.copyOf(copy);
|
||||
}
|
||||
|
||||
private EnvelopeDeadLetterRecord toDeadLetter(EnvelopeConsumerRecord record, EnvelopeIngestResult result) {
|
||||
return new EnvelopeDeadLetterRecord(
|
||||
service,
|
||||
|
||||
@@ -19,6 +19,7 @@ public final class RawArchiveKeys {
|
||||
public static final String META_EVENT_ID = "rawArchiveEventId";
|
||||
public static final String META_KEY = "rawArchiveKey";
|
||||
public static final String META_URI = "rawArchiveUri";
|
||||
public static final String META_PARSED_JSON = "rawArchiveParsedJson";
|
||||
|
||||
private static final DateTimeFormatter DATE_KEY =
|
||||
DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneId.of("Asia/Shanghai"));
|
||||
|
||||
@@ -150,6 +150,7 @@ public sealed interface VehicleEvent
|
||||
* @param command 协议主命令码(如 32960 0x02/0x03 等),冗余在事件里便于按命令分片归档
|
||||
* @param infoType 协议子类型(可为 0),同上
|
||||
* @param rawBytes 原始入站字节,不可为 null
|
||||
* @param parsedJson 已解析结构化 JSON。只用于 RAW 明细查询,不应复制到位置/字段等派生表。
|
||||
*/
|
||||
record RawArchive(
|
||||
String eventId,
|
||||
@@ -161,6 +162,25 @@ public sealed interface VehicleEvent
|
||||
Map<String, String> metadata,
|
||||
int command,
|
||||
int infoType,
|
||||
byte[] rawBytes
|
||||
) implements VehicleEvent {}
|
||||
byte[] rawBytes,
|
||||
String parsedJson
|
||||
) implements VehicleEvent {
|
||||
public RawArchive(String eventId,
|
||||
String vin,
|
||||
ProtocolId source,
|
||||
Instant eventTime,
|
||||
Instant ingestTime,
|
||||
String traceId,
|
||||
Map<String, String> metadata,
|
||||
int command,
|
||||
int infoType,
|
||||
byte[] rawBytes) {
|
||||
this(eventId, vin, source, eventTime, ingestTime, traceId, metadata,
|
||||
command, infoType, rawBytes, "");
|
||||
}
|
||||
|
||||
public RawArchive {
|
||||
parsedJson = parsedJson == null ? "" : parsedJson;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,14 @@
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-jsr310</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-autoconfigure</artifactId>
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
package com.lingniu.ingest.core.dispatcher;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.json.JsonMapper;
|
||||
import com.lingniu.ingest.api.event.RawArchiveKeys;
|
||||
import com.lingniu.ingest.api.event.VehicleEvent;
|
||||
import com.lingniu.ingest.api.pipeline.IngestContext;
|
||||
@@ -36,6 +39,9 @@ public final class Dispatcher {
|
||||
private static final Logger log = LoggerFactory.getLogger(Dispatcher.class);
|
||||
private static final AtomicLong RAW_ARCHIVE_SEQUENCE = new AtomicLong();
|
||||
private static final String REQUIRED_DURABLE_SINK = "kafka";
|
||||
private static final ObjectMapper RAW_ARCHIVE_JSON = JsonMapper.builder()
|
||||
.findAndAddModules()
|
||||
.build();
|
||||
|
||||
private final HandlerRegistry registry;
|
||||
private final InterceptorChain interceptors;
|
||||
@@ -167,7 +173,10 @@ public final class Dispatcher {
|
||||
byte[] bytes = frame.rawBytes();
|
||||
if (bytes == null || bytes.length == 0) return RawArchiveLookup.empty();
|
||||
|
||||
Map<String, String> meta = frame.sourceMeta() == null ? Map.of() : frame.sourceMeta();
|
||||
Map<String, String> meta = frame.sourceMeta() == null
|
||||
? Map.of()
|
||||
: new LinkedHashMap<>(frame.sourceMeta());
|
||||
String parsedJson = parsedJson(frame.payload(), meta.remove(RawArchiveKeys.META_PARSED_JSON));
|
||||
String vin = meta.getOrDefault("vin", "");
|
||||
Instant ingestTime = frame.receivedAt() != null ? frame.receivedAt() : Instant.now();
|
||||
String eventId = nextRawArchiveEventId(ingestTime);
|
||||
@@ -184,11 +193,34 @@ public final class Dispatcher {
|
||||
archiveMeta,
|
||||
frame.command(),
|
||||
frame.infoType(),
|
||||
bytes);
|
||||
bytes,
|
||||
parsedJson);
|
||||
out.add(raw);
|
||||
return new RawArchiveLookup(eventId, key, RawArchiveKeys.logicalUri(key));
|
||||
}
|
||||
|
||||
private static String parsedJson(Object payload, String explicitParsedJson) {
|
||||
if (explicitParsedJson != null && !explicitParsedJson.isBlank()) {
|
||||
return explicitParsedJson;
|
||||
}
|
||||
if (payload == null) {
|
||||
return "{}";
|
||||
}
|
||||
try {
|
||||
return RAW_ARCHIVE_JSON.writeValueAsString(payload);
|
||||
} catch (JsonProcessingException ex) {
|
||||
return "{\"serializationError\":\"" + escapeJson(ex.getMessage()) + "\",\"payloadType\":\""
|
||||
+ escapeJson(payload.getClass().getName()) + "\"}";
|
||||
}
|
||||
}
|
||||
|
||||
private static String escapeJson(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
return value.replace("\\", "\\\\").replace("\"", "\\\"");
|
||||
}
|
||||
|
||||
private static String nextRawArchiveEventId(Instant ingestTime) {
|
||||
// 文件名需要可排序且单进程内唯一:毫秒时间放大到微秒量级,再用 AtomicLong 递增兜底。
|
||||
long base = (ingestTime == null ? Instant.now() : ingestTime).toEpochMilli() * 1000L;
|
||||
|
||||
@@ -42,7 +42,7 @@ class DispatcherRawArchiveMetadataTest {
|
||||
ProtocolId.JT808,
|
||||
0x0200,
|
||||
0,
|
||||
new Object(),
|
||||
new SamplePayload("payload-0200", 7),
|
||||
new byte[]{0x7e, 0x01, 0x7e},
|
||||
Map.of("vin", "VIN001", "peer", "127.0.0.1:10001"),
|
||||
Instant.parse("2026-06-22T08:00:00Z")));
|
||||
@@ -52,6 +52,8 @@ class DispatcherRawArchiveMetadataTest {
|
||||
|
||||
String rawArchiveKey = raw.metadata().get("rawArchiveKey");
|
||||
assertThat(rawArchiveKey).isNotBlank();
|
||||
assertThat(raw.parsedJson()).contains("\"name\":\"payload-0200\"");
|
||||
assertThat(raw.metadata()).doesNotContainKey("rawArchiveParsedJson");
|
||||
assertThat(raw.eventId()).containsOnlyDigits();
|
||||
assertThat(rawArchiveKey).endsWith("/" + raw.eventId() + ".bin");
|
||||
assertThat(location.metadata())
|
||||
@@ -102,7 +104,7 @@ class DispatcherRawArchiveMetadataTest {
|
||||
|
||||
private static HandlerRegistry registryWithLocationHandler() throws NoSuchMethodException {
|
||||
LocationHandler bean = new LocationHandler();
|
||||
Method method = LocationHandler.class.getDeclaredMethod("onLocation", Object.class);
|
||||
Method method = LocationHandler.class.getDeclaredMethod("onLocation", SamplePayload.class);
|
||||
HandlerRegistry registry = new HandlerRegistry();
|
||||
registry.register(new HandlerDefinition(
|
||||
ProtocolId.JT808,
|
||||
@@ -169,7 +171,7 @@ class DispatcherRawArchiveMetadataTest {
|
||||
@ProtocolHandler(protocol = ProtocolId.JT808)
|
||||
static final class LocationHandler {
|
||||
@MessageMapping(command = 0x0200)
|
||||
VehicleEvent.Location onLocation(Object ignored) {
|
||||
VehicleEvent.Location onLocation(SamplePayload ignored) {
|
||||
return new VehicleEvent.Location(
|
||||
"location-1",
|
||||
"VIN001",
|
||||
@@ -182,6 +184,9 @@ class DispatcherRawArchiveMetadataTest {
|
||||
}
|
||||
}
|
||||
|
||||
record SamplePayload(String name, int value) {
|
||||
}
|
||||
|
||||
@ProtocolHandler(protocol = ProtocolId.JT808)
|
||||
public static final class AsyncLocationHandler {
|
||||
@MessageMapping(command = 0x0201)
|
||||
|
||||
@@ -24,6 +24,10 @@
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-autoconfigure</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
|
||||
@@ -1,70 +1,64 @@
|
||||
package com.lingniu.ingest.identity;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
|
||||
import org.springframework.jdbc.core.ConnectionCallback;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.time.Duration;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.Map;
|
||||
|
||||
public final class MySqlVehicleIdentityService implements VehicleIdentityResolver, VehicleIdentityRegistry, AutoCloseable {
|
||||
public final class MySqlVehicleIdentityService implements VehicleIdentityResolver, VehicleIdentityRegistry {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MySqlVehicleIdentityService.class);
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper mapper;
|
||||
private final String tableName;
|
||||
|
||||
private final DataSource dataSource;
|
||||
private final String table;
|
||||
private final String registrationTable;
|
||||
private final ScheduledExecutorService refresher;
|
||||
private volatile InMemoryVehicleIdentityService index;
|
||||
|
||||
public MySqlVehicleIdentityService(DataSource dataSource, String table) {
|
||||
this(dataSource, table, Duration.ZERO);
|
||||
}
|
||||
|
||||
public MySqlVehicleIdentityService(DataSource dataSource, String table, Duration refreshInterval) {
|
||||
if (dataSource == null) {
|
||||
throw new IllegalArgumentException("dataSource must not be null");
|
||||
public MySqlVehicleIdentityService(DataSource dataSource,
|
||||
VehicleIdentityProperties.Mysql properties,
|
||||
ObjectMapper mapper) {
|
||||
this.jdbc = new JdbcTemplate(dataSource);
|
||||
this.mapper = mapper == null ? new ObjectMapper() : mapper;
|
||||
VehicleIdentityProperties.Mysql mysql = properties == null ? new VehicleIdentityProperties.Mysql() : properties;
|
||||
this.tableName = safeTableName(mysql.getTableName());
|
||||
if (mysql.isInitializeSchema()) {
|
||||
initializeSchema();
|
||||
}
|
||||
this.dataSource = dataSource;
|
||||
this.table = sanitizeTable(table);
|
||||
this.registrationTable = this.table + "_registration";
|
||||
initializeSchema();
|
||||
this.index = loadIndex();
|
||||
this.refresher = startRefresher(refreshInterval);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void bind(VehicleIdentityBinding binding) {
|
||||
if (binding == null || !binding.hasResolvedVin()) {
|
||||
return;
|
||||
}
|
||||
upsert(binding.plate(), binding.vin());
|
||||
index.bind(binding);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void register(VehicleRegistrationBinding registration) {
|
||||
if (registration == null) {
|
||||
return;
|
||||
}
|
||||
upsertRegistration(registration);
|
||||
if (registration.hasResolvedVin()) {
|
||||
bind(new VehicleIdentityBinding(
|
||||
registration.protocol(),
|
||||
registration.vin(),
|
||||
registration.phone(),
|
||||
registration.deviceId(),
|
||||
registration.plate()));
|
||||
}
|
||||
String metadataJson = metadataJson(binding.metadata());
|
||||
jdbc.update("""
|
||||
INSERT INTO %s
|
||||
(protocol, vin, phone, device_id, plate, province, city, maker, device_type, plate_color, metadata_json,
|
||||
created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
vin = VALUES(vin),
|
||||
device_id = VALUES(device_id),
|
||||
province = VALUES(province),
|
||||
city = VALUES(city),
|
||||
maker = VALUES(maker),
|
||||
device_type = VALUES(device_type),
|
||||
plate_color = VALUES(plate_color),
|
||||
metadata_json = VALUES(metadata_json),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""".formatted(tableName),
|
||||
protocol(binding.protocol()), binding.vin(), binding.phone(), binding.deviceId(), binding.plate(),
|
||||
binding.metadata().getOrDefault("province", ""),
|
||||
binding.metadata().getOrDefault("city", ""),
|
||||
binding.metadata().getOrDefault("maker", ""),
|
||||
binding.metadata().getOrDefault("deviceType", ""),
|
||||
binding.metadata().getOrDefault("plateColor", ""),
|
||||
metadataJson);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -72,223 +66,167 @@ public final class MySqlVehicleIdentityService implements VehicleIdentityResolve
|
||||
if (lookup == null) {
|
||||
return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN);
|
||||
}
|
||||
return index.resolve(lookup);
|
||||
}
|
||||
|
||||
public void refresh() {
|
||||
index = loadIndex();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (refresher != null) {
|
||||
refresher.shutdownNow();
|
||||
if (!lookup.vin().isBlank()) {
|
||||
return new VehicleIdentity(lookup.vin(), true, VehicleIdentitySource.EXPLICIT_VIN);
|
||||
}
|
||||
VehicleIdentity byPhone = resolveBy("phone", lookup.protocol(), lookup.phone(), VehicleIdentitySource.BOUND_PHONE);
|
||||
if (byPhone != null) {
|
||||
return byPhone;
|
||||
}
|
||||
VehicleIdentity byDevice = resolveBy("device_id", lookup.protocol(), lookup.deviceId(), VehicleIdentitySource.BOUND_DEVICE_ID);
|
||||
if (byDevice != null) {
|
||||
return byDevice;
|
||||
}
|
||||
VehicleIdentity byPlate = resolveBy("plate", lookup.protocol(), lookup.plate(), VehicleIdentitySource.BOUND_PLATE);
|
||||
if (byPlate != null) {
|
||||
return byPlate;
|
||||
}
|
||||
if (!lookup.deviceId().isBlank()) {
|
||||
return new VehicleIdentity(lookup.deviceId(), false, VehicleIdentitySource.FALLBACK_DEVICE_ID);
|
||||
}
|
||||
if (!lookup.phone().isBlank()) {
|
||||
return new VehicleIdentity(lookup.phone(), false, VehicleIdentitySource.FALLBACK_PHONE);
|
||||
}
|
||||
if (!lookup.plate().isBlank()) {
|
||||
return new VehicleIdentity(lookup.plate(), false, VehicleIdentitySource.FALLBACK_PLATE);
|
||||
}
|
||||
return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN);
|
||||
}
|
||||
|
||||
private VehicleIdentity resolveBy(String column, ProtocolId protocol, String externalId, VehicleIdentitySource source) {
|
||||
if (externalId == null || externalId.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String safeColumn = switch (column) {
|
||||
case "phone", "device_id", "plate" -> column;
|
||||
default -> throw new IllegalArgumentException("unsupported identity lookup column: " + column);
|
||||
};
|
||||
List<VehicleIdentity> matches = jdbc.query("""
|
||||
SELECT vin
|
||||
FROM %s
|
||||
WHERE protocol = ? AND %s = ?
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1
|
||||
""".formatted(tableName, safeColumn),
|
||||
(rs, rowNum) -> identity(rs, source),
|
||||
protocol(protocol), externalId.trim());
|
||||
return matches.isEmpty() ? null : matches.getFirst();
|
||||
}
|
||||
|
||||
private VehicleIdentity identity(ResultSet rs, VehicleIdentitySource source) throws SQLException {
|
||||
return new VehicleIdentity(rs.getString("vin"), true, source);
|
||||
}
|
||||
|
||||
private void initializeSchema() {
|
||||
String sql = """
|
||||
jdbc.execute("""
|
||||
CREATE TABLE IF NOT EXISTS %s (
|
||||
plate VARCHAR(128) NOT NULL,
|
||||
vin VARCHAR(64) NOT NULL,
|
||||
PRIMARY KEY (plate),
|
||||
KEY idx_vehicle_identity_vin (vin)
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
phone VARCHAR(64) NOT NULL DEFAULT '',
|
||||
device_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
plate VARCHAR(64) NOT NULL DEFAULT '',
|
||||
province VARCHAR(32) NOT NULL DEFAULT '',
|
||||
city VARCHAR(32) NOT NULL DEFAULT '',
|
||||
maker VARCHAR(64) NOT NULL DEFAULT '',
|
||||
device_type VARCHAR(128) NOT NULL DEFAULT '',
|
||||
plate_color VARCHAR(32) NOT NULL DEFAULT '',
|
||||
metadata_json TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_vehicle_identity_protocol_phone (protocol, phone),
|
||||
KEY idx_vehicle_identity_protocol_device (protocol, device_id),
|
||||
KEY idx_vehicle_identity_protocol_plate (protocol, plate)
|
||||
)
|
||||
""".formatted(table);
|
||||
String registrationSql = """
|
||||
CREATE TABLE IF NOT EXISTS %s (
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
phone VARCHAR(64) NOT NULL,
|
||||
vin VARCHAR(64) NOT NULL DEFAULT 'unknown',
|
||||
device_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||
plate VARCHAR(128) NOT NULL DEFAULT '',
|
||||
province INT NULL,
|
||||
city INT NULL,
|
||||
maker VARCHAR(64) NOT NULL DEFAULT '',
|
||||
device_type VARCHAR(128) NOT NULL DEFAULT '',
|
||||
plate_color INT NULL,
|
||||
last_registered_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (protocol, phone),
|
||||
KEY idx_vehicle_registration_vin (vin),
|
||||
KEY idx_vehicle_registration_device (protocol, device_id),
|
||||
KEY idx_vehicle_registration_plate (protocol, plate)
|
||||
)
|
||||
""".formatted(registrationTable);
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute(sql);
|
||||
statement.execute(registrationSql);
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException("vehicle identity mysql schema initialize failed: " + table, e);
|
||||
""".formatted(tableName));
|
||||
ensureColumn("phone", "phone VARCHAR(64) NOT NULL DEFAULT ''");
|
||||
ensureColumn("device_id", "device_id VARCHAR(64) NOT NULL DEFAULT ''");
|
||||
ensureColumn("plate", "plate VARCHAR(64) NOT NULL DEFAULT ''");
|
||||
ensureColumn("province", "province VARCHAR(32) NOT NULL DEFAULT ''");
|
||||
ensureColumn("city", "city VARCHAR(32) NOT NULL DEFAULT ''");
|
||||
ensureColumn("maker", "maker VARCHAR(64) NOT NULL DEFAULT ''");
|
||||
ensureColumn("device_type", "device_type VARCHAR(128) NOT NULL DEFAULT ''");
|
||||
ensureColumn("plate_color", "plate_color VARCHAR(32) NOT NULL DEFAULT ''");
|
||||
ensureColumn("metadata_json", "metadata_json TEXT");
|
||||
ensureColumn("created_at", "created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP");
|
||||
ensureColumn("updated_at", "updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP");
|
||||
ensureIndex("idx_vehicle_identity_protocol_device", "CREATE INDEX idx_vehicle_identity_protocol_device ON %s (protocol, device_id)");
|
||||
ensureIndex("idx_vehicle_identity_protocol_plate", "CREATE INDEX idx_vehicle_identity_protocol_plate ON %s (protocol, plate)");
|
||||
}
|
||||
|
||||
private void ensureColumn(String columnName, String definition) {
|
||||
if (!columnExists(columnName)) {
|
||||
jdbc.execute("ALTER TABLE " + tableName + " ADD COLUMN " + definition);
|
||||
}
|
||||
}
|
||||
|
||||
private InMemoryVehicleIdentityService loadIndex() {
|
||||
InMemoryVehicleIdentityService loaded = new InMemoryVehicleIdentityService();
|
||||
String sql = "SELECT plate, vin FROM " + table;
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
PreparedStatement statement = connection.prepareStatement(sql);
|
||||
ResultSet resultSet = statement.executeQuery()) {
|
||||
while (resultSet.next()) {
|
||||
bindGlobalPlate(loaded, resultSet.getString("vin"), resultSet.getString("plate"));
|
||||
private boolean columnExists(String columnName) {
|
||||
return Boolean.TRUE.equals(jdbc.execute((ConnectionCallback<Boolean>) connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
for (String table : nameVariants(tableName)) {
|
||||
for (String column : nameVariants(columnName)) {
|
||||
try (ResultSet rs = metaData.getColumns(null, null, table, column)) {
|
||||
if (rs.next()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
loadRegistrationIndex(loaded, connection);
|
||||
return loaded;
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException("vehicle identity mysql index load failed: " + table, e);
|
||||
return false;
|
||||
}));
|
||||
}
|
||||
|
||||
private void ensureIndex(String indexName, String ddlTemplate) {
|
||||
if (!indexExists(indexName)) {
|
||||
jdbc.execute(ddlTemplate.formatted(tableName));
|
||||
}
|
||||
}
|
||||
|
||||
private static void bindGlobalPlate(InMemoryVehicleIdentityService loaded, String vin, String plate) {
|
||||
for (ProtocolId protocol : new ProtocolId[]{
|
||||
ProtocolId.GB32960,
|
||||
ProtocolId.JT808,
|
||||
ProtocolId.MQTT_YUTONG,
|
||||
ProtocolId.XINDA_PUSH
|
||||
}) {
|
||||
loaded.bind(new VehicleIdentityBinding(protocol, vin, "", "", plate));
|
||||
}
|
||||
}
|
||||
|
||||
private void loadRegistrationIndex(InMemoryVehicleIdentityService loaded, Connection connection) throws SQLException {
|
||||
String sql = "SELECT r.protocol, r.phone, r.device_id, r.plate, b.vin FROM " + registrationTable + " r "
|
||||
+ "JOIN " + table + " b ON b.plate = r.plate "
|
||||
+ "WHERE b.vin IS NOT NULL AND b.vin <> '' AND LOWER(b.vin) <> 'unknown'";
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql);
|
||||
ResultSet resultSet = statement.executeQuery()) {
|
||||
while (resultSet.next()) {
|
||||
loaded.bind(new VehicleIdentityBinding(
|
||||
protocol(resultSet.getString("protocol")),
|
||||
resultSet.getString("vin"),
|
||||
resultSet.getString("phone"),
|
||||
resultSet.getString("device_id"),
|
||||
resultSet.getString("plate")));
|
||||
private boolean indexExists(String indexName) {
|
||||
return Boolean.TRUE.equals(jdbc.execute((ConnectionCallback<Boolean>) connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
for (String table : nameVariants(tableName)) {
|
||||
try (ResultSet rs = metaData.getIndexInfo(null, null, table, false, false)) {
|
||||
while (rs.next()) {
|
||||
String existing = rs.getString("INDEX_NAME");
|
||||
if (existing != null && existing.equalsIgnoreCase(indexName)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
loadLegacyRegistrationVinIndex(loaded, connection);
|
||||
return false;
|
||||
}));
|
||||
}
|
||||
|
||||
private void loadLegacyRegistrationVinIndex(InMemoryVehicleIdentityService loaded, Connection connection) throws SQLException {
|
||||
String sql = "SELECT protocol, phone, device_id, plate, vin FROM " + registrationTable
|
||||
+ " WHERE vin IS NOT NULL AND vin <> '' AND LOWER(vin) <> 'unknown'";
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql);
|
||||
ResultSet resultSet = statement.executeQuery()) {
|
||||
while (resultSet.next()) {
|
||||
loaded.bind(new VehicleIdentityBinding(
|
||||
protocol(resultSet.getString("protocol")),
|
||||
resultSet.getString("vin"),
|
||||
resultSet.getString("phone"),
|
||||
resultSet.getString("device_id"),
|
||||
resultSet.getString("plate")));
|
||||
}
|
||||
}
|
||||
private static String[] nameVariants(String name) {
|
||||
return new String[]{
|
||||
name,
|
||||
name.toLowerCase(Locale.ROOT),
|
||||
name.toUpperCase(Locale.ROOT)
|
||||
};
|
||||
}
|
||||
|
||||
private ScheduledExecutorService startRefresher(Duration refreshInterval) {
|
||||
if (refreshInterval == null || refreshInterval.isZero() || refreshInterval.isNegative()) {
|
||||
return null;
|
||||
private String metadataJson(Map<String, String> metadata) {
|
||||
if (metadata == null || metadata.isEmpty()) {
|
||||
return "{}";
|
||||
}
|
||||
long delayMillis = Math.max(refreshInterval.toMillis(), 1000L);
|
||||
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = new Thread(r, "vehicle-identity-mysql-refresh");
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
});
|
||||
executor.scheduleWithFixedDelay(this::safeRefresh, delayMillis, delayMillis, TimeUnit.MILLISECONDS);
|
||||
return executor;
|
||||
}
|
||||
|
||||
private void safeRefresh() {
|
||||
try {
|
||||
refresh();
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("vehicle identity mysql refresh failed table={}", table, e);
|
||||
return mapper.writeValueAsString(metadata);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalArgumentException("vehicle identity metadata serialize failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void upsert(String plate, String vin) {
|
||||
String normalizedPlate = normalize(plate);
|
||||
if (normalizedPlate.isBlank()) {
|
||||
return;
|
||||
}
|
||||
String sql = "INSERT INTO " + table + " (plate, vin) "
|
||||
+ "VALUES (?, ?) "
|
||||
+ "ON DUPLICATE KEY UPDATE vin = VALUES(vin)";
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
statement.setString(1, normalizedPlate);
|
||||
statement.setString(2, vin.trim());
|
||||
statement.executeUpdate();
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException("vehicle identity mysql bind failed: " + table, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void upsertRegistration(VehicleRegistrationBinding registration) {
|
||||
String phone = normalize(registration.phone());
|
||||
if (phone.isBlank()) {
|
||||
return;
|
||||
}
|
||||
String vin = registration.hasResolvedVin() ? registration.vin().trim() : "unknown";
|
||||
String sql = "INSERT INTO " + registrationTable
|
||||
+ " (protocol, phone, vin, device_id, plate, province, city, maker, device_type, plate_color) "
|
||||
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
+ "ON DUPLICATE KEY UPDATE "
|
||||
+ "vin = CASE WHEN VALUES(vin) <> 'unknown' THEN VALUES(vin) ELSE vin END, "
|
||||
+ "device_id = VALUES(device_id), plate = VALUES(plate), province = VALUES(province), "
|
||||
+ "city = VALUES(city), maker = VALUES(maker), device_type = VALUES(device_type), "
|
||||
+ "plate_color = VALUES(plate_color), last_registered_at = CURRENT_TIMESTAMP";
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
statement.setString(1, protocolName(registration.protocol()));
|
||||
statement.setString(2, phone);
|
||||
statement.setString(3, vin);
|
||||
statement.setString(4, normalize(registration.deviceId()));
|
||||
statement.setString(5, normalize(registration.plate()));
|
||||
setInteger(statement, 6, registration.province());
|
||||
setInteger(statement, 7, registration.city());
|
||||
statement.setString(8, normalize(registration.maker()));
|
||||
statement.setString(9, normalize(registration.deviceType()));
|
||||
setInteger(statement, 10, registration.plateColor());
|
||||
statement.executeUpdate();
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException("vehicle identity mysql registration bind failed: " + registrationTable, e);
|
||||
}
|
||||
}
|
||||
|
||||
private static void setInteger(PreparedStatement statement, int index, Integer value) throws SQLException {
|
||||
if (value == null) {
|
||||
statement.setObject(index, null);
|
||||
} else {
|
||||
statement.setInt(index, value);
|
||||
}
|
||||
}
|
||||
|
||||
private static String sanitizeTable(String table) {
|
||||
String value = table == null || table.isBlank() ? "vehicle_identity_binding" : table.trim();
|
||||
if (!value.matches("[A-Za-z0-9_]+")) {
|
||||
throw new IllegalArgumentException("vehicle identity mysql table must contain only letters, digits, and underscore");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static String protocolName(ProtocolId protocol) {
|
||||
private static String protocol(ProtocolId protocol) {
|
||||
return protocol == null ? "UNKNOWN" : protocol.name();
|
||||
}
|
||||
|
||||
private static ProtocolId protocol(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return ProtocolId.UNKNOWN;
|
||||
private static String safeTableName(String value) {
|
||||
String name = value == null || value.isBlank() ? "vehicle_identity_bindings" : value.trim();
|
||||
if (!name.matches("[A-Za-z0-9_]+")) {
|
||||
throw new IllegalArgumentException("invalid vehicle identity mysql table name: " + value);
|
||||
}
|
||||
try {
|
||||
return ProtocolId.valueOf(value.trim().toUpperCase(Locale.ROOT));
|
||||
} catch (IllegalArgumentException ignored) {
|
||||
return ProtocolId.UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
private static String normalize(String value) {
|
||||
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
|
||||
return name.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,18 +2,26 @@ package com.lingniu.ingest.identity;
|
||||
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public record VehicleIdentityBinding(
|
||||
ProtocolId protocol,
|
||||
String vin,
|
||||
String phone,
|
||||
String deviceId,
|
||||
String plate
|
||||
String plate,
|
||||
Map<String, String> metadata
|
||||
) {
|
||||
public VehicleIdentityBinding(ProtocolId protocol, String vin, String phone, String deviceId, String plate) {
|
||||
this(protocol, vin, phone, deviceId, plate, Map.of());
|
||||
}
|
||||
|
||||
public VehicleIdentityBinding {
|
||||
vin = normalize(vin);
|
||||
phone = normalize(phone);
|
||||
deviceId = normalize(deviceId);
|
||||
plate = normalize(plate);
|
||||
metadata = metadata == null ? Map.of() : Map.copyOf(metadata);
|
||||
if (vin.isBlank()) {
|
||||
throw new IllegalArgumentException("vin must not be blank");
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.lingniu.ingest.identity;
|
||||
|
||||
public enum VehicleIdentitySource {
|
||||
EXPLICIT_VIN,
|
||||
REGISTERED,
|
||||
BOUND_PHONE,
|
||||
BOUND_DEVICE_ID,
|
||||
BOUND_PLATE,
|
||||
|
||||
@@ -11,15 +11,10 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.io.PrintWriter;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.SQLFeatureNotSupportedException;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
@AutoConfiguration
|
||||
@EnableConfigurationProperties(VehicleIdentityProperties.class)
|
||||
@@ -40,22 +35,15 @@ public class VehicleIdentityAutoConfiguration {
|
||||
return new FileVehicleIdentityService(Path.of(properties.getFile().getPath()), objectMapper);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean(DataSource.class)
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql")
|
||||
public DataSource vehicleIdentityDataSource(VehicleIdentityProperties properties) {
|
||||
VehicleIdentityProperties.Mysql mysql = properties.getMysql();
|
||||
return new DriverManagerVehicleIdentityDataSource(
|
||||
mysql.getDriverClassName(), mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean({VehicleIdentityResolver.class, VehicleIdentityRegistry.class})
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql")
|
||||
public MySqlVehicleIdentityService mySqlVehicleIdentityService(VehicleIdentityProperties properties,
|
||||
DataSource dataSource) {
|
||||
public MySqlVehicleIdentityService mysqlVehicleIdentityService(VehicleIdentityProperties properties,
|
||||
ObjectMapper objectMapper) {
|
||||
VehicleIdentityProperties.Mysql mysql = properties.getMysql();
|
||||
return new MySqlVehicleIdentityService(dataSource, mysql.getTable(), mysql.getRefreshInterval());
|
||||
DataSource dataSource = new DriverManagerDataSource(
|
||||
mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword());
|
||||
return new MySqlVehicleIdentityService(dataSource, mysql, objectMapper);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@@ -65,61 +53,4 @@ public class VehicleIdentityAutoConfiguration {
|
||||
public InMemoryVehicleIdentityService vehicleIdentityService() {
|
||||
return new InMemoryVehicleIdentityService();
|
||||
}
|
||||
|
||||
private static final class DriverManagerVehicleIdentityDataSource implements DataSource {
|
||||
private final String jdbcUrl;
|
||||
private final String username;
|
||||
private final String password;
|
||||
private PrintWriter logWriter;
|
||||
private int loginTimeout;
|
||||
|
||||
private DriverManagerVehicleIdentityDataSource(String driverClassName,
|
||||
String jdbcUrl,
|
||||
String username,
|
||||
String password) {
|
||||
if (jdbcUrl == null || jdbcUrl.isBlank()) {
|
||||
throw new IllegalArgumentException("lingniu.ingest.identity.mysql.jdbc-url is required");
|
||||
}
|
||||
if (driverClassName != null && !driverClassName.isBlank()) {
|
||||
try {
|
||||
Class.forName(driverClassName.trim());
|
||||
} catch (ClassNotFoundException e) {
|
||||
throw new IllegalArgumentException("vehicle identity mysql driver class not found: "
|
||||
+ driverClassName, e);
|
||||
}
|
||||
}
|
||||
this.jdbcUrl = jdbcUrl;
|
||||
this.username = username == null ? "" : username;
|
||||
this.password = password == null ? "" : password;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Connection getConnection() throws SQLException {
|
||||
DriverManager.setLoginTimeout(loginTimeout);
|
||||
return DriverManager.getConnection(jdbcUrl, username, password);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Connection getConnection(String username, String password) throws SQLException {
|
||||
DriverManager.setLoginTimeout(loginTimeout);
|
||||
return DriverManager.getConnection(jdbcUrl, username, password);
|
||||
}
|
||||
|
||||
@Override public PrintWriter getLogWriter() { return logWriter; }
|
||||
@Override public void setLogWriter(PrintWriter out) { this.logWriter = out; }
|
||||
@Override public void setLoginTimeout(int seconds) { this.loginTimeout = seconds; }
|
||||
@Override public int getLoginTimeout() { return loginTimeout; }
|
||||
@Override public Logger getParentLogger() throws SQLFeatureNotSupportedException {
|
||||
throw new SQLFeatureNotSupportedException();
|
||||
}
|
||||
@Override public <T> T unwrap(Class<T> iface) throws SQLException {
|
||||
if (iface.isInstance(this)) {
|
||||
return iface.cast(this);
|
||||
}
|
||||
throw new SQLException("not a wrapper for " + iface);
|
||||
}
|
||||
@Override public boolean isWrapperFor(Class<?> iface) {
|
||||
return iface.isInstance(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,6 @@ package com.lingniu.ingest.identity.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
@ConfigurationProperties(prefix = "lingniu.ingest.identity")
|
||||
public class VehicleIdentityProperties {
|
||||
|
||||
@@ -26,24 +24,21 @@ public class VehicleIdentityProperties {
|
||||
}
|
||||
|
||||
public static class Mysql {
|
||||
private String table = "vehicle_identity_binding";
|
||||
private String jdbcUrl = "";
|
||||
private String username = "";
|
||||
private String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai";
|
||||
private String username = "root";
|
||||
private String password = "";
|
||||
private String driverClassName = "com.mysql.cj.jdbc.Driver";
|
||||
private Duration refreshInterval = Duration.ofSeconds(60);
|
||||
private String tableName = "vehicle_identity_bindings";
|
||||
private boolean initializeSchema = true;
|
||||
|
||||
public String getTable() { return table; }
|
||||
public void setTable(String table) { this.table = table; }
|
||||
public String getJdbcUrl() { return jdbcUrl; }
|
||||
public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; }
|
||||
public String getUsername() { return username; }
|
||||
public void setUsername(String username) { this.username = username; }
|
||||
public String getPassword() { return password; }
|
||||
public void setPassword(String password) { this.password = password; }
|
||||
public String getDriverClassName() { return driverClassName; }
|
||||
public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; }
|
||||
public Duration getRefreshInterval() { return refreshInterval; }
|
||||
public void setRefreshInterval(Duration refreshInterval) { this.refreshInterval = refreshInterval; }
|
||||
public String getTableName() { return tableName; }
|
||||
public void setTableName(String tableName) { this.tableName = tableName; }
|
||||
public boolean isInitializeSchema() { return initializeSchema; }
|
||||
public void setInitializeSchema(boolean initializeSchema) { this.initializeSchema = initializeSchema; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package com.lingniu.ingest.identity;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
|
||||
import org.h2.jdbcx.JdbcDataSource;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -9,6 +11,7 @@ import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -21,62 +24,67 @@ class MySqlVehicleIdentityServiceJdbcTest {
|
||||
dataSource.setURL("jdbc:h2:mem:" + UUID.randomUUID()
|
||||
+ ";MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1");
|
||||
|
||||
try (MySqlVehicleIdentityService service =
|
||||
new MySqlVehicleIdentityService(dataSource, "vehicle_identity_binding")) {
|
||||
service.register(new VehicleRegistrationBinding(
|
||||
ProtocolId.JT808,
|
||||
"unknown",
|
||||
"13079961001",
|
||||
"dev61001",
|
||||
"粤B61001",
|
||||
44,
|
||||
4401,
|
||||
"maker-a",
|
||||
"type-a",
|
||||
1));
|
||||
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
|
||||
mysql.setTableName("vehicle_identity_binding");
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
|
||||
dataSource,
|
||||
mysql,
|
||||
new ObjectMapper());
|
||||
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
ResultSet registered = statement.executeQuery("""
|
||||
SELECT vin, device_id, plate, province, city, maker, device_type, plate_color
|
||||
FROM vehicle_identity_binding_registration
|
||||
WHERE protocol = 'JT808' AND phone = '13079961001'
|
||||
""");
|
||||
assertThat(registered.next()).isTrue();
|
||||
assertThat(registered.getString("vin")).isEqualTo("unknown");
|
||||
assertThat(registered.getString("device_id")).isEqualTo("DEV61001");
|
||||
assertThat(registered.getString("plate")).isEqualTo("粤B61001");
|
||||
assertThat(registered.getInt("province")).isEqualTo(44);
|
||||
assertThat(registered.getInt("city")).isEqualTo(4401);
|
||||
assertThat(registered.getString("maker")).isEqualTo("MAKER-A");
|
||||
assertThat(registered.getString("device_type")).isEqualTo("TYPE-A");
|
||||
assertThat(registered.getInt("plate_color")).isEqualTo(1);
|
||||
service.bind(new VehicleIdentityBinding(
|
||||
ProtocolId.JT808,
|
||||
"LNVIN00000061001",
|
||||
"13079961001",
|
||||
"dev61001",
|
||||
"粤B61001",
|
||||
Map.of(
|
||||
"province", "44",
|
||||
"city", "4401",
|
||||
"maker", "maker-a",
|
||||
"deviceType", "type-a",
|
||||
"plateColor", "1")));
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
ResultSet bindingColumns = connection.getMetaData()
|
||||
.getColumns(null, null, "vehicle_identity_binding", null);
|
||||
while (bindingColumns.next()) {
|
||||
columns.add(bindingColumns.getString("COLUMN_NAME"));
|
||||
}
|
||||
assertThat(columns).containsExactly("plate", "vin");
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
ResultSet binding = statement.executeQuery("""
|
||||
SELECT vin, phone, device_id, plate, province, city, maker, device_type, plate_color
|
||||
FROM vehicle_identity_binding
|
||||
WHERE protocol = 'JT808' AND phone = '13079961001'
|
||||
""");
|
||||
assertThat(binding.next()).isTrue();
|
||||
assertThat(binding.getString("vin")).isEqualTo("LNVIN00000061001");
|
||||
assertThat(binding.getString("phone")).isEqualTo("13079961001");
|
||||
assertThat(binding.getString("device_id")).isEqualTo("dev61001");
|
||||
assertThat(binding.getString("plate")).isEqualTo("粤B61001");
|
||||
assertThat(binding.getString("province")).isEqualTo("44");
|
||||
assertThat(binding.getString("city")).isEqualTo("4401");
|
||||
assertThat(binding.getString("maker")).isEqualTo("maker-a");
|
||||
assertThat(binding.getString("device_type")).isEqualTo("type-a");
|
||||
assertThat(binding.getString("plate_color")).isEqualTo("1");
|
||||
|
||||
statement.executeUpdate("""
|
||||
INSERT INTO vehicle_identity_binding (plate, vin)
|
||||
VALUES ('粤B61001', 'LNVIN00000061001')
|
||||
""");
|
||||
List<String> columns = new ArrayList<>();
|
||||
ResultSet bindingColumns = connection.getMetaData()
|
||||
.getColumns(null, null, "vehicle_identity_binding", null);
|
||||
while (bindingColumns.next()) {
|
||||
columns.add(bindingColumns.getString("COLUMN_NAME"));
|
||||
}
|
||||
|
||||
service.refresh();
|
||||
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079961001", "", "")).vin())
|
||||
.isEqualTo("LNVIN00000061001");
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "dev61001", "")).vin())
|
||||
.isEqualTo("LNVIN00000061001");
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "", "粤B61001")).vin())
|
||||
.isEqualTo("LNVIN00000061001");
|
||||
assertThat(columns).contains(
|
||||
"protocol",
|
||||
"phone",
|
||||
"device_id",
|
||||
"plate",
|
||||
"vin",
|
||||
"metadata_json");
|
||||
}
|
||||
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079961001", "", "")).vin())
|
||||
.isEqualTo("LNVIN00000061001");
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "dev61001", "")).vin())
|
||||
.isEqualTo("LNVIN00000061001");
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "", "粤B61001")).vin())
|
||||
.isEqualTo("LNVIN00000061001");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,478 +1,157 @@
|
||||
package com.lingniu.ingest.identity;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.io.PrintWriter;
|
||||
import java.lang.reflect.InvocationHandler;
|
||||
import java.lang.reflect.Proxy;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.SQLFeatureNotSupportedException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class MySqlVehicleIdentityServiceTest {
|
||||
|
||||
@Test
|
||||
void bindsAndResolvesIdentityThroughJdbcStore() {
|
||||
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
|
||||
void upsertsRegistrationBindingAndResolvesByExternalIdentifiers() {
|
||||
DriverManagerDataSource dataSource = new DriverManagerDataSource(
|
||||
"jdbc:h2:mem:mysql-identity;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
|
||||
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
|
||||
mysql.setTableName("vehicle_identity_bindings");
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
|
||||
dataSource,
|
||||
mysql,
|
||||
new ObjectMapper());
|
||||
|
||||
service.bind(new VehicleIdentityBinding(
|
||||
ProtocolId.JT808, "LNVIN000000000123", "13079960001", "DEV001", "粤B00001"));
|
||||
|
||||
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079960001", "", ""));
|
||||
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "DEV001", ""));
|
||||
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "", "粤B00001"));
|
||||
|
||||
assertThat(jdbc.createdTables)
|
||||
.anySatisfy(sql -> assertThat(sql).contains("CREATE TABLE IF NOT EXISTS vehicle_identity_binding"));
|
||||
assertThat(jdbc.bindings).containsEntry("粤B00001", "LNVIN000000000123");
|
||||
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000123");
|
||||
assertThat(byPhone.resolved()).isTrue();
|
||||
assertThat(byPhone.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
|
||||
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000123");
|
||||
assertThat(byDevice.source()).isEqualTo(VehicleIdentitySource.BOUND_DEVICE_ID);
|
||||
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000123");
|
||||
assertThat(byPlate.source()).isEqualTo(VehicleIdentitySource.BOUND_PLATE);
|
||||
}
|
||||
|
||||
@Test
|
||||
void ignoresUnknownVinBinding() {
|
||||
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
|
||||
|
||||
service.bind(new VehicleIdentityBinding(
|
||||
ProtocolId.JT808, "unknown", "13079960001", "DEV001", "粤B00001"));
|
||||
|
||||
assertThat(jdbc.bindings).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvesFromStartupCacheWithoutPerFrameDatabaseLookup() {
|
||||
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
|
||||
jdbc.seedRegistration("JT808", "13079960001", "DEV001", "粤B00001");
|
||||
jdbc.seed("粤B00001", "LNVIN000000000123");
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
|
||||
|
||||
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079960001", "", ""));
|
||||
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "DEV001", ""));
|
||||
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "", "粤B00001"));
|
||||
|
||||
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000123");
|
||||
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000123");
|
||||
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000123");
|
||||
assertThat(jdbc.loadAllSelects).isEqualTo(1);
|
||||
assertThat(jdbc.lookupSelects).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvesGlobalPlateBindingForXindaPush() {
|
||||
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
|
||||
jdbc.seed("浙F00780F", "LA9GG68L8PBAF4776");
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
|
||||
|
||||
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.XINDA_PUSH, "", "", "", "浙F00780F"));
|
||||
|
||||
assertThat(byPlate.vin()).isEqualTo("LA9GG68L8PBAF4776");
|
||||
assertThat(byPlate.resolved()).isTrue();
|
||||
assertThat(byPlate.source()).isEqualTo(VehicleIdentitySource.BOUND_PLATE);
|
||||
assertThat(jdbc.loadAllSelects).isEqualTo(1);
|
||||
assertThat(jdbc.lookupSelects).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void refreshLoadsExternallyInsertedBindingWithoutPerFrameDatabaseLookup() {
|
||||
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
|
||||
jdbc.seedRegistration("JT808", "13079960002", "DEV002", "粤B00002");
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
|
||||
|
||||
VehicleIdentity before = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079960002", "", ""));
|
||||
|
||||
jdbc.seed("粤B00002", "LNVIN000000000456");
|
||||
service.refresh();
|
||||
VehicleIdentity after = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079960002", "", ""));
|
||||
|
||||
assertThat(before.resolved()).isFalse();
|
||||
assertThat(after.vin()).isEqualTo("LNVIN000000000456");
|
||||
assertThat(after.resolved()).isTrue();
|
||||
assertThat(after.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
|
||||
assertThat(jdbc.loadAllSelects).isEqualTo(2);
|
||||
assertThat(jdbc.lookupSelects).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void storesRegistrationWithoutVinAndResolvesAfterPlateVinIsWrittenBack() {
|
||||
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
|
||||
|
||||
service.register(new VehicleRegistrationBinding(
|
||||
ProtocolId.JT808,
|
||||
"unknown",
|
||||
"13079960003",
|
||||
"DEV003",
|
||||
"粤B00003",
|
||||
44,
|
||||
4401,
|
||||
"MAKER",
|
||||
"TYPE-A",
|
||||
1));
|
||||
"JT808AABBCCDDEE1",
|
||||
"13079963289",
|
||||
"9963289",
|
||||
"沪A00113F",
|
||||
Map.of(
|
||||
"province", "31",
|
||||
"city", "113",
|
||||
"maker", "70112",
|
||||
"deviceType", "SEG-9888G",
|
||||
"plateColor", "2")));
|
||||
|
||||
VehicleIdentity before = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079960003", "", ""));
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "13079963289", "", "")))
|
||||
.extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
|
||||
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_PHONE);
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "", "9963289", "")))
|
||||
.extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
|
||||
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_DEVICE_ID);
|
||||
assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "", "", "沪A00113F")))
|
||||
.extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
|
||||
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_PLATE);
|
||||
|
||||
jdbc.seed("粤B00003", "LNVIN000000000789");
|
||||
service.refresh();
|
||||
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079960003", "", ""));
|
||||
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "DEV003", ""));
|
||||
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "", "", "粤B00003"));
|
||||
|
||||
assertThat(jdbc.createdTables)
|
||||
.anySatisfy(sql -> assertThat(sql).contains("CREATE TABLE IF NOT EXISTS vehicle_identity_binding_registration"));
|
||||
assertThat(jdbc.registrations).containsKey("JT808|13079960003");
|
||||
assertThat(jdbc.bindings).containsEntry("粤B00003", "LNVIN000000000789");
|
||||
assertThat(before.resolved()).isFalse();
|
||||
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000789");
|
||||
assertThat(byPhone.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
|
||||
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000789");
|
||||
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000789");
|
||||
JdbcTemplate jdbc = new JdbcTemplate(dataSource);
|
||||
Map<String, Object> row = jdbc.queryForMap("""
|
||||
SELECT vin, phone, device_id, plate, province, city, maker, device_type, plate_color
|
||||
FROM vehicle_identity_bindings
|
||||
WHERE protocol = 'JT808' AND phone = '13079963289'
|
||||
""");
|
||||
assertThat(row)
|
||||
.containsEntry("vin", "JT808AABBCCDDEE1")
|
||||
.containsEntry("phone", "13079963289")
|
||||
.containsEntry("device_id", "9963289")
|
||||
.containsEntry("plate", "沪A00113F")
|
||||
.containsEntry("province", "31")
|
||||
.containsEntry("city", "113")
|
||||
.containsEntry("maker", "70112")
|
||||
.containsEntry("device_type", "SEG-9888G")
|
||||
.containsEntry("plate_color", "2");
|
||||
}
|
||||
|
||||
private static final class RecordingIdentityJdbc {
|
||||
private final List<String> createdTables = new ArrayList<>();
|
||||
private final Map<String, String> bindings = new HashMap<>();
|
||||
private final Map<String, RegistrationRow> registrations = new HashMap<>();
|
||||
private int loadAllSelects;
|
||||
private int lookupSelects;
|
||||
@Test
|
||||
void initializesMissingColumnsOnExistingLegacyTable() {
|
||||
DriverManagerDataSource dataSource = new DriverManagerDataSource(
|
||||
"jdbc:h2:mem:mysql-identity-legacy;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
|
||||
JdbcTemplate jdbc = new JdbcTemplate(dataSource);
|
||||
jdbc.execute("""
|
||||
CREATE TABLE vehicle_identity_binding (
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
legacy_key VARCHAR(64) NOT NULL DEFAULT ''
|
||||
)
|
||||
""");
|
||||
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
|
||||
mysql.setTableName("vehicle_identity_binding");
|
||||
|
||||
private void seed(String plate, String vin) {
|
||||
bindings.put(normalize(plate), vin);
|
||||
}
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
|
||||
dataSource,
|
||||
mysql,
|
||||
new ObjectMapper());
|
||||
service.bind(new VehicleIdentityBinding(
|
||||
ProtocolId.JT808,
|
||||
"JT808LEGACY00001",
|
||||
"13079963289",
|
||||
"9963289",
|
||||
"沪A00113F",
|
||||
Map.of("maker", "70112", "deviceType", "SEG-9888G")));
|
||||
|
||||
private void seedRegistration(String protocol, String phone, String deviceId, String plate) {
|
||||
registrations.put(protocol + "|" + normalize(phone),
|
||||
new RegistrationRow(protocol, normalize(phone), "unknown", normalize(deviceId), normalize(plate)));
|
||||
}
|
||||
Map<String, Object> row = jdbc.queryForMap("""
|
||||
SELECT vin, maker, device_type, metadata_json
|
||||
FROM vehicle_identity_binding
|
||||
WHERE protocol = 'JT808' AND phone = '13079963289'
|
||||
""");
|
||||
assertThat(row)
|
||||
.containsEntry("vin", "JT808LEGACY00001")
|
||||
.containsEntry("maker", "70112")
|
||||
.containsEntry("device_type", "SEG-9888G");
|
||||
assertThat((String) row.get("metadata_json")).contains("SEG-9888G");
|
||||
}
|
||||
|
||||
private DataSource dataSource() {
|
||||
return new DataSource() {
|
||||
@Override public Connection getConnection() { return connection(); }
|
||||
@Override public Connection getConnection(String username, String password) { return connection(); }
|
||||
@Override public PrintWriter getLogWriter() { return null; }
|
||||
@Override public void setLogWriter(PrintWriter out) {}
|
||||
@Override public void setLoginTimeout(int seconds) {}
|
||||
@Override public int getLoginTimeout() { return 0; }
|
||||
@Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { throw new SQLFeatureNotSupportedException(); }
|
||||
@Override public <T> T unwrap(Class<T> iface) { throw new UnsupportedOperationException(); }
|
||||
@Override public boolean isWrapperFor(Class<?> iface) { return false; }
|
||||
};
|
||||
}
|
||||
@Test
|
||||
void upsertsByCompositeProtocolPhonePlatePrimaryKey() {
|
||||
DriverManagerDataSource dataSource = new DriverManagerDataSource(
|
||||
"jdbc:h2:mem:mysql-identity-composite;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
|
||||
JdbcTemplate jdbc = new JdbcTemplate(dataSource);
|
||||
jdbc.execute("""
|
||||
CREATE TABLE vehicle_identity_binding (
|
||||
protocol VARCHAR(32) NOT NULL DEFAULT 'ALL',
|
||||
phone VARCHAR(128) NOT NULL DEFAULT '',
|
||||
plate VARCHAR(128) NOT NULL DEFAULT '',
|
||||
vin VARCHAR(64) NOT NULL,
|
||||
province VARCHAR(32) NOT NULL DEFAULT '',
|
||||
city VARCHAR(32) NOT NULL DEFAULT '',
|
||||
maker VARCHAR(64) NOT NULL DEFAULT '',
|
||||
device_type VARCHAR(128) NOT NULL DEFAULT '',
|
||||
plate_color VARCHAR(32) NOT NULL DEFAULT '',
|
||||
metadata_json TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
device_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (protocol, phone, plate)
|
||||
)
|
||||
""");
|
||||
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
|
||||
mysql.setTableName("vehicle_identity_binding");
|
||||
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
|
||||
dataSource,
|
||||
mysql,
|
||||
new ObjectMapper());
|
||||
service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
|
||||
"VIN-A", "13079963289", "DEV-A", "沪A00113F"));
|
||||
|
||||
private Connection connection() {
|
||||
InvocationHandler handler = (proxy, method, args) -> switch (method.getName()) {
|
||||
case "createStatement" -> statement();
|
||||
case "prepareStatement" -> preparedStatement((String) args[0]);
|
||||
case "close" -> null;
|
||||
case "isClosed" -> false;
|
||||
case "unwrap" -> throw new UnsupportedOperationException();
|
||||
case "isWrapperFor" -> false;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
};
|
||||
return (Connection) Proxy.newProxyInstance(
|
||||
Connection.class.getClassLoader(), new Class<?>[]{Connection.class}, handler);
|
||||
}
|
||||
service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
|
||||
"VIN-B", "13079963289", "DEV-B", "沪A00114F"));
|
||||
service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
|
||||
"VIN-B2", "13079963289", "DEV-B2", "沪A00114F"));
|
||||
|
||||
private Statement statement() {
|
||||
InvocationHandler handler = (proxy, method, args) -> {
|
||||
if ("execute".equals(method.getName())) {
|
||||
createdTables.add((String) args[0]);
|
||||
return false;
|
||||
}
|
||||
if ("close".equals(method.getName())) {
|
||||
return null;
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
};
|
||||
return (Statement) Proxy.newProxyInstance(
|
||||
Statement.class.getClassLoader(), new Class<?>[]{Statement.class}, handler);
|
||||
}
|
||||
|
||||
private PreparedStatement preparedStatement(String sql) {
|
||||
List<Object> params = new ArrayList<>();
|
||||
InvocationHandler handler = (proxy, method, args) -> {
|
||||
switch (method.getName()) {
|
||||
case "setString", "setInt", "setObject" -> {
|
||||
set(params, (Integer) args[0], args[1]);
|
||||
return null;
|
||||
}
|
||||
case "executeUpdate" -> {
|
||||
if (sql.startsWith("INSERT INTO vehicle_identity_binding_registration ")) {
|
||||
upsertRegistration(params);
|
||||
} else {
|
||||
upsert(params);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
case "executeQuery" -> {
|
||||
if (sql.startsWith("SELECT plate, vin FROM ")) {
|
||||
loadAllSelects++;
|
||||
return allBindingsResultSet();
|
||||
}
|
||||
if (sql.startsWith("SELECT protocol, identifier_type, identifier_value, vin FROM ")) {
|
||||
loadAllSelects++;
|
||||
return legacyAllBindingsResultSet();
|
||||
}
|
||||
if (sql.startsWith("SELECT r.protocol, r.phone, r.device_id, r.plate, b.vin FROM ")) {
|
||||
return registrationResultSet();
|
||||
}
|
||||
if (sql.startsWith("SELECT protocol, phone, device_id, plate, vin FROM ")) {
|
||||
return legacyRegistrationResultSet();
|
||||
}
|
||||
lookupSelects++;
|
||||
return resultSet(resolve(params));
|
||||
}
|
||||
case "close" -> {
|
||||
return null;
|
||||
}
|
||||
default -> {
|
||||
return defaultValue(method.getReturnType());
|
||||
}
|
||||
}
|
||||
};
|
||||
return (PreparedStatement) Proxy.newProxyInstance(
|
||||
PreparedStatement.class.getClassLoader(), new Class<?>[]{PreparedStatement.class}, handler);
|
||||
}
|
||||
|
||||
private void upsert(List<Object> params) {
|
||||
String plate;
|
||||
String vin;
|
||||
if (params.size() >= 4) {
|
||||
plate = normalize((String) params.get(2));
|
||||
vin = (String) params.get(3);
|
||||
} else {
|
||||
plate = normalize((String) params.get(0));
|
||||
vin = (String) params.get(1);
|
||||
}
|
||||
bindings.put(plate, vin);
|
||||
}
|
||||
|
||||
private void upsertRegistration(List<Object> params) {
|
||||
String protocol = (String) params.get(0);
|
||||
String phone = normalize((String) params.get(1));
|
||||
String vin = (String) params.get(2);
|
||||
String key = protocol + "|" + phone;
|
||||
RegistrationRow existing = registrations.get(key);
|
||||
String effectiveVin = existing != null && "unknown".equalsIgnoreCase(vin) ? existing.vin : vin;
|
||||
registrations.put(key, new RegistrationRow(
|
||||
protocol,
|
||||
phone,
|
||||
effectiveVin,
|
||||
normalize((String) params.get(3)),
|
||||
normalize((String) params.get(4))));
|
||||
}
|
||||
|
||||
private String resolve(List<Object> params) {
|
||||
String protocol = (String) params.get(0);
|
||||
String identifierType = (String) params.get(1);
|
||||
String identifierValue = normalize((String) params.get(2));
|
||||
if (!"PLATE".equals(identifierType)) {
|
||||
return null;
|
||||
}
|
||||
return bindings.get(identifierValue);
|
||||
}
|
||||
|
||||
private ResultSet allBindingsResultSet() {
|
||||
List<Map.Entry<String, String>> entries = new ArrayList<>(bindings.entrySet());
|
||||
InvocationHandler handler = new InvocationHandler() {
|
||||
private int index = -1;
|
||||
|
||||
@Override
|
||||
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
|
||||
return switch (method.getName()) {
|
||||
case "next" -> {
|
||||
index++;
|
||||
yield index < entries.size();
|
||||
}
|
||||
case "getString" -> getColumn(entries.get(index), args[0]);
|
||||
case "close" -> null;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
};
|
||||
}
|
||||
};
|
||||
return (ResultSet) Proxy.newProxyInstance(
|
||||
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
|
||||
}
|
||||
|
||||
private ResultSet legacyAllBindingsResultSet() {
|
||||
List<Map.Entry<String, String>> entries = new ArrayList<>(bindings.entrySet());
|
||||
InvocationHandler handler = new InvocationHandler() {
|
||||
private int index = -1;
|
||||
|
||||
@Override
|
||||
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
|
||||
return switch (method.getName()) {
|
||||
case "next" -> {
|
||||
index++;
|
||||
yield index < entries.size();
|
||||
}
|
||||
case "getString" -> legacyColumn(entries.get(index), args[0]);
|
||||
case "close" -> null;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
};
|
||||
}
|
||||
};
|
||||
return (ResultSet) Proxy.newProxyInstance(
|
||||
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
|
||||
}
|
||||
|
||||
private static String getColumn(Map.Entry<String, String> entry, Object column) {
|
||||
String name = column.toString();
|
||||
return switch (name) {
|
||||
case "plate" -> entry.getKey();
|
||||
case "vin" -> entry.getValue();
|
||||
default -> throw new IllegalArgumentException("unsupported column " + name);
|
||||
};
|
||||
}
|
||||
|
||||
private static String legacyColumn(Map.Entry<String, String> entry, Object column) {
|
||||
return switch (column.toString()) {
|
||||
case "protocol" -> "JT808";
|
||||
case "identifier_type" -> "PLATE";
|
||||
case "identifier_value" -> entry.getKey();
|
||||
case "vin" -> entry.getValue();
|
||||
default -> throw new IllegalArgumentException("unsupported column " + column);
|
||||
};
|
||||
}
|
||||
|
||||
private ResultSet registrationResultSet() {
|
||||
List<RegistrationRow> rows = registrations.values().stream()
|
||||
.filter(row -> bindings.containsKey(row.plate))
|
||||
.map(row -> new RegistrationRow(row.protocol, row.phone, bindings.get(row.plate), row.deviceId, row.plate))
|
||||
.toList();
|
||||
return registrationRowsResultSet(rows);
|
||||
}
|
||||
|
||||
private ResultSet legacyRegistrationResultSet() {
|
||||
List<RegistrationRow> rows = registrations.values().stream()
|
||||
.filter(row -> row.vin != null && !row.vin.isBlank() && !"unknown".equalsIgnoreCase(row.vin))
|
||||
.toList();
|
||||
return registrationRowsResultSet(rows);
|
||||
}
|
||||
|
||||
private ResultSet registrationRowsResultSet(List<RegistrationRow> rows) {
|
||||
InvocationHandler handler = new InvocationHandler() {
|
||||
private int index = -1;
|
||||
|
||||
@Override
|
||||
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
|
||||
return switch (method.getName()) {
|
||||
case "next" -> {
|
||||
index++;
|
||||
yield index < rows.size();
|
||||
}
|
||||
case "getString" -> registrationColumn(rows.get(index), args[0]);
|
||||
case "close" -> null;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
};
|
||||
}
|
||||
};
|
||||
return (ResultSet) Proxy.newProxyInstance(
|
||||
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
|
||||
}
|
||||
|
||||
private static String registrationColumn(RegistrationRow row, Object column) {
|
||||
return switch (column.toString()) {
|
||||
case "protocol" -> row.protocol;
|
||||
case "phone" -> row.phone;
|
||||
case "device_id" -> row.deviceId;
|
||||
case "plate" -> row.plate;
|
||||
case "vin" -> row.vin;
|
||||
default -> throw new IllegalArgumentException("unsupported column " + column);
|
||||
};
|
||||
}
|
||||
|
||||
private static ResultSet resultSet(String vin) {
|
||||
InvocationHandler handler = new InvocationHandler() {
|
||||
private boolean read;
|
||||
|
||||
@Override
|
||||
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
|
||||
return switch (method.getName()) {
|
||||
case "next" -> {
|
||||
boolean hasNext = !read && vin != null;
|
||||
read = true;
|
||||
yield hasNext;
|
||||
}
|
||||
case "getString" -> vin;
|
||||
case "close" -> null;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
};
|
||||
}
|
||||
};
|
||||
return (ResultSet) Proxy.newProxyInstance(
|
||||
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
|
||||
}
|
||||
|
||||
private static void set(List<Object> params, int oneBasedIndex, Object value) {
|
||||
while (params.size() < oneBasedIndex) {
|
||||
params.add(null);
|
||||
}
|
||||
params.set(oneBasedIndex - 1, value);
|
||||
}
|
||||
|
||||
private static String normalize(String value) {
|
||||
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private static Object defaultValue(Class<?> type) {
|
||||
if (type == Void.TYPE) return null;
|
||||
if (type == Boolean.TYPE) return false;
|
||||
if (type == Integer.TYPE) return 0;
|
||||
if (type == Long.TYPE) return 0L;
|
||||
if (type == Double.TYPE) return 0D;
|
||||
if (type == Float.TYPE) return 0F;
|
||||
if (type == Short.TYPE) return (short) 0;
|
||||
if (type == Byte.TYPE) return (byte) 0;
|
||||
return null;
|
||||
}
|
||||
|
||||
private static final class RegistrationRow {
|
||||
private final String protocol;
|
||||
private final String phone;
|
||||
private String vin;
|
||||
private final String deviceId;
|
||||
private final String plate;
|
||||
|
||||
private RegistrationRow(String protocol, String phone, String vin, String deviceId, String plate) {
|
||||
this.protocol = protocol;
|
||||
this.phone = phone;
|
||||
this.vin = vin;
|
||||
this.deviceId = deviceId;
|
||||
this.plate = plate;
|
||||
}
|
||||
}
|
||||
Integer rows = jdbc.queryForObject("""
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_identity_binding
|
||||
WHERE protocol = 'JT808' AND phone = '13079963289'
|
||||
""", Integer.class);
|
||||
assertThat(rows).isEqualTo(2);
|
||||
assertThat(jdbc.queryForObject("""
|
||||
SELECT vin
|
||||
FROM vehicle_identity_binding
|
||||
WHERE protocol = 'JT808' AND phone = '13079963289' AND plate = '沪A00114F'
|
||||
""", String.class)).isEqualTo("VIN-B2");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,14 +7,7 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.lang.reflect.Proxy;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -52,75 +45,19 @@ class VehicleIdentityAutoConfigurationTest {
|
||||
@Test
|
||||
void createsMysqlIdentityServiceWhenConfigured() {
|
||||
runner
|
||||
.withBean(DataSource.class, StubDataSource::new)
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.identity.store=mysql",
|
||||
"lingniu.ingest.identity.mysql.table=vehicle_identity_binding")
|
||||
"lingniu.ingest.identity.mysql.jdbc-url=jdbc:h2:mem:auto-identity;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1",
|
||||
"lingniu.ingest.identity.mysql.username=sa",
|
||||
"lingniu.ingest.identity.mysql.password=",
|
||||
"lingniu.ingest.identity.mysql.table-name=vehicle_identity_bindings")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(VehicleIdentityResolver.class);
|
||||
assertThat(context.getBean(VehicleIdentityResolver.class).getClass().getSimpleName())
|
||||
.isEqualTo("MySqlVehicleIdentityService");
|
||||
assertThat(context).hasSingleBean(VehicleIdentityRegistry.class);
|
||||
assertThat(context.getBean(VehicleIdentityResolver.class))
|
||||
.isSameAs(context.getBean(VehicleIdentityRegistry.class));
|
||||
assertThat(context.getBean(VehicleIdentityResolver.class).getClass().getSimpleName())
|
||||
.isEqualTo("MySqlVehicleIdentityService");
|
||||
});
|
||||
}
|
||||
|
||||
private static final class StubDataSource implements DataSource {
|
||||
@Override public Connection getConnection() { return connection(); }
|
||||
@Override public Connection getConnection(String username, String password) { return connection(); }
|
||||
@Override public java.io.PrintWriter getLogWriter() { return null; }
|
||||
@Override public void setLogWriter(java.io.PrintWriter out) {}
|
||||
@Override public void setLoginTimeout(int seconds) {}
|
||||
@Override public int getLoginTimeout() { return 0; }
|
||||
@Override public java.util.logging.Logger getParentLogger() throws java.sql.SQLFeatureNotSupportedException { throw new java.sql.SQLFeatureNotSupportedException(); }
|
||||
@Override public <T> T unwrap(Class<T> iface) { throw new UnsupportedOperationException(); }
|
||||
@Override public boolean isWrapperFor(Class<?> iface) { return false; }
|
||||
|
||||
private static Connection connection() {
|
||||
return (Connection) Proxy.newProxyInstance(Connection.class.getClassLoader(), new Class<?>[]{Connection.class},
|
||||
(proxy, method, args) -> switch (method.getName()) {
|
||||
case "createStatement" -> statement();
|
||||
case "prepareStatement" -> preparedStatement();
|
||||
case "close" -> null;
|
||||
case "isClosed" -> false;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
});
|
||||
}
|
||||
|
||||
private static Statement statement() {
|
||||
return (Statement) Proxy.newProxyInstance(Statement.class.getClassLoader(), new Class<?>[]{Statement.class},
|
||||
(proxy, method, args) -> switch (method.getName()) {
|
||||
case "execute" -> false;
|
||||
case "close" -> null;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
});
|
||||
}
|
||||
|
||||
private static PreparedStatement preparedStatement() {
|
||||
return (PreparedStatement) Proxy.newProxyInstance(
|
||||
PreparedStatement.class.getClassLoader(), new Class<?>[]{PreparedStatement.class},
|
||||
(proxy, method, args) -> switch (method.getName()) {
|
||||
case "executeQuery" -> emptyResultSet();
|
||||
case "close" -> null;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
});
|
||||
}
|
||||
|
||||
private static ResultSet emptyResultSet() {
|
||||
return (ResultSet) Proxy.newProxyInstance(ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class},
|
||||
(proxy, method, args) -> switch (method.getName()) {
|
||||
case "next" -> false;
|
||||
case "close" -> null;
|
||||
default -> defaultValue(method.getReturnType());
|
||||
});
|
||||
}
|
||||
|
||||
private static Object defaultValue(Class<?> type) {
|
||||
if (type == Void.TYPE) return null;
|
||||
if (type == Boolean.TYPE) return false;
|
||||
if (type == Integer.TYPE) return 0;
|
||||
if (type == Long.TYPE) return 0L;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
</parent>
|
||||
<artifactId>inbound-mqtt</artifactId>
|
||||
<name>inbound-mqtt</name>
|
||||
<description>MQTT 接入模块(支持多 endpoint,宇通等),Fusesource MQTT Client + 双向 TLS。</description>
|
||||
<description>MQTT 接入模块(支持多 endpoint,宇通等),Eclipse Paho MQTT Client + 双向 TLS。</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -34,8 +34,8 @@
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.fusesource.mqtt-client</groupId>
|
||||
<artifactId>mqtt-client</artifactId>
|
||||
<groupId>org.eclipse.paho</groupId>
|
||||
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
|
||||
@@ -11,30 +11,33 @@ import com.lingniu.ingest.identity.VehicleIdentitySource;
|
||||
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
|
||||
import com.lingniu.ingest.inbound.mqtt.model.MqttPayload;
|
||||
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
|
||||
import org.fusesource.mqtt.client.Future;
|
||||
import org.fusesource.mqtt.client.FutureConnection;
|
||||
import org.fusesource.mqtt.client.MQTT;
|
||||
import org.fusesource.mqtt.client.Message;
|
||||
import org.fusesource.mqtt.client.QoS;
|
||||
import org.fusesource.mqtt.client.Topic;
|
||||
import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
|
||||
import org.eclipse.paho.client.mqttv3.MqttAsyncClient;
|
||||
import org.eclipse.paho.client.mqttv3.MqttCallbackExtended;
|
||||
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
|
||||
import org.eclipse.paho.client.mqttv3.MqttException;
|
||||
import org.eclipse.paho.client.mqttv3.MqttMessage;
|
||||
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.EOFException;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* MQTT 多 endpoint 生命周期管理。
|
||||
*
|
||||
* <p>为每个 endpoint 创建一个 fusesource MQTT FutureConnection → 连接 → 订阅 → 接收循环 →
|
||||
* <p>为每个 endpoint 创建一个 Paho 异步客户端 → 连接 → 订阅 → 注册回调 →
|
||||
* 回调里解析 payload → {@link Dispatcher#dispatch(RawFrame)}。
|
||||
*
|
||||
* <p>每个 endpoint 在独立虚拟线程中阻塞接收,业务处理由 Dispatcher 侧的 Disruptor 承接。
|
||||
* <p>MQTT 回调不做业务处理,虚拟线程由 Dispatcher 侧的 Disruptor 承接。
|
||||
*/
|
||||
public final class MqttEndpointManager implements AutoCloseable {
|
||||
|
||||
@@ -44,9 +47,8 @@ public final class MqttEndpointManager implements AutoCloseable {
|
||||
private final MqttProfileRegistry profileRegistry;
|
||||
private final VehicleIdentityResolver identityResolver;
|
||||
private final Dispatcher dispatcher;
|
||||
private final List<FutureConnection> connections = new ArrayList<>();
|
||||
private final List<Thread> workers = new ArrayList<>();
|
||||
private volatile boolean running;
|
||||
private final List<MqttAsyncClient> clients = new ArrayList<>();
|
||||
private boolean started;
|
||||
|
||||
public MqttEndpointManager(MqttInboundProperties props,
|
||||
MqttProfileRegistry profileRegistry,
|
||||
@@ -64,111 +66,78 @@ public final class MqttEndpointManager implements AutoCloseable {
|
||||
this.dispatcher = dispatcher;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
running = true;
|
||||
log.info("mqtt endpoint manager starting, endpoints={}", props.getEndpoints().size());
|
||||
public synchronized void start() {
|
||||
if (started) {
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
if (props.getEndpoints().isEmpty()) {
|
||||
log.warn("mqtt inbound enabled but no endpoints configured");
|
||||
return;
|
||||
}
|
||||
log.info("mqtt inbound starting endpoints={}", props.getEndpoints().size());
|
||||
for (MqttInboundProperties.Endpoint ep : props.getEndpoints()) {
|
||||
Thread worker = Thread.ofVirtual()
|
||||
.name("mqtt-endpoint-" + safeName(ep.getName()))
|
||||
.start(() -> runEndpoint(ep));
|
||||
workers.add(worker);
|
||||
try {
|
||||
// 多 endpoint 互不影响:单个连接初始化失败会派发 operationalError,但不会阻止其他 endpoint。
|
||||
log.info("mqtt endpoint [{}] initializing uri={} topic={} profile={} qos={} cleanSession={} tlsHostnameVerification={}",
|
||||
ep.getName(), ep.getUri(), ep.getTopic(), ep.getProfile(), ep.getQos(), ep.isCleanSession(),
|
||||
ep.getTls() == null || ep.getTls().isHostnameVerificationEnabled());
|
||||
MqttAsyncClient client = buildClient(ep);
|
||||
clients.add(client);
|
||||
connectAndSubscribe(client, ep);
|
||||
} catch (Exception e) {
|
||||
log.error("mqtt endpoint [{}] init failed, skipped", ep.getName(), e);
|
||||
publishOperationalError(ep, "init", e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void runEndpoint(MqttInboundProperties.Endpoint ep) {
|
||||
while (running && !Thread.currentThread().isInterrupted()) {
|
||||
FutureConnection connection = null;
|
||||
try {
|
||||
log.info("mqtt endpoint [{}] initializing uri={} topic={} qos={} tls={}",
|
||||
ep.getName(), ep.getUri(), ep.getTopic(), ep.getQos(), hasCustomTls(ep.getTls()));
|
||||
MQTT mqtt = buildClient(ep);
|
||||
connection = mqtt.futureConnection();
|
||||
connections.add(connection);
|
||||
log.info("mqtt endpoint [{}] connect sending", ep.getName());
|
||||
connection.connect().await();
|
||||
log.info("mqtt endpoint [{}] connected", ep.getName());
|
||||
connection.subscribe(new Topic[]{new Topic(ep.getTopic(), mapQos(ep.getQos()))}).await();
|
||||
log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
|
||||
receiveLoop(ep, connection);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
if (isPeerDisconnect(e)) {
|
||||
log.warn("mqtt endpoint [{}] peer disconnected, reconnecting: {}", ep.getName(), e.getMessage());
|
||||
} else {
|
||||
log.error("mqtt endpoint [{}] connect/receive failed", ep.getName(), e);
|
||||
publishOperationalError(ep, "connect",
|
||||
private MqttAsyncClient buildClient(MqttInboundProperties.Endpoint ep) throws MqttException {
|
||||
String clientId = ep.getClientId() == null || ep.getClientId().isBlank()
|
||||
? "lingniu-" + UUID.randomUUID() : ep.getClientId();
|
||||
return new MqttAsyncClient(pahoServerUri(ep.getUri()), clientId, new MemoryPersistence());
|
||||
}
|
||||
|
||||
private void connectAndSubscribe(MqttAsyncClient client, MqttInboundProperties.Endpoint ep) throws MqttException {
|
||||
client.setCallback(new MqttCallbackExtended() {
|
||||
@Override
|
||||
public void connectComplete(boolean reconnect, String serverURI) {
|
||||
if (!reconnect) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep));
|
||||
log.info("mqtt endpoint [{}] re-subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
|
||||
} catch (Exception e) {
|
||||
log.error("mqtt endpoint [{}] re-subscribe failed topic={}", ep.getName(), ep.getTopic(), e);
|
||||
publishOperationalError(ep, "subscribe",
|
||||
e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
|
||||
}
|
||||
sleepBeforeReconnect(ep);
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
connections.remove(connection);
|
||||
try {
|
||||
connection.disconnect().await();
|
||||
} catch (Exception ignored) {
|
||||
// best-effort close
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private MQTT buildClient(MqttInboundProperties.Endpoint ep) throws Exception {
|
||||
MQTT mqtt = new MQTT();
|
||||
mqtt.setHost(ep.getUri());
|
||||
if (ep.getClientId() != null && !ep.getClientId().isBlank()) {
|
||||
mqtt.setClientId(ep.getClientId());
|
||||
}
|
||||
mqtt.setCleanSession(ep.isCleanSession());
|
||||
if (ep.getUsername() != null && !ep.getUsername().isBlank()) {
|
||||
mqtt.setUserName(ep.getUsername());
|
||||
mqtt.setPassword(ep.getPassword() == null ? "" : ep.getPassword());
|
||||
}
|
||||
mqtt.setConnectAttemptsMax(1);
|
||||
mqtt.setReconnectAttemptsMax(0);
|
||||
if (ep.getKeepAliveSeconds() > 0) {
|
||||
mqtt.setKeepAlive((short) ep.getKeepAliveSeconds());
|
||||
}
|
||||
mqtt.setVersion("3.1.1");
|
||||
if (hasCustomTls(ep.getTls())) {
|
||||
mqtt.setSslContext(MqttTlsSupport.buildSslContext(ep.getTls()));
|
||||
}
|
||||
return mqtt;
|
||||
}
|
||||
|
||||
private void receiveLoop(MqttInboundProperties.Endpoint ep, FutureConnection connection) throws Exception {
|
||||
while (running && !Thread.currentThread().isInterrupted()) {
|
||||
Future<Message> futureMessage = connection.receive();
|
||||
Message message = futureMessage.await();
|
||||
try {
|
||||
message.ack();
|
||||
onMessage(ep, message.getTopic(), message.getPayload());
|
||||
} catch (Exception e) {
|
||||
log.warn("mqtt endpoint [{}] message ack/dispatch failed topic={} len={}",
|
||||
ep.getName(), message.getTopic(), message.getPayload() == null ? 0 : message.getPayload().length, e);
|
||||
@Override
|
||||
public void connectionLost(Throwable cause) {
|
||||
log.warn("mqtt endpoint [{}] connection lost", ep.getName(), cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isPeerDisconnect(Exception e) {
|
||||
Throwable current = e;
|
||||
while (current != null) {
|
||||
if (current instanceof EOFException) {
|
||||
return true;
|
||||
@Override
|
||||
public void messageArrived(String topic, MqttMessage message) {
|
||||
log.info("mqtt endpoint [{}] received topic={} bytes={}", ep.getName(), topic, message.getPayload().length);
|
||||
onMessage(ep, topic, message.getPayload());
|
||||
}
|
||||
current = current.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void sleepBeforeReconnect(MqttInboundProperties.Endpoint ep) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
@Override
|
||||
public void deliveryComplete(IMqttDeliveryToken token) {
|
||||
// inbound only
|
||||
}
|
||||
});
|
||||
|
||||
MqttConnectOptions options = connectOptions(ep);
|
||||
client.connect(options).waitForCompletion(timeoutMs(ep));
|
||||
log.info("mqtt endpoint [{}] connected serverUri={} clientId={}",
|
||||
ep.getName(), client.getServerURI(), client.getClientId());
|
||||
client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep));
|
||||
log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
|
||||
}
|
||||
|
||||
private void onMessage(MqttInboundProperties.Endpoint ep, String topic, byte[] payload) {
|
||||
@@ -319,11 +288,58 @@ public final class MqttEndpointManager implements AutoCloseable {
|
||||
return value.replace("\\", "\\\\").replace("\"", "\\\"");
|
||||
}
|
||||
|
||||
private static QoS mapQos(int level) {
|
||||
private static MqttConnectOptions connectOptions(MqttInboundProperties.Endpoint ep) {
|
||||
MqttConnectOptions options = new MqttConnectOptions();
|
||||
options.setAutomaticReconnect(true);
|
||||
options.setCleanSession(ep.isCleanSession());
|
||||
if (ep.getKeepAliveSeconds() > 0) {
|
||||
options.setKeepAliveInterval(ep.getKeepAliveSeconds());
|
||||
}
|
||||
if (ep.getConnectionTimeoutSeconds() > 0) {
|
||||
options.setConnectionTimeout(ep.getConnectionTimeoutSeconds());
|
||||
}
|
||||
if (hasText(ep.getUsername())) {
|
||||
options.setUserName(ep.getUsername());
|
||||
options.setPassword(ep.getPassword() == null ? new char[0] : ep.getPassword().toCharArray());
|
||||
}
|
||||
if (isSslUri(ep.getUri()) && hasCustomTls(ep.getTls())) {
|
||||
options.setSocketFactory(MqttTlsSupport.socketFactory(ep.getTls()));
|
||||
if (ep.getTls() != null && !ep.getTls().isHostnameVerificationEnabled()) {
|
||||
options.setSSLHostnameVerifier((hostname, session) -> true);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
private static String pahoServerUri(String value) {
|
||||
URI uri = URI.create(value);
|
||||
String scheme = uri.getScheme();
|
||||
String pahoScheme = switch (scheme == null ? "" : scheme.toLowerCase()) {
|
||||
case "ssl", "tls", "mqtts" -> "ssl";
|
||||
default -> "tcp";
|
||||
};
|
||||
int port = uri.getPort() > 0 ? uri.getPort() : defaultPort(scheme);
|
||||
return pahoScheme + "://" + uri.getHost() + ":" + port;
|
||||
}
|
||||
|
||||
private static int mapQos(int level) {
|
||||
return switch (level) {
|
||||
case 0 -> QoS.AT_MOST_ONCE;
|
||||
case 2 -> QoS.EXACTLY_ONCE;
|
||||
default -> QoS.AT_LEAST_ONCE;
|
||||
case 0 -> 0;
|
||||
case 2 -> 2;
|
||||
default -> 1;
|
||||
};
|
||||
}
|
||||
|
||||
private static long timeoutMs(MqttInboundProperties.Endpoint ep) {
|
||||
int seconds = ep.getConnectionTimeoutSeconds() > 0 ? ep.getConnectionTimeoutSeconds() : 10;
|
||||
return TimeUnit.SECONDS.toMillis(seconds);
|
||||
}
|
||||
|
||||
private static int defaultPort(String scheme) {
|
||||
if (scheme == null) return 1883;
|
||||
return switch (scheme.toLowerCase()) {
|
||||
case "ssl", "tls", "mqtts" -> 8883;
|
||||
default -> 1883;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -331,13 +347,6 @@ public final class MqttEndpointManager implements AutoCloseable {
|
||||
return value != null && value.trim().length() == 17;
|
||||
}
|
||||
|
||||
private static String safeName(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return "unnamed";
|
||||
}
|
||||
return value.replaceAll("[^a-zA-Z0-9_.-]", "_");
|
||||
}
|
||||
|
||||
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}
|
||||
|
||||
private static boolean hasCustomTls(MqttInboundProperties.Tls tls) {
|
||||
@@ -345,23 +354,30 @@ public final class MqttEndpointManager implements AutoCloseable {
|
||||
|| (hasText(tls.getClientPem()) && hasText(tls.getClientKey())));
|
||||
}
|
||||
|
||||
private static boolean isSslUri(String value) {
|
||||
String scheme = URI.create(value).getScheme();
|
||||
return scheme != null && switch (scheme.toLowerCase()) {
|
||||
case "ssl", "tls", "mqtts" -> true;
|
||||
default -> false;
|
||||
};
|
||||
}
|
||||
|
||||
private static boolean hasText(String value) {
|
||||
return value != null && !value.isBlank();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
running = false;
|
||||
for (FutureConnection connection : List.copyOf(connections)) {
|
||||
for (MqttAsyncClient c : clients) {
|
||||
try {
|
||||
connection.disconnect().await();
|
||||
if (c.isConnected()) {
|
||||
c.disconnectForcibly(1000, 1000, false);
|
||||
}
|
||||
c.close(true);
|
||||
} catch (Exception ignored) {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
for (Thread worker : workers) {
|
||||
worker.interrupt();
|
||||
}
|
||||
log.info("mqtt endpoint manager stopped, connections={}", connections.size());
|
||||
log.info("mqtt endpoint manager stopped, clients={}", clients.size());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,19 +2,31 @@ package com.lingniu.ingest.inbound.mqtt.client;
|
||||
|
||||
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
|
||||
|
||||
import javax.net.ssl.KeyManager;
|
||||
import javax.net.ssl.KeyManagerFactory;
|
||||
import javax.net.ssl.SSLEngine;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLParameters;
|
||||
import javax.net.ssl.SSLSocket;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import javax.net.ssl.TrustManager;
|
||||
import javax.net.ssl.TrustManagerFactory;
|
||||
import java.io.InputStream;
|
||||
import javax.net.ssl.X509ExtendedTrustManager;
|
||||
import javax.net.ssl.X509TrustManager;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.Socket;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.KeyFactory;
|
||||
import java.security.KeyStore;
|
||||
import java.security.PrivateKey;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.cert.Certificate;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.security.cert.CertificateFactory;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.security.spec.PKCS8EncodedKeySpec;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
@@ -25,93 +37,81 @@ public final class MqttTlsSupport {
|
||||
|
||||
private MqttTlsSupport() {}
|
||||
|
||||
public static SSLContext buildSslContext(MqttInboundProperties.Tls tls) {
|
||||
public static SSLSocketFactory socketFactory(MqttInboundProperties.Tls tls) {
|
||||
try {
|
||||
TrustManagerFactory tmf = null;
|
||||
KeyManagerFactory kmf = null;
|
||||
KeyManager[] keyManagers = null;
|
||||
TrustManager[] trustManagers = null;
|
||||
if (tls != null && hasText(tls.getCaPem())) {
|
||||
tmf = trustManagerFactory(Path.of(tls.getCaPem()));
|
||||
trustManagers = trustManagerFactory(Path.of(tls.getCaPem())).getTrustManagers();
|
||||
if (!tls.isHostnameVerificationEnabled()) {
|
||||
trustManagers = withoutHostnameVerification(trustManagers);
|
||||
}
|
||||
}
|
||||
if (tls != null && hasText(tls.getClientPem()) && hasText(tls.getClientKey())) {
|
||||
kmf = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey()));
|
||||
keyManagers = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey())).getKeyManagers();
|
||||
}
|
||||
SSLContext context = SSLContext.getInstance("TLSv1.2");
|
||||
context.init(
|
||||
kmf == null ? null : kmf.getKeyManagers(),
|
||||
tmf == null ? null : tmf.getTrustManagers(),
|
||||
new SecureRandom());
|
||||
return context;
|
||||
SSLContext context = SSLContext.getInstance("TLS");
|
||||
context.init(keyManagers, trustManagers, null);
|
||||
SSLSocketFactory factory = context.getSocketFactory();
|
||||
if (tls != null && !tls.isHostnameVerificationEnabled()) {
|
||||
return new HostnameVerificationDisablingSocketFactory(factory);
|
||||
}
|
||||
return factory;
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException("mqtt tls config build failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static TrustManagerFactory trustManagerFactory(Path caPath) throws Exception {
|
||||
private static TrustManagerFactory trustManagerFactory(Path caPem) throws Exception {
|
||||
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
|
||||
trustStore.load(null, null);
|
||||
int i = 0;
|
||||
for (Certificate cert : readCertificates(caPath)) {
|
||||
for (Certificate cert : readCertificates(caPem)) {
|
||||
trustStore.setCertificateEntry("ca-" + i++, cert);
|
||||
}
|
||||
TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
|
||||
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
|
||||
tmf.init(trustStore);
|
||||
return tmf;
|
||||
}
|
||||
|
||||
private static KeyManagerFactory keyManagerFactory(Path clientCertPath, Path clientKeyPath) throws Exception {
|
||||
List<Certificate> chain = readCertificates(clientCertPath);
|
||||
PrivateKey key = readPrivateKey(clientKeyPath);
|
||||
private static KeyManagerFactory keyManagerFactory(Path clientPem, Path clientKey) throws Exception {
|
||||
List<Certificate> chain = readCertificates(clientPem);
|
||||
PrivateKey key = readPrivateKey(clientKey);
|
||||
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
|
||||
keyStore.load(null, null);
|
||||
char[] password = new char[0];
|
||||
keyStore.setCertificateEntry("client", chain.getFirst());
|
||||
keyStore.setKeyEntry("clientKey", key, password, chain.toArray(Certificate[]::new));
|
||||
KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX");
|
||||
keyStore.setKeyEntry("mqtt-client", key, password, chain.toArray(Certificate[]::new));
|
||||
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
|
||||
kmf.init(keyStore, password);
|
||||
return kmf;
|
||||
}
|
||||
|
||||
private static List<Certificate> readCertificates(Path path) throws Exception {
|
||||
private static List<Certificate> readCertificates(Path pem) throws Exception {
|
||||
String text = Files.readString(pem, StandardCharsets.UTF_8);
|
||||
List<Certificate> certificates = new ArrayList<>();
|
||||
CertificateFactory factory = CertificateFactory.getInstance("X.509");
|
||||
String text = readUtf8IfPossible(path);
|
||||
if (text != null && text.contains("-----BEGIN CERTIFICATE-----")) {
|
||||
List<Certificate> certificates = new ArrayList<>();
|
||||
for (String block : pemBlocks(text, "CERTIFICATE")) {
|
||||
certificates.add(factory.generateCertificate(
|
||||
new java.io.ByteArrayInputStream(Base64.getMimeDecoder().decode(block))));
|
||||
}
|
||||
if (!certificates.isEmpty()) {
|
||||
return certificates;
|
||||
for (String block : pemBlocks(text, "CERTIFICATE")) {
|
||||
try (ByteArrayInputStream in = new ByteArrayInputStream(Base64.getMimeDecoder().decode(block))) {
|
||||
certificates.add(factory.generateCertificate(in));
|
||||
}
|
||||
}
|
||||
try (InputStream in = Files.newInputStream(path)) {
|
||||
Collection<? extends Certificate> certificates = factory.generateCertificates(in);
|
||||
if (certificates.isEmpty()) {
|
||||
throw new IllegalArgumentException("no certificate found in " + path);
|
||||
}
|
||||
return List.copyOf(certificates);
|
||||
if (certificates.isEmpty()) {
|
||||
throw new IllegalArgumentException("no certificate found in " + pem);
|
||||
}
|
||||
return certificates;
|
||||
}
|
||||
|
||||
private static PrivateKey readPrivateKey(Path path) throws Exception {
|
||||
String text = Files.readString(path, StandardCharsets.UTF_8);
|
||||
private static PrivateKey readPrivateKey(Path pem) throws Exception {
|
||||
String text = Files.readString(pem, StandardCharsets.UTF_8);
|
||||
List<String> blocks = pemBlocks(text, "PRIVATE KEY");
|
||||
if (blocks.isEmpty()) {
|
||||
throw new IllegalArgumentException("no PKCS#8 private key found in " + path);
|
||||
throw new IllegalArgumentException("no PKCS#8 private key found in " + pem);
|
||||
}
|
||||
byte[] der = Base64.getMimeDecoder().decode(blocks.getFirst());
|
||||
PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der);
|
||||
return KeyFactory.getInstance("RSA").generatePrivate(spec);
|
||||
}
|
||||
|
||||
private static String readUtf8IfPossible(Path path) {
|
||||
try {
|
||||
return Files.readString(path, StandardCharsets.UTF_8);
|
||||
} catch (Exception ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static List<String> pemBlocks(String text, String type) {
|
||||
String begin = "-----BEGIN " + type + "-----";
|
||||
String end = "-----END " + type + "-----";
|
||||
@@ -132,4 +132,129 @@ public final class MqttTlsSupport {
|
||||
private static boolean hasText(String value) {
|
||||
return value != null && !value.isBlank();
|
||||
}
|
||||
|
||||
private static TrustManager[] withoutHostnameVerification(TrustManager[] managers) {
|
||||
TrustManager[] wrapped = new TrustManager[managers.length];
|
||||
for (int i = 0; i < managers.length; i++) {
|
||||
TrustManager manager = managers[i];
|
||||
if (manager instanceof X509TrustManager x509) {
|
||||
wrapped[i] = new ChainOnlyTrustManager(x509);
|
||||
} else {
|
||||
wrapped[i] = manager;
|
||||
}
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
private static final class ChainOnlyTrustManager extends X509ExtendedTrustManager {
|
||||
private final X509TrustManager delegate;
|
||||
|
||||
private ChainOnlyTrustManager(X509TrustManager delegate) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket)
|
||||
throws CertificateException {
|
||||
if (delegate instanceof X509ExtendedTrustManager extended) {
|
||||
extended.checkClientTrusted(chain, authType, socket);
|
||||
} else {
|
||||
delegate.checkClientTrusted(chain, authType);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket)
|
||||
throws CertificateException {
|
||||
delegate.checkServerTrusted(chain, authType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
|
||||
throws CertificateException {
|
||||
if (delegate instanceof X509ExtendedTrustManager extended) {
|
||||
extended.checkClientTrusted(chain, authType, engine);
|
||||
} else {
|
||||
delegate.checkClientTrusted(chain, authType);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
|
||||
throws CertificateException {
|
||||
delegate.checkServerTrusted(chain, authType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
|
||||
delegate.checkClientTrusted(chain, authType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
|
||||
delegate.checkServerTrusted(chain, authType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public X509Certificate[] getAcceptedIssuers() {
|
||||
return delegate.getAcceptedIssuers();
|
||||
}
|
||||
}
|
||||
|
||||
private static final class HostnameVerificationDisablingSocketFactory extends SSLSocketFactory {
|
||||
private final SSLSocketFactory delegate;
|
||||
|
||||
private HostnameVerificationDisablingSocketFactory(SSLSocketFactory delegate) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getDefaultCipherSuites() {
|
||||
return delegate.getDefaultCipherSuites();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getSupportedCipherSuites() {
|
||||
return delegate.getSupportedCipherSuites();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException {
|
||||
return prepare(delegate.createSocket(socket, host, port, autoClose));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket() throws IOException {
|
||||
return prepare(delegate.createSocket());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(String host, int port) throws IOException {
|
||||
return prepare(delegate.createSocket(host, port));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException {
|
||||
return prepare(delegate.createSocket(host, port, localHost, localPort));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(InetAddress host, int port) throws IOException {
|
||||
return prepare(delegate.createSocket(host, port));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException {
|
||||
return prepare(delegate.createSocket(address, port, localAddress, localPort));
|
||||
}
|
||||
|
||||
private static Socket prepare(Socket socket) {
|
||||
if (socket instanceof SSLSocket sslSocket) {
|
||||
SSLParameters parameters = sslSocket.getSSLParameters();
|
||||
parameters.setEndpointIdentificationAlgorithm(null);
|
||||
sslSocket.setSSLParameters(parameters);
|
||||
}
|
||||
return socket;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,19 +10,20 @@ import com.lingniu.ingest.inbound.mqtt.parser.MqttPayloadParser;
|
||||
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfile;
|
||||
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
|
||||
import com.lingniu.ingest.inbound.mqtt.profile.YutongMqttProfile;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Conditional;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.core.env.Environment;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@AutoConfiguration
|
||||
@EnableConfigurationProperties(MqttInboundProperties.class)
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.mqtt", name = "enabled", havingValue = "true")
|
||||
@Conditional(MqttInboundEnabledCondition.class)
|
||||
public class MqttInboundAutoConfiguration {
|
||||
|
||||
@Bean
|
||||
@@ -61,25 +62,144 @@ public class MqttInboundAutoConfiguration {
|
||||
return new MqttRealtimeHandler(profileRegistry);
|
||||
}
|
||||
|
||||
@Bean(destroyMethod = "close")
|
||||
@Bean(initMethod = "start", destroyMethod = "close")
|
||||
@Lazy(false)
|
||||
@ConditionalOnMissingBean
|
||||
public MqttEndpointManager mqttEndpointManager(MqttInboundProperties props,
|
||||
MqttProfileRegistry profileRegistry,
|
||||
VehicleIdentityResolver identityResolver,
|
||||
Dispatcher dispatcher) {
|
||||
Dispatcher dispatcher,
|
||||
Environment environment) {
|
||||
applyLegacySingleEndpointFallback(props, environment);
|
||||
return new MqttEndpointManager(props, profileRegistry, identityResolver, dispatcher);
|
||||
}
|
||||
|
||||
/**
|
||||
* Spring Boot 启动完成后触发 MQTT 连接,避免与 Netty / Kafka 启动竞争。
|
||||
*/
|
||||
@Bean
|
||||
public ApplicationRunner mqttStartupRunner(MqttEndpointManager manager) {
|
||||
return new ApplicationRunner() {
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
manager.start();
|
||||
private static void applyLegacySingleEndpointFallback(MqttInboundProperties props, Environment env) {
|
||||
if (!props.getEndpoints().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String uri = firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].uri",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_URI",
|
||||
"MQTT_URI",
|
||||
"YUTONG_MQTT_URI");
|
||||
if (uri.isBlank() || uri.contains("${")) {
|
||||
return;
|
||||
}
|
||||
MqttInboundProperties.Endpoint endpoint = new MqttInboundProperties.Endpoint();
|
||||
endpoint.setName(defaultText(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].name",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_NAME",
|
||||
"MQTT_ENDPOINT_NAME",
|
||||
"YUTONG_MQTT_ENDPOINT_NAME"), "yutong"));
|
||||
endpoint.setUri(uri);
|
||||
endpoint.setTopic(defaultText(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].topic",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TOPIC",
|
||||
"MQTT_TOPIC",
|
||||
"YUTONG_MQTT_TOPIC"), "#"));
|
||||
endpoint.setQos(intValue(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].qos",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_QOS",
|
||||
"MQTT_QOS",
|
||||
"YUTONG_MQTT_QOS"), 1));
|
||||
endpoint.setClientId(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].client-id",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLIENT_ID",
|
||||
"MQTT_CLIENT_ID",
|
||||
"YUTONG_MQTT_CLIENT_ID"));
|
||||
endpoint.setUsername(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].username",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_USERNAME",
|
||||
"MQTT_USERNAME",
|
||||
"YUTONG_MQTT_USERNAME"));
|
||||
endpoint.setPassword(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].password",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_PASSWORD",
|
||||
"MQTT_PASSWORD",
|
||||
"YUTONG_MQTT_PASSWORD"));
|
||||
endpoint.setCleanSession(booleanValue(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].clean-session",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLEAN_SESSION",
|
||||
"MQTT_CLEAN_SESSION",
|
||||
"YUTONG_MQTT_CLEAN_SESSION"), false));
|
||||
endpoint.setKeepAliveSeconds(intValue(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].keep-alive-seconds",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_KEEP_ALIVE_SECONDS",
|
||||
"MQTT_KEEP_ALIVE_SECONDS",
|
||||
"YUTONG_MQTT_KEEP_ALIVE_SECONDS"), 20));
|
||||
endpoint.setConnectionTimeoutSeconds(intValue(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].connection-timeout-seconds",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CONNECTION_TIMEOUT_SECONDS",
|
||||
"MQTT_CONNECTION_TIMEOUT_SECONDS",
|
||||
"YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS"), 10));
|
||||
endpoint.setProfile(defaultText(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].profile",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_PROFILE",
|
||||
"MQTT_PROFILE",
|
||||
"YUTONG_MQTT_PROFILE"), "yutong"));
|
||||
MqttInboundProperties.Tls tls = endpoint.getTls();
|
||||
tls.setCaPem(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].tls.ca-pem",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CA_PEM",
|
||||
"MQTT_TLS_CA_PEM",
|
||||
"YUTONG_MQTT_TLS_CA_PEM"));
|
||||
tls.setClientPem(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].tls.client-pem",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_PEM",
|
||||
"MQTT_TLS_CLIENT_PEM",
|
||||
"YUTONG_MQTT_TLS_CLIENT_PEM"));
|
||||
tls.setClientKey(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].tls.client-key",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_KEY",
|
||||
"MQTT_TLS_CLIENT_KEY",
|
||||
"YUTONG_MQTT_TLS_CLIENT_KEY"));
|
||||
tls.setHostnameVerificationEnabled(booleanValue(firstText(env,
|
||||
"lingniu.ingest.mqtt.endpoints[0].tls.hostname-verification-enabled",
|
||||
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_HOSTNAME_VERIFICATION_ENABLED",
|
||||
"MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED",
|
||||
"YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED"), true));
|
||||
props.setEndpoints(new ArrayList<>(List.of(endpoint)));
|
||||
}
|
||||
|
||||
private static String firstText(Environment env, String... keys) {
|
||||
for (String key : keys) {
|
||||
String value = env.getProperty(key);
|
||||
if (value != null && !value.isBlank() && !value.contains("${")) {
|
||||
return value;
|
||||
}
|
||||
};
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private static String defaultText(String value, String fallback) {
|
||||
return value == null || value.isBlank() ? fallback : value;
|
||||
}
|
||||
|
||||
private static int intValue(String value, int fallback) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return fallback;
|
||||
}
|
||||
try {
|
||||
return Integer.parseInt(value.trim());
|
||||
} catch (NumberFormatException ex) {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean booleanValue(String value, boolean fallback) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return fallback;
|
||||
}
|
||||
String normalized = value.trim().toLowerCase();
|
||||
if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized)
|
||||
|| "on".equals(normalized)) {
|
||||
return true;
|
||||
}
|
||||
if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized)
|
||||
|| "off".equals(normalized)) {
|
||||
return false;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.lingniu.ingest.inbound.mqtt.config;
|
||||
|
||||
import org.springframework.context.annotation.Condition;
|
||||
import org.springframework.context.annotation.ConditionContext;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.core.type.AnnotatedTypeMetadata;
|
||||
|
||||
final class MqttInboundEnabledCondition implements Condition {
|
||||
|
||||
@Override
|
||||
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
|
||||
Environment env = context.getEnvironment();
|
||||
Boolean direct = parseBoolean(env.getProperty("lingniu.ingest.mqtt.enabled"));
|
||||
if (direct != null) {
|
||||
return direct;
|
||||
}
|
||||
return isTrue(env.getProperty("MQTT_ENABLED"))
|
||||
|| isTrue(env.getProperty("YUTONG_MQTT_ENABLED"));
|
||||
}
|
||||
|
||||
private static boolean isTrue(String value) {
|
||||
Boolean parsed = parseBoolean(value);
|
||||
return parsed != null && parsed;
|
||||
}
|
||||
|
||||
private static Boolean parseBoolean(String value) {
|
||||
if (value == null || value.isBlank() || value.contains("${")) {
|
||||
return null;
|
||||
}
|
||||
String normalized = value.trim().toLowerCase();
|
||||
if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized)
|
||||
|| "on".equals(normalized)) {
|
||||
return true;
|
||||
}
|
||||
if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized)
|
||||
|| "off".equals(normalized)) {
|
||||
return false;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -82,6 +82,8 @@ public class MqttInboundProperties {
|
||||
public String getClientKey() { return clientKey; }
|
||||
public void setClientKey(String clientKey) { this.clientKey = clientKey; }
|
||||
public boolean isHostnameVerificationEnabled() { return hostnameVerificationEnabled; }
|
||||
public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) { this.hostnameVerificationEnabled = hostnameVerificationEnabled; }
|
||||
public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) {
|
||||
this.hostnameVerificationEnabled = hostnameVerificationEnabled;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@ class MqttEndpointManagerTest {
|
||||
.containsEntry("endpoint", "bad-endpoint")
|
||||
.containsEntry("profile", "yutong")
|
||||
.containsEntry("operationalError", "true")
|
||||
.containsEntry("phase", "connect")
|
||||
.containsEntry("phase", "init")
|
||||
.containsEntry("identityResolved", "false")
|
||||
.containsEntry("identitySource", "UNKNOWN");
|
||||
});
|
||||
@@ -184,7 +184,7 @@ class MqttEndpointManagerTest {
|
||||
.containsEntry("operationalError", "true")
|
||||
.containsEntry("endpoint", "bad-endpoint")
|
||||
.containsEntry("profile", "yutong")
|
||||
.containsEntry("phase", "connect");
|
||||
.containsEntry("phase", "init");
|
||||
});
|
||||
|
||||
batchExecutor.close();
|
||||
|
||||
@@ -5,11 +5,10 @@ import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class MqttTlsSupportTest {
|
||||
@@ -24,9 +23,9 @@ class MqttTlsSupportTest {
|
||||
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
|
||||
tls.setCaPem(ca.toString());
|
||||
|
||||
SSLContext config = MqttTlsSupport.buildSslContext(tls);
|
||||
SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls);
|
||||
|
||||
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
|
||||
assertThat(factory).isNotNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -42,22 +41,9 @@ class MqttTlsSupportTest {
|
||||
tls.setClientPem(cert.toString());
|
||||
tls.setClientKey(key.toString());
|
||||
|
||||
SSLContext config = MqttTlsSupport.buildSslContext(tls);
|
||||
SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls);
|
||||
|
||||
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void hostnameVerificationFlagDoesNotBlockSslContextCreation() throws Exception {
|
||||
Path ca = tempDir.resolve("ca.pem");
|
||||
Files.writeString(ca, CERT);
|
||||
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
|
||||
tls.setCaPem(ca.toString());
|
||||
tls.setHostnameVerificationEnabled(false);
|
||||
|
||||
SSLContext config = MqttTlsSupport.buildSslContext(tls);
|
||||
|
||||
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
|
||||
assertThat(factory).isNotNull();
|
||||
}
|
||||
|
||||
private static final String CERT = """
|
||||
|
||||
@@ -41,6 +41,30 @@
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-all</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-buffer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-transport</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-resolver</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-codec</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-handler</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.gps31.push.netty.PushClient;
|
||||
import com.gps31.push.netty.PushMsg;
|
||||
import com.gps31.push.netty.client.TcpClient;
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
import com.lingniu.ingest.api.event.RawArchiveKeys;
|
||||
import com.lingniu.ingest.api.pipeline.RawFrame;
|
||||
import com.lingniu.ingest.core.dispatcher.Dispatcher;
|
||||
import com.lingniu.ingest.identity.InMemoryVehicleIdentityService;
|
||||
@@ -220,6 +221,7 @@ public class XindaPushClient extends PushClient {
|
||||
meta.put("parseError", "true");
|
||||
meta.put("parseErrorMessage", parsed.errorMessage());
|
||||
}
|
||||
meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(payload.command(), json, parsed));
|
||||
dispatcher.dispatch(new RawFrame(
|
||||
ProtocolId.XINDA_PUSH,
|
||||
payload.commandCode(),
|
||||
@@ -247,6 +249,8 @@ public class XindaPushClient extends PushClient {
|
||||
meta.put("operationalError", "true");
|
||||
meta.put("phase", phase == null ? "" : phase);
|
||||
meta.put("reason", reason == null ? "" : reason);
|
||||
meta.put(RawArchiveKeys.META_PARSED_JSON,
|
||||
parsedJson(payload.command(), payload.json(), new JsonParseResult(json, false, "")));
|
||||
dispatcher.dispatch(new RawFrame(
|
||||
ProtocolId.XINDA_PUSH,
|
||||
payload.commandCode(),
|
||||
@@ -271,6 +275,8 @@ public class XindaPushClient extends PushClient {
|
||||
meta.put("operationalError", "true");
|
||||
meta.put("phase", "message");
|
||||
meta.put("reason", reason == null ? "" : reason);
|
||||
JsonParseResult parsed = parseForMetadata(json);
|
||||
meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(cmd, json, parsed));
|
||||
dispatcher.dispatch(new RawFrame(
|
||||
ProtocolId.XINDA_PUSH,
|
||||
0,
|
||||
@@ -388,6 +394,19 @@ public class XindaPushClient extends PushClient {
|
||||
}
|
||||
}
|
||||
|
||||
private static String parsedJson(String command, String rawJson, JsonParseResult parsed) {
|
||||
JSONObject root = new JSONObject(true);
|
||||
root.put("command", command == null ? "" : command);
|
||||
root.put("parseError", parsed != null && parsed.parseError());
|
||||
if (parsed != null && parsed.parseError()) {
|
||||
root.put("parseErrorMessage", parsed.errorMessage() == null ? "" : parsed.errorMessage());
|
||||
root.put("rawJson", rawJson == null ? "" : rawJson);
|
||||
} else {
|
||||
root.put("body", parsed == null || parsed.json() == null ? new JSONObject() : parsed.json());
|
||||
}
|
||||
return root.toJSONString();
|
||||
}
|
||||
|
||||
private record JsonParseResult(JSONObject json, boolean parseError, String errorMessage) {}
|
||||
|
||||
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}
|
||||
|
||||
@@ -185,7 +185,6 @@ public final class XindaPushEventMapper implements EventMapper<XindaPushMessage>
|
||||
Map<String, String> out = new java.util.LinkedHashMap<>();
|
||||
out.put("source", "xinda-push");
|
||||
out.put("cmd", message.command());
|
||||
out.put("rawJson", message.json());
|
||||
out.put("vin", normalizedVin(identity));
|
||||
out.put("externalVin", firstNonBlank(json, "vin"));
|
||||
out.put("phone", phone(json));
|
||||
|
||||
@@ -37,7 +37,8 @@ class XindaPushEventMapperTest {
|
||||
.containsEntry("externalVin", "LNVIN000000000101")
|
||||
.containsEntry("phone", "13800138000")
|
||||
.containsEntry("deviceId", "XD-101")
|
||||
.containsEntry("plateNo", "粤B10101");
|
||||
.containsEntry("plateNo", "粤B10101")
|
||||
.doesNotContainKey("rawJson");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -43,7 +43,7 @@ class Gb32960ChannelHandlerAckBoundaryTest {
|
||||
dispatch.awaitPublishCount(1);
|
||||
dispatch.completeDurability();
|
||||
|
||||
ByteBuf ack = readOutboundAfterRunningTasks(channel);
|
||||
ByteBuf ack = awaitOutbound(channel);
|
||||
assertThat(ack).isNotNull();
|
||||
assertThat(ack.getUnsignedByte(2)).isEqualTo((short) 0x02);
|
||||
assertThat(ack.getUnsignedByte(3)).isEqualTo((short) 0x01);
|
||||
@@ -85,8 +85,8 @@ class Gb32960ChannelHandlerAckBoundaryTest {
|
||||
|
||||
dispatch.completeDurability(0);
|
||||
|
||||
ByteBuf firstAck = readOutboundAfterRunningTasks(channel);
|
||||
ByteBuf secondAck = readOutboundAfterRunningTasks(channel);
|
||||
ByteBuf firstAck = awaitOutbound(channel);
|
||||
ByteBuf secondAck = awaitOutbound(channel);
|
||||
assertThat(firstAck).isNotNull();
|
||||
assertThat(secondAck).isNotNull();
|
||||
assertThat(firstAck.getUnsignedByte(2)).isEqualTo((short) 0x02);
|
||||
@@ -97,42 +97,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void decodeFailureIsArchivedWithParseErrorMetadata() {
|
||||
try (DispatchHarness dispatch = new DispatchHarness()) {
|
||||
EmbeddedChannel channel = new EmbeddedChannel(handlerWith(dispatch.dispatcher()));
|
||||
byte[] malformed = new byte[]{0x23, 0x23, 0x02};
|
||||
|
||||
assertThat(channel.writeInbound(malformed)).isFalse();
|
||||
|
||||
dispatch.awaitPublishCount(1);
|
||||
VehicleEvent event = dispatch.event(0);
|
||||
assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class);
|
||||
VehicleEvent.RawArchive raw = (VehicleEvent.RawArchive) event;
|
||||
assertThat(raw.source()).isEqualTo(com.lingniu.ingest.api.ProtocolId.GB32960);
|
||||
assertThat(raw.command()).isZero();
|
||||
assertThat(raw.rawBytes()).containsExactly(malformed);
|
||||
assertThat(raw.metadata())
|
||||
.containsEntry("vin", "unknown")
|
||||
.containsEntry("parseError", "true")
|
||||
.containsKey("parseErrorMessage");
|
||||
channel.finishAndReleaseAll();
|
||||
}
|
||||
}
|
||||
|
||||
private static ByteBuf readOutboundAfterRunningTasks(EmbeddedChannel channel) {
|
||||
long deadline = System.currentTimeMillis() + 3000;
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
channel.runPendingTasks();
|
||||
ByteBuf outbound = channel.readOutbound();
|
||||
if (outbound != null) {
|
||||
return outbound;
|
||||
}
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Gb32960ChannelHandler handlerWith(Dispatcher dispatcher) {
|
||||
return new Gb32960ChannelHandler(
|
||||
decoder(),
|
||||
@@ -174,6 +138,19 @@ class Gb32960ChannelHandlerAckBoundaryTest {
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private static ByteBuf awaitOutbound(EmbeddedChannel channel) {
|
||||
long deadline = System.currentTimeMillis() + 3000;
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
channel.runPendingTasks();
|
||||
ByteBuf outbound = channel.readOutbound();
|
||||
if (outbound != null) {
|
||||
return outbound;
|
||||
}
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static final class DispatchHarness implements AutoCloseable {
|
||||
private final ControlledSink sink = new ControlledSink();
|
||||
private final DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
|
||||
@@ -214,12 +191,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
|
||||
throw new AssertionError("expected " + count + " sink publishes, actual=" + sink.futures.size());
|
||||
}
|
||||
|
||||
VehicleEvent event(int index) {
|
||||
synchronized (sink.futures) {
|
||||
return sink.events.get(index);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
batchExecutor.close();
|
||||
@@ -229,7 +200,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
|
||||
|
||||
private static final class ControlledSink implements EventSink {
|
||||
private final List<CompletableFuture<Void>> futures = new ArrayList<>();
|
||||
private final List<VehicleEvent> events = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
@@ -240,7 +210,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
|
||||
public CompletableFuture<Void> publish(VehicleEvent event) {
|
||||
CompletableFuture<Void> future = new CompletableFuture<>();
|
||||
synchronized (futures) {
|
||||
events.add(event);
|
||||
futures.add(future);
|
||||
}
|
||||
return future;
|
||||
|
||||
@@ -9,13 +9,14 @@ import com.lingniu.ingest.identity.VehicleIdentityLookup;
|
||||
import com.lingniu.ingest.identity.VehicleIdentityRegistry;
|
||||
import com.lingniu.ingest.identity.VehicleIdentityResolver;
|
||||
import com.lingniu.ingest.identity.VehicleIdentitySource;
|
||||
import com.lingniu.ingest.identity.VehicleRegistrationBinding;
|
||||
import com.lingniu.ingest.protocol.jt808.codec.Jt808FrameEncoder;
|
||||
import com.lingniu.ingest.protocol.jt808.codec.Jt808MalformedFrame;
|
||||
import com.lingniu.ingest.protocol.jt808.codec.Jt808MessageDecoder;
|
||||
import com.lingniu.ingest.protocol.jt808.downlink.Jt808Commands;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808Body;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808Header;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationAdditionalInfo;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationFlagInfo;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808Message;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808MessageId;
|
||||
import com.lingniu.ingest.protocol.jt808.session.Jt808ChannelRegistry;
|
||||
@@ -25,17 +26,18 @@ import com.lingniu.ingest.session.SessionStore;
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import io.netty.util.AttributeKey;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.Instant;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
/**
|
||||
* JT808 Netty 入站处理器。职责:
|
||||
@@ -51,8 +53,6 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(Jt808ChannelHandler.class);
|
||||
private static final String PROCESSING_ERROR_PREFIX = "processing failed: ";
|
||||
private static final AttributeKey<CompletableFuture<Void>> DURABLE_ACK_CHAIN_ATTR =
|
||||
AttributeKey.valueOf("jt808.durableAckChain");
|
||||
|
||||
private final Jt808MessageDecoder decoder;
|
||||
private final Dispatcher dispatcher;
|
||||
@@ -123,21 +123,29 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
// ====== 会话维护 & 同步应答匹配 ======
|
||||
String phone = msg.header().phone();
|
||||
IdentityResolution identity = resolveIdentity(msg);
|
||||
byte[] durableAck = null;
|
||||
if (msg.body() instanceof Jt808Body.Register reg) {
|
||||
identity = registerIdentity(msg, reg, identity);
|
||||
}
|
||||
byte[] registerAck = null;
|
||||
switch (msg.header().messageId()) {
|
||||
case Jt808MessageId.TERMINAL_REGISTER -> {
|
||||
DeviceSession session = upsertSession(ctx, msg, identity.identity());
|
||||
channelRegistry.bind(phone, ctx.channel());
|
||||
durableAck = registerAckFrame(phone, msg.header().serialNo(), session.token());
|
||||
var command = Jt808Commands.registerAck(msg.header().serialNo(), 0, session.token());
|
||||
registerAck = Jt808FrameEncoder.encode(
|
||||
command.messageId(),
|
||||
phone,
|
||||
channelRegistry.nextSerial(phone),
|
||||
command.body());
|
||||
}
|
||||
case Jt808MessageId.TERMINAL_AUTH -> {
|
||||
upsertSession(ctx, msg, identity.identity());
|
||||
channelRegistry.bind(phone, ctx.channel());
|
||||
durableAck = terminalAckFrame(phone, msg.header().serialNo(), msg.header().messageId(), 0);
|
||||
ackTerminal(ctx, phone, msg.header().serialNo(), msg.header().messageId(), 0);
|
||||
}
|
||||
case Jt808MessageId.TERMINAL_HEARTBEAT -> {
|
||||
channelRegistry.bind(phone, ctx.channel());
|
||||
durableAck = terminalAckFrame(phone, msg.header().serialNo(), msg.header().messageId(), 0);
|
||||
ackTerminal(ctx, phone, msg.header().serialNo(), msg.header().messageId(), 0);
|
||||
}
|
||||
case Jt808MessageId.TERMINAL_GENERAL_RESPONSE -> {
|
||||
// body: ackSerial(2) + ackMsgId(2) + result(1)
|
||||
@@ -153,7 +161,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
}
|
||||
|
||||
// ====== 派发到 Dispatcher ======
|
||||
Map<String, String> meta = new HashMap<>(6);
|
||||
Map<String, String> meta = new HashMap<>(32);
|
||||
meta.put("vin", identity.identity().vin());
|
||||
meta.put("phone", phone);
|
||||
meta.put("identityResolved", Boolean.toString(identity.identity().resolved()));
|
||||
@@ -163,8 +171,11 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
meta.put("identityErrorMessage", identity.errorMessage());
|
||||
}
|
||||
meta.put("seq", Integer.toString(msg.header().serialNo()));
|
||||
meta.put("messageId", "0x" + Jt808MessageId.hex(msg.header().messageId()));
|
||||
meta.put("messageName", Jt808MessageId.nameOf(msg.header().messageId()));
|
||||
meta.put("reportType", Jt808MessageId.reportTypeOf(msg.header().messageId()));
|
||||
putBodyMetadata(meta, msg.body());
|
||||
meta.put("peer", addr(ctx));
|
||||
addRegistrationMetadata(meta, msg.body());
|
||||
RawFrame rf = new RawFrame(
|
||||
ProtocolId.JT808,
|
||||
msg.header().messageId(),
|
||||
@@ -173,63 +184,19 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
frame,
|
||||
meta,
|
||||
Instant.now());
|
||||
if (durableAck == null) {
|
||||
dispatcher.dispatch(rf);
|
||||
if (registerAck != null) {
|
||||
byte[] ack = registerAck;
|
||||
dispatcher.dispatchAndAwait(rf)
|
||||
.thenRun(() -> ctx.writeAndFlush(Unpooled.wrappedBuffer(ack)))
|
||||
.exceptionally(ex -> {
|
||||
log.warn("[jt808] register ack withheld because dispatch durability failed peer={} phone={} seq={} error={}",
|
||||
addr(ctx), phone, msg.header().serialNo(),
|
||||
ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
|
||||
return null;
|
||||
});
|
||||
return;
|
||||
}
|
||||
enqueueDurableAck(ctx, phone, msg.header().messageId(), dispatcher.dispatchAndAwait(rf), durableAck);
|
||||
}
|
||||
|
||||
private void enqueueDurableAck(ChannelHandlerContext ctx,
|
||||
String phone,
|
||||
int messageId,
|
||||
CompletableFuture<Void> dispatchFuture,
|
||||
byte[] ackFrame) {
|
||||
CompletableFuture<Void> previous = ctx.channel().attr(DURABLE_ACK_CHAIN_ATTR).get();
|
||||
if (previous == null) {
|
||||
previous = CompletableFuture.completedFuture(null);
|
||||
}
|
||||
CompletableFuture<Void> next = previous
|
||||
.handle((ignored, previousFailure) -> null)
|
||||
.thenCompose(ignored -> dispatchFuture.handle((ok, failure) -> failure))
|
||||
.thenCompose(failure -> runOnEventLoop(ctx,
|
||||
() -> handleDurableAckResult(ctx, phone, messageId, failure, ackFrame)));
|
||||
ctx.channel().attr(DURABLE_ACK_CHAIN_ATTR).set(next);
|
||||
}
|
||||
|
||||
private CompletableFuture<Void> runOnEventLoop(ChannelHandlerContext ctx, Runnable task) {
|
||||
CompletableFuture<Void> done = new CompletableFuture<>();
|
||||
Runnable wrapped = () -> {
|
||||
try {
|
||||
task.run();
|
||||
done.complete(null);
|
||||
} catch (Throwable t) {
|
||||
done.completeExceptionally(t);
|
||||
}
|
||||
};
|
||||
if (ctx.executor().inEventLoop()) {
|
||||
wrapped.run();
|
||||
} else {
|
||||
ctx.executor().execute(wrapped);
|
||||
}
|
||||
return done;
|
||||
}
|
||||
|
||||
private void handleDurableAckResult(ChannelHandlerContext ctx,
|
||||
String phone,
|
||||
int messageId,
|
||||
Throwable failure,
|
||||
byte[] ackFrame) {
|
||||
if (!ctx.channel().isActive()) {
|
||||
return;
|
||||
}
|
||||
if (failure != null) {
|
||||
log.warn("[jt808] dispatch failed before ack peer={} phone={} msgId=0x{}",
|
||||
addr(ctx), phone, Integer.toHexString(messageId), failure);
|
||||
ctx.close();
|
||||
return;
|
||||
}
|
||||
ctx.writeAndFlush(Unpooled.wrappedBuffer(ackFrame));
|
||||
dispatcher.dispatch(rf);
|
||||
}
|
||||
|
||||
private IdentityResolution resolveIdentity(Jt808Message msg) {
|
||||
@@ -267,6 +234,28 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
return new VehicleIdentity("unknown", false, identity.source());
|
||||
}
|
||||
|
||||
private IdentityResolution registerIdentity(Jt808Message msg,
|
||||
Jt808Body.Register reg,
|
||||
IdentityResolution current) {
|
||||
VehicleIdentity currentIdentity = current.identity();
|
||||
boolean alreadyResolved = currentIdentity != null && currentIdentity.resolved();
|
||||
String vin = alreadyResolved ? currentIdentity.vin() : generatedInternalVin(msg.header().phone(), reg);
|
||||
identityRegistry.bind(new VehicleIdentityBinding(
|
||||
ProtocolId.JT808,
|
||||
vin,
|
||||
msg.header().phone(),
|
||||
reg.deviceId(),
|
||||
reg.plate(),
|
||||
Map.of(
|
||||
"province", Integer.toString(reg.province()),
|
||||
"city", Integer.toString(reg.city()),
|
||||
"maker", reg.maker(),
|
||||
"deviceType", reg.deviceType(),
|
||||
"plateColor", Integer.toString(reg.plateColor()))));
|
||||
VehicleIdentitySource source = alreadyResolved ? currentIdentity.source() : VehicleIdentitySource.REGISTERED;
|
||||
return new IdentityResolution(new VehicleIdentity(vin, true, source), current.errorMessage());
|
||||
}
|
||||
|
||||
private void dispatchFrameError(ChannelHandlerContext ctx, Jt808MalformedFrame malformed) {
|
||||
String peer = malformed.peer() == null || malformed.peer().isBlank() ? addr(ctx) : malformed.peer();
|
||||
dispatchMalformed(peer, malformed.rawBytes(), malformed.reason(), true);
|
||||
@@ -283,7 +272,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
new Jt808Header(0, frame == null ? 0 : frame.length, 0, false,
|
||||
Jt808Header.ProtocolVersion.V2013, "unknown", 0, 0, 0),
|
||||
new Jt808Body.Malformed(frame == null ? new byte[0] : frame, peer, message));
|
||||
Map<String, String> meta = new HashMap<>(4);
|
||||
Map<String, String> meta = new HashMap<>(8);
|
||||
meta.put("vin", "unknown");
|
||||
meta.put("peer", peer);
|
||||
meta.put("identityResolved", "false");
|
||||
@@ -295,6 +284,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
meta.put("parseError", "true");
|
||||
meta.put("parseErrorMessage", message);
|
||||
}
|
||||
putBodyMetadata(meta, malformed.body());
|
||||
dispatcher.dispatch(new RawFrame(
|
||||
ProtocolId.JT808,
|
||||
0,
|
||||
@@ -312,15 +302,19 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
msg.header().version(), msg.header().phone(), msg.header().serialNo(), 0, 0),
|
||||
new Jt808Body.Malformed(frame == null ? new byte[0] : frame, addr(ctx),
|
||||
PROCESSING_ERROR_PREFIX + message));
|
||||
Map<String, String> meta = new HashMap<>(8);
|
||||
Map<String, String> meta = new HashMap<>(16);
|
||||
meta.put("vin", "unknown");
|
||||
meta.put("phone", msg.header().phone());
|
||||
meta.put("peer", addr(ctx));
|
||||
meta.put("seq", Integer.toString(msg.header().serialNo()));
|
||||
meta.put("messageId", "0x" + Jt808MessageId.hex(msg.header().messageId()));
|
||||
meta.put("messageName", Jt808MessageId.nameOf(msg.header().messageId()));
|
||||
meta.put("reportType", Jt808MessageId.reportTypeOf(msg.header().messageId()));
|
||||
meta.put("identityResolved", "false");
|
||||
meta.put("identitySource", VehicleIdentitySource.UNKNOWN.name());
|
||||
meta.put("processingError", "true");
|
||||
meta.put("processingErrorMessage", message);
|
||||
putBodyMetadata(meta, failed.body());
|
||||
dispatcher.dispatch(new RawFrame(
|
||||
ProtocolId.JT808,
|
||||
0,
|
||||
@@ -343,25 +337,6 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
Instant.now(), Instant.now(),
|
||||
Map.of("protocolVersion", msg.header().version().name())));
|
||||
session = session.withVin(identity.vin());
|
||||
if (msg.body() instanceof Jt808Body.Register reg) {
|
||||
identityRegistry.register(new VehicleRegistrationBinding(
|
||||
ProtocolId.JT808,
|
||||
session.vin(),
|
||||
phone,
|
||||
reg.deviceId(),
|
||||
reg.plate(),
|
||||
reg.province(),
|
||||
reg.city(),
|
||||
reg.maker(),
|
||||
reg.deviceType(),
|
||||
reg.plateColor()));
|
||||
if (identity.resolved() && session.vin() != null
|
||||
&& !session.vin().isBlank()
|
||||
&& !"unknown".equalsIgnoreCase(session.vin())) {
|
||||
identityRegistry.bind(new VehicleIdentityBinding(
|
||||
ProtocolId.JT808, session.vin(), phone, reg.deviceId(), reg.plate()));
|
||||
}
|
||||
}
|
||||
sessions.put(session);
|
||||
return session;
|
||||
}
|
||||
@@ -380,37 +355,175 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
|
||||
return body instanceof Jt808Body.Register reg ? reg.plate() : "";
|
||||
}
|
||||
|
||||
private static void addRegistrationMetadata(Map<String, String> meta, Jt808Body body) {
|
||||
if (!(body instanceof Jt808Body.Register reg)) {
|
||||
return;
|
||||
private static void putBodyMetadata(Map<String, String> meta, Jt808Body body) {
|
||||
switch (body) {
|
||||
case Jt808Body.Register reg -> {
|
||||
put(meta, "body.province", reg.province());
|
||||
put(meta, "body.city", reg.city());
|
||||
put(meta, "body.maker", reg.maker());
|
||||
put(meta, "body.deviceType", reg.deviceType());
|
||||
put(meta, "body.deviceId", reg.deviceId());
|
||||
put(meta, "body.plateColor", reg.plateColor());
|
||||
put(meta, "body.plate", reg.plate());
|
||||
}
|
||||
case Jt808Body.Auth auth -> {
|
||||
put(meta, "body.token", auth.token());
|
||||
put(meta, "body.imei", auth.imei());
|
||||
put(meta, "body.softwareVersion", auth.softwareVersion());
|
||||
}
|
||||
case Jt808Body.Location location -> putLocationMetadata(meta, "body.", location);
|
||||
case Jt808Body.LocationBatch batch -> {
|
||||
put(meta, "body.batchType", batch.batchType());
|
||||
put(meta, "body.locationCount", batch.locations() == null ? 0 : batch.locations().size());
|
||||
}
|
||||
case Jt808Body.ParamsReport report -> {
|
||||
put(meta, "body.responseSerialNo", report.responseSerialNo());
|
||||
put(meta, "body.parameterCount", report.parameters() == null ? 0 : report.parameters().size());
|
||||
if (report.parameters() != null) {
|
||||
report.parameters().forEach((id, bytes) ->
|
||||
put(meta, "body.param.0x" + String.format("%08X", id), toHex(bytes)));
|
||||
}
|
||||
}
|
||||
case Jt808Body.TerminalAttrs attrs -> {
|
||||
put(meta, "body.terminalType", attrs.terminalType());
|
||||
put(meta, "body.maker", attrs.maker());
|
||||
put(meta, "body.terminalModel", attrs.terminalModel());
|
||||
put(meta, "body.terminalId", attrs.terminalId());
|
||||
put(meta, "body.iccid", attrs.iccid());
|
||||
put(meta, "body.hardwareVersion", attrs.hardwareVersion());
|
||||
put(meta, "body.firmwareVersion", attrs.firmwareVersion());
|
||||
put(meta, "body.gnssProperty", attrs.gnssProperty());
|
||||
put(meta, "body.commProperty", attrs.commProperty());
|
||||
}
|
||||
case Jt808Body.MediaEvent event -> {
|
||||
put(meta, "body.mediaId", event.mediaId());
|
||||
put(meta, "body.mediaType", event.mediaType());
|
||||
put(meta, "body.mediaFormat", event.mediaFormat());
|
||||
put(meta, "body.eventCode", event.eventCode());
|
||||
put(meta, "body.channelId", event.channelId());
|
||||
}
|
||||
case Jt808Body.MediaUpload upload -> {
|
||||
put(meta, "body.mediaId", upload.mediaId());
|
||||
put(meta, "body.mediaType", upload.mediaType());
|
||||
put(meta, "body.mediaFormat", upload.mediaFormat());
|
||||
put(meta, "body.eventCode", upload.eventCode());
|
||||
put(meta, "body.channelId", upload.channelId());
|
||||
put(meta, "body.dataSizeBytes", upload.data() == null ? 0 : upload.data().length);
|
||||
if (upload.location() != null) {
|
||||
putLocationMetadata(meta, "body.location.", upload.location());
|
||||
}
|
||||
}
|
||||
case Jt808Body.MediaAttributes attrs -> {
|
||||
put(meta, "body.inputAudioEncoding", attrs.inputAudioEncoding());
|
||||
put(meta, "body.inputAudioChannels", attrs.inputAudioChannels());
|
||||
put(meta, "body.inputAudioSampleRate", attrs.inputAudioSampleRate());
|
||||
put(meta, "body.inputAudioBitDepth", attrs.inputAudioBitDepth());
|
||||
put(meta, "body.audioFrameLength", attrs.audioFrameLength());
|
||||
put(meta, "body.audioOutputSupported", attrs.audioOutputSupported());
|
||||
put(meta, "body.videoEncoding", attrs.videoEncoding());
|
||||
put(meta, "body.maxAudioChannels", attrs.maxAudioChannels());
|
||||
put(meta, "body.maxVideoChannels", attrs.maxVideoChannels());
|
||||
put(meta, "body.rawSizeBytes", attrs.raw() == null ? 0 : attrs.raw().length);
|
||||
}
|
||||
case Jt808Body.FileUploadComplete complete -> {
|
||||
put(meta, "body.responseSerialNo", complete.responseSerialNo());
|
||||
put(meta, "body.result", complete.result());
|
||||
put(meta, "body.rawSizeBytes", complete.raw() == null ? 0 : complete.raw().length);
|
||||
}
|
||||
case Jt808Body.FileList files -> {
|
||||
put(meta, "body.responseSerialNo", files.responseSerialNo());
|
||||
put(meta, "body.fileCount", files.fileCount());
|
||||
put(meta, "body.rawSizeBytes", files.raw() == null ? 0 : files.raw().length);
|
||||
}
|
||||
case Jt808Body.PassengerVolume volume -> {
|
||||
put(meta, "body.channelId", volume.channelId());
|
||||
put(meta, "body.startTime", volume.startTime());
|
||||
put(meta, "body.endTime", volume.endTime());
|
||||
put(meta, "body.passengerGetOn", volume.passengerGetOn());
|
||||
put(meta, "body.passengerGetOff", volume.passengerGetOff());
|
||||
put(meta, "body.rawSizeBytes", volume.raw() == null ? 0 : volume.raw().length);
|
||||
}
|
||||
case Jt808Body.Passthrough passthrough -> {
|
||||
put(meta, "body.passthroughType", "0x" + String.format("%02X", passthrough.passthroughType()));
|
||||
put(meta, "body.dataSizeBytes", passthrough.data() == null ? 0 : passthrough.data().length);
|
||||
put(meta, "body.dataHex", toHex(passthrough.data()));
|
||||
}
|
||||
case Jt808Body.Raw raw -> {
|
||||
put(meta, "body.rawMessageId", "0x" + Jt808MessageId.hex(raw.messageId()));
|
||||
put(meta, "body.rawSizeBytes", raw.bytes() == null ? 0 : raw.bytes().length);
|
||||
put(meta, "body.rawHex", toHex(raw.bytes()));
|
||||
}
|
||||
case Jt808Body.Malformed malformed -> {
|
||||
put(meta, "body.peer", malformed.peer());
|
||||
put(meta, "body.errorMessage", malformed.errorMessage());
|
||||
put(meta, "body.rawSizeBytes", malformed.bytes() == null ? 0 : malformed.bytes().length);
|
||||
}
|
||||
case Jt808Body.Heartbeat ignored -> {
|
||||
}
|
||||
case Jt808Body.Unregister ignored -> {
|
||||
}
|
||||
}
|
||||
meta.put("jt808.register.province", Integer.toString(reg.province()));
|
||||
meta.put("jt808.register.city", Integer.toString(reg.city()));
|
||||
meta.put("jt808.register.maker", nullToEmpty(reg.maker()));
|
||||
meta.put("jt808.register.deviceType", nullToEmpty(reg.deviceType()));
|
||||
meta.put("jt808.register.deviceId", nullToEmpty(reg.deviceId()));
|
||||
meta.put("jt808.register.plateColor", Integer.toString(reg.plateColor()));
|
||||
meta.put("jt808.register.plate", nullToEmpty(reg.plate()));
|
||||
}
|
||||
|
||||
private static String nullToEmpty(String value) {
|
||||
return value == null ? "" : value;
|
||||
private static void putLocationMetadata(Map<String, String> meta, String prefix, Jt808Body.Location location) {
|
||||
put(meta, prefix + "alarmFlag", location.alarmFlag());
|
||||
put(meta, prefix + "statusFlag", location.statusFlag());
|
||||
meta.putAll(Jt808LocationFlagInfo.decodeStatus(location.statusFlag(), prefix + "status."));
|
||||
meta.putAll(Jt808LocationFlagInfo.decodeAlarm(location.alarmFlag(), prefix + "alarm."));
|
||||
put(meta, prefix + "longitude", location.longitude());
|
||||
put(meta, prefix + "latitude", location.latitude());
|
||||
put(meta, prefix + "altitudeM", location.altitudeM());
|
||||
put(meta, prefix + "speedKmh", location.speedKmh());
|
||||
put(meta, prefix + "directionDeg", location.directionDeg());
|
||||
put(meta, prefix + "deviceTime", location.deviceTime());
|
||||
if (location.extensionItems() != null) {
|
||||
location.extensionItems().forEach((id, bytes) ->
|
||||
put(meta, prefix + "extra.0x" + String.format("%02X", id), toHex(bytes)));
|
||||
meta.putAll(Jt808LocationAdditionalInfo.decode(location.extensionItems(), prefix + "extra."));
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] registerAckFrame(String phone, int ackSerial, String token) {
|
||||
var cmd = Jt808Commands.registerAck(ackSerial, 0, token);
|
||||
return Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body());
|
||||
private static void put(Map<String, String> meta, String key, Object value) {
|
||||
if (value != null) {
|
||||
meta.put(key, String.valueOf(value));
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] terminalAckFrame(String phone, int ackSerial, int ackMsgId, int result) {
|
||||
private static String toHex(byte[] bytes) {
|
||||
if (bytes == null || bytes.length == 0) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder out = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
out.append(String.format("%02X", b & 0xFF));
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
private void ackTerminal(ChannelHandlerContext ctx, String phone, int ackSerial, int ackMsgId, int result) {
|
||||
var cmd = Jt808Commands.platformAck(ackSerial, ackMsgId, result);
|
||||
return Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body());
|
||||
byte[] frame = Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body());
|
||||
ctx.writeAndFlush(Unpooled.wrappedBuffer(frame));
|
||||
}
|
||||
|
||||
private static String generateToken(String phone) {
|
||||
return UUID.nameUUIDFromBytes(("jt808:" + phone).getBytes()).toString().replace("-", "").substring(0, 16);
|
||||
}
|
||||
|
||||
private static String generatedInternalVin(String phone, Jt808Body.Register reg) {
|
||||
String material = String.join("|", phone, reg.deviceId(), reg.plate(), reg.maker(), reg.deviceType());
|
||||
try {
|
||||
byte[] digest = MessageDigest.getInstance("SHA-256").digest(material.getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder suffix = new StringBuilder(12);
|
||||
for (int i = 0; i < digest.length && suffix.length() < 12; i++) {
|
||||
suffix.append(String.format("%02X", digest[i] & 0xFF));
|
||||
}
|
||||
return "JT808" + suffix.substring(0, 12);
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new IllegalStateException("SHA-256 not available", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void channelInactive(ChannelHandlerContext ctx) {
|
||||
channelRegistry.unbind(ctx.channel())
|
||||
|
||||
@@ -10,6 +10,8 @@ import com.lingniu.ingest.identity.VehicleIdentityResolver;
|
||||
import com.lingniu.ingest.identity.VehicleIdentitySource;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808Body;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808Header;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationAdditionalInfo;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationFlagInfo;
|
||||
import com.lingniu.ingest.protocol.jt808.model.Jt808Message;
|
||||
|
||||
import java.time.Instant;
|
||||
@@ -276,12 +278,14 @@ public final class Jt808EventMapper implements EventMapper<Jt808Message> {
|
||||
}
|
||||
|
||||
private static Map<String, String> locationMeta(Map<String, String> meta, Jt808Body.Location loc) {
|
||||
if (loc.extensionItems().isEmpty()) {
|
||||
return meta;
|
||||
}
|
||||
java.util.HashMap<String, String> copy = new java.util.HashMap<>(meta);
|
||||
loc.extensionItems().forEach((id, bytes) ->
|
||||
copy.put("jt808.extra.0x" + String.format("%02X", id), hex(bytes)));
|
||||
copy.putAll(Jt808LocationFlagInfo.decodeStatus(loc.statusFlag(), "jt808.status."));
|
||||
copy.putAll(Jt808LocationFlagInfo.decodeAlarm(loc.alarmFlag(), "jt808.alarm."));
|
||||
if (!loc.extensionItems().isEmpty()) {
|
||||
loc.extensionItems().forEach((id, bytes) ->
|
||||
copy.put("jt808.extra.0x" + String.format("%02X", id), hex(bytes)));
|
||||
copy.putAll(Jt808LocationAdditionalInfo.decode(loc.extensionItems()));
|
||||
}
|
||||
return Map.copyOf(copy);
|
||||
}
|
||||
|
||||
@@ -304,15 +308,7 @@ public final class Jt808EventMapper implements EventMapper<Jt808Message> {
|
||||
}
|
||||
|
||||
private static Double totalMileageKm(Jt808Body.Location loc) {
|
||||
byte[] bytes = loc.extensionItems().get(0x01);
|
||||
if (bytes == null || bytes.length != 4) {
|
||||
return null;
|
||||
}
|
||||
long raw = ((long) bytes[0] & 0xFF) << 24
|
||||
| ((long) bytes[1] & 0xFF) << 16
|
||||
| ((long) bytes[2] & 0xFF) << 8
|
||||
| ((long) bytes[3] & 0xFF);
|
||||
return raw * 0.1;
|
||||
return Jt808LocationAdditionalInfo.mileageKm(loc.extensionItems());
|
||||
}
|
||||
|
||||
private static String hex(byte[] bytes) {
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
package com.lingniu.ingest.protocol.jt808.model;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public final class Jt808LocationAdditionalInfo {
|
||||
|
||||
public static final String DEFAULT_PREFIX = "jt808.extra.";
|
||||
|
||||
private static final String[] VEHICLE_SIGNAL_BITS = {
|
||||
"low_beam",
|
||||
"high_beam",
|
||||
"right_turn",
|
||||
"left_turn",
|
||||
"brake",
|
||||
"reverse",
|
||||
"fog_lamp",
|
||||
"position_lamp",
|
||||
"horn",
|
||||
"air_conditioner",
|
||||
"neutral",
|
||||
"retarder",
|
||||
"abs",
|
||||
"heater",
|
||||
"clutch"
|
||||
};
|
||||
|
||||
private static final String[] IO_STATUS_BITS = {
|
||||
"deep_sleep",
|
||||
"sleep"
|
||||
};
|
||||
|
||||
private Jt808LocationAdditionalInfo() {
|
||||
}
|
||||
|
||||
public static Map<String, String> decode(Map<Integer, byte[]> extensionItems) {
|
||||
return decode(extensionItems, DEFAULT_PREFIX);
|
||||
}
|
||||
|
||||
public static Map<String, String> decode(Map<Integer, byte[]> extensionItems, String prefix) {
|
||||
if (extensionItems == null || extensionItems.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
String p = prefix == null ? "" : prefix;
|
||||
Map<String, String> out = new LinkedHashMap<>();
|
||||
extensionItems.forEach((id, bytes) -> decodeItem(out, p, id == null ? -1 : id, bytes));
|
||||
return Map.copyOf(out);
|
||||
}
|
||||
|
||||
public static Double mileageKm(Map<Integer, byte[]> extensionItems) {
|
||||
if (extensionItems == null) {
|
||||
return null;
|
||||
}
|
||||
byte[] bytes = extensionItems.get(0x01);
|
||||
if (bytes == null || bytes.length != 4) {
|
||||
return null;
|
||||
}
|
||||
return u32(bytes, 0) * 0.1;
|
||||
}
|
||||
|
||||
private static void decodeItem(Map<String, String> out, String prefix, int id, byte[] bytes) {
|
||||
if (bytes == null) {
|
||||
return;
|
||||
}
|
||||
switch (id) {
|
||||
case 0x01 -> put(out, prefix + "mileage_km", scaled(bytes, 4, 1));
|
||||
case 0x02 -> put(out, prefix + "fuel_l", scaled(bytes, 2, 1));
|
||||
case 0x03 -> put(out, prefix + "recorder_speed_kmh", scaled(bytes, 2, 1));
|
||||
case 0x04 -> put(out, prefix + "manual_alarm_event_id", unsigned(bytes, 2));
|
||||
case 0x05 -> decodeTirePressure(out, prefix, bytes);
|
||||
case 0x06 -> put(out, prefix + "compartment_temp_c", signedHighBitWord(bytes));
|
||||
case 0x11 -> decodeOverspeed(out, prefix, bytes);
|
||||
case 0x12 -> decodeAreaRouteAlarm(out, prefix, bytes);
|
||||
case 0x13 -> decodeRouteTime(out, prefix, bytes);
|
||||
case 0x25 -> decodeVehicleSignal(out, prefix, bytes);
|
||||
case 0x2A -> decodeIoStatus(out, prefix, bytes);
|
||||
case 0x2B -> decodeAnalog(out, prefix, bytes);
|
||||
case 0x30 -> put(out, prefix + "network_signal_strength", unsigned(bytes, 1));
|
||||
case 0x31 -> put(out, prefix + "gnss_satellite_count", unsigned(bytes, 1));
|
||||
default -> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void decodeTirePressure(Map<String, String> out, String prefix, byte[] bytes) {
|
||||
if (bytes.length != 30) {
|
||||
return;
|
||||
}
|
||||
List<String> values = new ArrayList<>();
|
||||
List<Integer> validIndexes = new ArrayList<>();
|
||||
List<Integer> invalidIndexes = new ArrayList<>();
|
||||
for (int i = 0; i < bytes.length; i++) {
|
||||
int value = u8(bytes, i);
|
||||
if (value == 0xFF) {
|
||||
invalidIndexes.add(i);
|
||||
} else {
|
||||
validIndexes.add(i);
|
||||
values.add(Integer.toString(value));
|
||||
}
|
||||
}
|
||||
put(out, prefix + "tire_pressure_pa_values", String.join(",", values));
|
||||
put(out, prefix + "tire_pressure_valid_indexes", ranges(validIndexes));
|
||||
put(out, prefix + "tire_pressure_invalid_indexes", ranges(invalidIndexes));
|
||||
}
|
||||
|
||||
private static void decodeOverspeed(Map<String, String> out, String prefix, byte[] bytes) {
|
||||
if (bytes.length != 1 && bytes.length != 5) {
|
||||
return;
|
||||
}
|
||||
put(out, prefix + "overspeed.position_type", Integer.toString(u8(bytes, 0)));
|
||||
if (bytes.length == 5) {
|
||||
put(out, prefix + "overspeed.area_route_id", Long.toString(u32(bytes, 1)));
|
||||
}
|
||||
}
|
||||
|
||||
private static void decodeAreaRouteAlarm(Map<String, String> out, String prefix, byte[] bytes) {
|
||||
if (bytes.length != 6) {
|
||||
return;
|
||||
}
|
||||
put(out, prefix + "area_route_alarm.position_type", Integer.toString(u8(bytes, 0)));
|
||||
put(out, prefix + "area_route_alarm.area_route_id", Long.toString(u32(bytes, 1)));
|
||||
put(out, prefix + "area_route_alarm.direction", Integer.toString(u8(bytes, 5)));
|
||||
}
|
||||
|
||||
private static void decodeRouteTime(Map<String, String> out, String prefix, byte[] bytes) {
|
||||
if (bytes.length != 7) {
|
||||
return;
|
||||
}
|
||||
put(out, prefix + "route_time.segment_id", Long.toString(u32(bytes, 0)));
|
||||
put(out, prefix + "route_time.duration_seconds", Integer.toString(u16(bytes, 4)));
|
||||
put(out, prefix + "route_time.result", Integer.toString(u8(bytes, 6)));
|
||||
}
|
||||
|
||||
private static void decodeVehicleSignal(Map<String, String> out, String prefix, byte[] bytes) {
|
||||
if (bytes.length != 4) {
|
||||
return;
|
||||
}
|
||||
long raw = u32(bytes, 0);
|
||||
put(out, prefix + "vehicle_signal_raw", Long.toString(raw));
|
||||
for (int i = 0; i < VEHICLE_SIGNAL_BITS.length; i++) {
|
||||
put(out, prefix + "vehicle_signal." + VEHICLE_SIGNAL_BITS[i], Boolean.toString(bit(raw, i)));
|
||||
}
|
||||
}
|
||||
|
||||
private static void decodeIoStatus(Map<String, String> out, String prefix, byte[] bytes) {
|
||||
if (bytes.length != 2) {
|
||||
return;
|
||||
}
|
||||
int raw = u16(bytes, 0);
|
||||
put(out, prefix + "io_status_raw", Integer.toString(raw));
|
||||
for (int i = 0; i < IO_STATUS_BITS.length; i++) {
|
||||
put(out, prefix + "io_status." + IO_STATUS_BITS[i], Boolean.toString(bit(raw, i)));
|
||||
}
|
||||
}
|
||||
|
||||
private static void decodeAnalog(Map<String, String> out, String prefix, byte[] bytes) {
|
||||
if (bytes.length != 4) {
|
||||
return;
|
||||
}
|
||||
long raw = u32(bytes, 0);
|
||||
put(out, prefix + "analog_raw", Long.toString(raw));
|
||||
put(out, prefix + "analog_ad0", Integer.toString((int) (raw & 0xFFFF)));
|
||||
put(out, prefix + "analog_ad1", Integer.toString((int) ((raw >>> 16) & 0xFFFF)));
|
||||
}
|
||||
|
||||
private static String scaled(byte[] bytes, int expectedLength, int scale) {
|
||||
if (bytes.length != expectedLength) {
|
||||
return null;
|
||||
}
|
||||
long raw = expectedLength == 4 ? u32(bytes, 0) : u16(bytes, 0);
|
||||
return BigDecimal.valueOf(raw, scale).stripTrailingZeros().toPlainString();
|
||||
}
|
||||
|
||||
private static String unsigned(byte[] bytes, int expectedLength) {
|
||||
if (bytes.length != expectedLength) {
|
||||
return null;
|
||||
}
|
||||
return switch (expectedLength) {
|
||||
case 1 -> Integer.toString(u8(bytes, 0));
|
||||
case 2 -> Integer.toString(u16(bytes, 0));
|
||||
case 4 -> Long.toString(u32(bytes, 0));
|
||||
default -> null;
|
||||
};
|
||||
}
|
||||
|
||||
private static String signedHighBitWord(byte[] bytes) {
|
||||
if (bytes.length != 2) {
|
||||
return null;
|
||||
}
|
||||
int raw = u16(bytes, 0);
|
||||
int value = (raw & 0x8000) == 0 ? raw : -(raw & 0x7FFF);
|
||||
return Integer.toString(value);
|
||||
}
|
||||
|
||||
private static int u8(byte[] bytes, int offset) {
|
||||
return bytes[offset] & 0xFF;
|
||||
}
|
||||
|
||||
private static int u16(byte[] bytes, int offset) {
|
||||
return (u8(bytes, offset) << 8) | u8(bytes, offset + 1);
|
||||
}
|
||||
|
||||
private static long u32(byte[] bytes, int offset) {
|
||||
return ((long) u8(bytes, offset) << 24)
|
||||
| ((long) u8(bytes, offset + 1) << 16)
|
||||
| ((long) u8(bytes, offset + 2) << 8)
|
||||
| u8(bytes, offset + 3);
|
||||
}
|
||||
|
||||
private static boolean bit(long raw, int bit) {
|
||||
return ((raw >>> bit) & 1L) == 1L;
|
||||
}
|
||||
|
||||
private static void put(Map<String, String> out, String key, String value) {
|
||||
if (value != null) {
|
||||
out.put(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
private static String ranges(List<Integer> indexes) {
|
||||
if (indexes.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
List<String> ranges = new ArrayList<>();
|
||||
int start = indexes.getFirst();
|
||||
int previous = start;
|
||||
for (int i = 1; i < indexes.size(); i++) {
|
||||
int current = indexes.get(i);
|
||||
if (current == previous + 1) {
|
||||
previous = current;
|
||||
continue;
|
||||
}
|
||||
ranges.add(formatRange(start, previous));
|
||||
start = current;
|
||||
previous = current;
|
||||
}
|
||||
ranges.add(formatRange(start, previous));
|
||||
return String.join(",", ranges);
|
||||
}
|
||||
|
||||
private static String formatRange(int start, int end) {
|
||||
return start == end ? Integer.toString(start) : start + "-" + end;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package com.lingniu.ingest.protocol.jt808.model;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public final class Jt808LocationFlagInfo {
|
||||
|
||||
private static final String[] ALARM_BITS = {
|
||||
"emergency",
|
||||
"overspeed",
|
||||
"fatigue_driving",
|
||||
"warning",
|
||||
"gnss_module_fault",
|
||||
"gnss_antenna_disconnected",
|
||||
"gnss_antenna_short_circuit",
|
||||
"terminal_main_power_undervoltage",
|
||||
"terminal_main_power_lost",
|
||||
"display_fault",
|
||||
"tts_fault",
|
||||
"camera_fault",
|
||||
"transport_ic_card_fault",
|
||||
"overspeed_warning",
|
||||
"fatigue_driving_warning",
|
||||
"prohibited_driving",
|
||||
null,
|
||||
null,
|
||||
"daily_driving_timeout",
|
||||
"overtime_parking",
|
||||
"area_in_out",
|
||||
"route_in_out",
|
||||
"route_time_abnormal",
|
||||
"route_deviation",
|
||||
"vss_fault",
|
||||
"fuel_abnormal",
|
||||
"stolen",
|
||||
"illegal_ignition",
|
||||
"illegal_displacement",
|
||||
"collision_rollover",
|
||||
"rollover_warning"
|
||||
};
|
||||
|
||||
private Jt808LocationFlagInfo() {
|
||||
}
|
||||
|
||||
public static Map<String, String> decodeAlarm(long alarmFlag, String prefix) {
|
||||
Map<String, String> out = new LinkedHashMap<>();
|
||||
String p = prefix == null ? "" : prefix;
|
||||
for (int i = 0; i < ALARM_BITS.length; i++) {
|
||||
String name = ALARM_BITS[i];
|
||||
if (name != null) {
|
||||
out.put(p + name, Boolean.toString(bit(alarmFlag, i)));
|
||||
}
|
||||
}
|
||||
return Map.copyOf(out);
|
||||
}
|
||||
|
||||
public static Map<String, String> decodeStatus(long statusFlag, String prefix) {
|
||||
Map<String, String> out = new LinkedHashMap<>();
|
||||
String p = prefix == null ? "" : prefix;
|
||||
out.put(p + "acc_on", Boolean.toString(bit(statusFlag, 0)));
|
||||
out.put(p + "positioned", Boolean.toString(bit(statusFlag, 1)));
|
||||
out.put(p + "south_latitude", Boolean.toString(bit(statusFlag, 2)));
|
||||
out.put(p + "west_longitude", Boolean.toString(bit(statusFlag, 3)));
|
||||
out.put(p + "out_of_service", Boolean.toString(bit(statusFlag, 4)));
|
||||
out.put(p + "encrypted", Boolean.toString(bit(statusFlag, 5)));
|
||||
out.put(p + "forward_collision_warning", Boolean.toString(bit(statusFlag, 6)));
|
||||
out.put(p + "lane_departure_warning", Boolean.toString(bit(statusFlag, 7)));
|
||||
out.put(p + "load_status", loadStatus(statusFlag));
|
||||
out.put(p + "fuel_circuit_cut", Boolean.toString(bit(statusFlag, 10)));
|
||||
out.put(p + "electric_circuit_cut", Boolean.toString(bit(statusFlag, 11)));
|
||||
out.put(p + "door_locked", Boolean.toString(bit(statusFlag, 12)));
|
||||
out.put(p + "door1_open", Boolean.toString(bit(statusFlag, 13)));
|
||||
out.put(p + "door2_open", Boolean.toString(bit(statusFlag, 14)));
|
||||
out.put(p + "door3_open", Boolean.toString(bit(statusFlag, 15)));
|
||||
out.put(p + "door4_open", Boolean.toString(bit(statusFlag, 16)));
|
||||
out.put(p + "door5_open", Boolean.toString(bit(statusFlag, 17)));
|
||||
out.put(p + "gps_used", Boolean.toString(bit(statusFlag, 18)));
|
||||
out.put(p + "beidou_used", Boolean.toString(bit(statusFlag, 19)));
|
||||
out.put(p + "glonass_used", Boolean.toString(bit(statusFlag, 20)));
|
||||
out.put(p + "galileo_used", Boolean.toString(bit(statusFlag, 21)));
|
||||
out.put(p + "moving", Boolean.toString(bit(statusFlag, 22)));
|
||||
return Map.copyOf(out);
|
||||
}
|
||||
|
||||
private static String loadStatus(long statusFlag) {
|
||||
int code = (int) ((statusFlag >>> 8) & 0x03);
|
||||
return switch (code) {
|
||||
case 0 -> "EMPTY";
|
||||
case 1 -> "HALF_LOADED";
|
||||
case 2 -> "RESERVED";
|
||||
case 3 -> "FULL";
|
||||
default -> "UNKNOWN";
|
||||
};
|
||||
}
|
||||
|
||||
private static boolean bit(long raw, int bit) {
|
||||
return ((raw >>> bit) & 1L) == 1L;
|
||||
}
|
||||
}
|
||||
@@ -36,4 +36,56 @@ public final class Jt808MessageId {
|
||||
public static final int PLATFORM_TRACK_CONTROL = 0x8202;
|
||||
public static final int PLATFORM_ALARM_ACK = 0x8203;
|
||||
public static final int PLATFORM_DELETE_CIRCLE_AREA = 0x8601;
|
||||
|
||||
public static String nameOf(int messageId) {
|
||||
return switch (messageId) {
|
||||
case TERMINAL_GENERAL_RESPONSE -> "TERMINAL_GENERAL_RESPONSE";
|
||||
case TERMINAL_HEARTBEAT -> "TERMINAL_HEARTBEAT";
|
||||
case TERMINAL_UNREGISTER -> "TERMINAL_UNREGISTER";
|
||||
case TERMINAL_REGISTER -> "TERMINAL_REGISTER";
|
||||
case TERMINAL_AUTH -> "TERMINAL_AUTH";
|
||||
case TERMINAL_PARAMS_REPORT -> "TERMINAL_PARAMS_REPORT";
|
||||
case TERMINAL_ATTRS_REPORT -> "TERMINAL_ATTRS_REPORT";
|
||||
case TERMINAL_LOCATION -> "TERMINAL_LOCATION";
|
||||
case TERMINAL_LOCATION_BATCH -> "TERMINAL_LOCATION_BATCH";
|
||||
case TERMINAL_EVENT_REPORT -> "TERMINAL_EVENT_REPORT";
|
||||
case TERMINAL_CAN_DATA_REPORT -> "TERMINAL_CAN_DATA_REPORT";
|
||||
case TERMINAL_MEDIA_EVENT -> "TERMINAL_MEDIA_EVENT";
|
||||
case TERMINAL_MEDIA_UPLOAD -> "TERMINAL_MEDIA_UPLOAD";
|
||||
case TERMINAL_PASSTHROUGH -> "TERMINAL_PASSTHROUGH";
|
||||
case PLATFORM_GENERAL_RESPONSE -> "PLATFORM_GENERAL_RESPONSE";
|
||||
case PLATFORM_REGISTER_ACK -> "PLATFORM_REGISTER_ACK";
|
||||
case PLATFORM_SET_PARAMS -> "PLATFORM_SET_PARAMS";
|
||||
case PLATFORM_QUERY_PARAMS -> "PLATFORM_QUERY_PARAMS";
|
||||
case PLATFORM_TERMINAL_CONTROL -> "PLATFORM_TERMINAL_CONTROL";
|
||||
case PLATFORM_QUERY_ATTRS -> "PLATFORM_QUERY_ATTRS";
|
||||
case PLATFORM_QUERY_LOCATION -> "PLATFORM_QUERY_LOCATION";
|
||||
case PLATFORM_TRACK_CONTROL -> "PLATFORM_TRACK_CONTROL";
|
||||
case PLATFORM_ALARM_ACK -> "PLATFORM_ALARM_ACK";
|
||||
case PLATFORM_DELETE_CIRCLE_AREA -> "PLATFORM_DELETE_CIRCLE_AREA";
|
||||
default -> "UNKNOWN_0x" + hex(messageId);
|
||||
};
|
||||
}
|
||||
|
||||
public static String reportTypeOf(int messageId) {
|
||||
return switch (messageId) {
|
||||
case TERMINAL_LOCATION -> "LOCATION";
|
||||
case TERMINAL_LOCATION_BATCH -> "LOCATION_BATCH";
|
||||
case TERMINAL_HEARTBEAT -> "HEARTBEAT";
|
||||
case TERMINAL_AUTH -> "AUTH";
|
||||
case TERMINAL_REGISTER -> "REGISTER";
|
||||
case TERMINAL_UNREGISTER -> "UNREGISTER";
|
||||
case TERMINAL_GENERAL_RESPONSE -> "TERMINAL_RESPONSE";
|
||||
case TERMINAL_PARAMS_REPORT -> "PARAMS_REPORT";
|
||||
case TERMINAL_ATTRS_REPORT -> "ATTRS_REPORT";
|
||||
case TERMINAL_MEDIA_EVENT -> "MEDIA_EVENT";
|
||||
case TERMINAL_MEDIA_UPLOAD -> "MEDIA_UPLOAD";
|
||||
case TERMINAL_PASSTHROUGH -> "PASSTHROUGH";
|
||||
default -> "RAW";
|
||||
};
|
||||
}
|
||||
|
||||
public static String hex(int messageId) {
|
||||
return String.format("%04X", messageId & 0xFFFF);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,7 @@ import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.within;
|
||||
|
||||
class Jt808ChannelHandlerTest {
|
||||
|
||||
@@ -86,7 +87,7 @@ class Jt808ChannelHandlerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void unresolvedRegisterDoesNotBindUnknownVinToExternalIdentifiers() {
|
||||
void unresolvedRegisterBindsGeneratedInternalVinToExternalIdentifiers() {
|
||||
InMemorySessionStore sessions = new InMemorySessionStore();
|
||||
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
|
||||
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(new ImmediateKafkaSink()));
|
||||
@@ -115,9 +116,9 @@ class Jt808ChannelHandlerTest {
|
||||
assertThat(identity.resolve(new VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "123456789012", "", "")))
|
||||
.satisfies(resolved -> {
|
||||
assertThat(resolved.vin()).isEqualTo("123456789012");
|
||||
assertThat(resolved.resolved()).isFalse();
|
||||
assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.FALLBACK_PHONE);
|
||||
assertThat(resolved.vin()).startsWith("JT808");
|
||||
assertThat(resolved.resolved()).isTrue();
|
||||
assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
|
||||
});
|
||||
|
||||
batchExecutor.close();
|
||||
@@ -269,14 +270,143 @@ class Jt808ChannelHandlerTest {
|
||||
assertThat(archive.metadata())
|
||||
.containsEntry("vin", "LNVIN000000000808")
|
||||
.containsEntry("phone", "123456789012")
|
||||
.containsEntry("messageId", "0x0200")
|
||||
.containsEntry("messageName", "TERMINAL_LOCATION")
|
||||
.containsEntry("reportType", "LOCATION")
|
||||
.containsEntry("body.alarmFlag", "0")
|
||||
.containsEntry("body.statusFlag", "196608")
|
||||
.containsEntry("body.status.acc_on", "false")
|
||||
.containsEntry("body.status.load_status", "EMPTY")
|
||||
.containsEntry("body.status.door4_open", "true")
|
||||
.containsEntry("body.status.door5_open", "true")
|
||||
.containsEntry("body.alarm.overspeed", "false")
|
||||
.containsEntry("body.longitude", "116.397128")
|
||||
.containsEntry("body.latitude", "39.916527")
|
||||
.containsEntry("body.altitudeM", "50")
|
||||
.containsEntry("body.directionDeg", "90")
|
||||
.containsEntry("body.deviceTime", "2024-01-01T19:04:05Z")
|
||||
.containsEntry("body.extra.0x01", "00003039")
|
||||
.containsEntry("body.extra.mileage_km", "1234.5")
|
||||
.containsEntry("body.extra.0x30", "58")
|
||||
.containsEntry("body.extra.network_signal_strength", "88")
|
||||
.containsEntry("identityResolved", "true")
|
||||
.containsEntry("identitySource", "BOUND_PHONE");
|
||||
assertThat(Double.parseDouble(archive.metadata().get("body.speedKmh"))).isCloseTo(52.3, within(0.001));
|
||||
});
|
||||
|
||||
batchExecutor.close();
|
||||
eventBus.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void registerArchiveIncludesParsedBodyFieldsForRawQuery() throws Exception {
|
||||
RecordingSink sink = new RecordingSink(1);
|
||||
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
|
||||
AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish);
|
||||
Dispatcher dispatcher = new Dispatcher(
|
||||
new HandlerRegistry(),
|
||||
new InterceptorChain(List.of()),
|
||||
new HandlerInvoker(),
|
||||
eventBus,
|
||||
batchExecutor);
|
||||
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
|
||||
identity.bind(new VehicleIdentityBinding(
|
||||
ProtocolId.JT808, "LNVIN000000000808", "123456789012", "DEV808", "B80808"));
|
||||
Jt808ChannelHandler handler = new Jt808ChannelHandler(
|
||||
new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser()))),
|
||||
dispatcher,
|
||||
new InMemorySessionStore(),
|
||||
identity,
|
||||
new Jt808ChannelRegistry(),
|
||||
new Jt808PendingRequests());
|
||||
EmbeddedChannel channel = new EmbeddedChannel(handler);
|
||||
byte[] frame = buildFrame(
|
||||
Jt808MessageId.TERMINAL_REGISTER,
|
||||
"123456789012",
|
||||
3,
|
||||
buildRegisterBody("DEV808", "B80808"));
|
||||
|
||||
channel.writeInbound(frame);
|
||||
|
||||
assertThat(sink.await()).isTrue();
|
||||
assertThat(sink.events).singleElement().satisfies(event -> {
|
||||
assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class);
|
||||
VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event;
|
||||
assertThat(archive.metadata())
|
||||
.containsEntry("messageId", "0x0100")
|
||||
.containsEntry("messageName", "TERMINAL_REGISTER")
|
||||
.containsEntry("reportType", "REGISTER")
|
||||
.containsEntry("body.province", "44")
|
||||
.containsEntry("body.city", "4401")
|
||||
.containsEntry("body.maker", "MAKER")
|
||||
.containsEntry("body.deviceType", "TYPE-A")
|
||||
.containsEntry("body.deviceId", "DEV808")
|
||||
.containsEntry("body.plateColor", "1")
|
||||
.containsEntry("body.plate", "B80808");
|
||||
});
|
||||
|
||||
batchExecutor.close();
|
||||
eventBus.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void unboundRegisterCreatesVinBindingForRegisterAndFollowingRawFrames() throws Exception {
|
||||
RecordingSink sink = new RecordingSink(2);
|
||||
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
|
||||
AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish);
|
||||
Dispatcher dispatcher = new Dispatcher(
|
||||
new HandlerRegistry(),
|
||||
new InterceptorChain(List.of()),
|
||||
new HandlerInvoker(),
|
||||
eventBus,
|
||||
batchExecutor);
|
||||
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
|
||||
Jt808ChannelHandler handler = new Jt808ChannelHandler(
|
||||
new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser(), new LocationBodyParser()))),
|
||||
dispatcher,
|
||||
new InMemorySessionStore(),
|
||||
identity,
|
||||
new Jt808ChannelRegistry(),
|
||||
new Jt808PendingRequests());
|
||||
EmbeddedChannel channel = new EmbeddedChannel(handler);
|
||||
|
||||
channel.writeInbound(buildFrame(
|
||||
Jt808MessageId.TERMINAL_REGISTER,
|
||||
"13079963289",
|
||||
3,
|
||||
buildRegisterBody("9963289", "HUA00113F")));
|
||||
channel.writeInbound(buildFrame(
|
||||
Jt808MessageId.TERMINAL_LOCATION,
|
||||
"13079963289",
|
||||
4,
|
||||
buildLocationBody()));
|
||||
|
||||
assertThat(sink.await()).isTrue();
|
||||
VehicleEvent.RawArchive register = (VehicleEvent.RawArchive) sink.events.get(0);
|
||||
VehicleEvent.RawArchive location = (VehicleEvent.RawArchive) sink.events.get(1);
|
||||
assertThat(register.metadata())
|
||||
.containsEntry("reportType", "REGISTER")
|
||||
.containsEntry("body.deviceId", "9963289")
|
||||
.containsEntry("body.plate", "HUA00113F")
|
||||
.containsEntry("identityResolved", "true")
|
||||
.containsEntry("identitySource", "REGISTERED");
|
||||
assertThat(register.vin())
|
||||
.startsWith("JT808")
|
||||
.hasSize(17);
|
||||
assertThat(location.metadata())
|
||||
.containsEntry("reportType", "LOCATION")
|
||||
.containsEntry("vin", register.vin())
|
||||
.containsEntry("identityResolved", "true")
|
||||
.containsEntry("identitySource", "BOUND_PHONE");
|
||||
assertThat(location.vin()).isEqualTo(register.vin());
|
||||
assertThat(identity.resolve(new com.lingniu.ingest.identity.VehicleIdentityLookup(
|
||||
ProtocolId.JT808, "", "13079963289", "", "")).vin())
|
||||
.isEqualTo(register.vin());
|
||||
|
||||
batchExecutor.close();
|
||||
eventBus.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void unboundLocationArchiveUsesUnknownVinInsteadOfPhoneFallback() throws Exception {
|
||||
RecordingSink sink = new RecordingSink(1);
|
||||
@@ -309,6 +439,9 @@ class Jt808ChannelHandlerTest {
|
||||
assertThat(archive.metadata())
|
||||
.containsEntry("vin", "unknown")
|
||||
.containsEntry("phone", "123456789012")
|
||||
.containsEntry("messageId", "0x0200")
|
||||
.containsEntry("messageName", "TERMINAL_LOCATION")
|
||||
.containsEntry("reportType", "LOCATION")
|
||||
.containsEntry("identityResolved", "false")
|
||||
.containsEntry("identitySource", "FALLBACK_PHONE");
|
||||
});
|
||||
@@ -351,13 +484,13 @@ class Jt808ChannelHandlerTest {
|
||||
VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event;
|
||||
assertThat(archive.command()).isEqualTo(Jt808MessageId.TERMINAL_REGISTER);
|
||||
assertThat(archive.metadata())
|
||||
.containsEntry("jt808.register.province", "44")
|
||||
.containsEntry("jt808.register.city", "4401")
|
||||
.containsEntry("jt808.register.maker", "MAKER")
|
||||
.containsEntry("jt808.register.deviceType", "TYPE-A")
|
||||
.containsEntry("jt808.register.deviceId", "DEV808")
|
||||
.containsEntry("jt808.register.plateColor", "1")
|
||||
.containsEntry("jt808.register.plate", "B80808");
|
||||
.containsEntry("body.province", "44")
|
||||
.containsEntry("body.city", "4401")
|
||||
.containsEntry("body.maker", "MAKER")
|
||||
.containsEntry("body.deviceType", "TYPE-A")
|
||||
.containsEntry("body.deviceId", "DEV808")
|
||||
.containsEntry("body.plateColor", "1")
|
||||
.containsEntry("body.plate", "B80808");
|
||||
});
|
||||
|
||||
batchExecutor.close();
|
||||
@@ -581,6 +714,12 @@ class Jt808ChannelHandlerTest {
|
||||
writeU16(os, 90);
|
||||
byte[] bcd = BcdCodec.encode("240102030405");
|
||||
os.write(bcd, 0, 6);
|
||||
os.write(0x01);
|
||||
os.write(4);
|
||||
writeU32(os, 12345);
|
||||
os.write(0x30);
|
||||
os.write(1);
|
||||
os.write(0x58);
|
||||
return os.toByteArray();
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,10 @@ class Jt808EventMapperTest {
|
||||
assertThat(events.get(0).metadata())
|
||||
.containsEntry("vin", "unknown")
|
||||
.containsEntry("phone", "123456789012")
|
||||
.containsEntry("jt808.status.acc_on", "false")
|
||||
.containsEntry("jt808.status.positioned", "false")
|
||||
.containsEntry("jt808.status.load_status", "EMPTY")
|
||||
.containsEntry("jt808.alarm.overspeed", "false")
|
||||
.containsEntry("identityResolved", "false")
|
||||
.containsEntry("identitySource", "FALLBACK_PHONE");
|
||||
}
|
||||
@@ -58,7 +62,66 @@ class Jt808EventMapperTest {
|
||||
.extracting(VehicleEvent::metadata)
|
||||
.satisfies(meta -> assertThat(meta)
|
||||
.containsEntry("jt808.extra.0x01", "00003039")
|
||||
.containsEntry("jt808.extra.0x30", "58"));
|
||||
.containsEntry("jt808.extra.0x30", "58")
|
||||
.containsEntry("jt808.extra.mileage_km", "1234.5")
|
||||
.containsEntry("jt808.extra.network_signal_strength", "88"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void locationAdditionalItemsAreDecodedAsTable27Fields() {
|
||||
var header = new Jt808Header(
|
||||
Jt808MessageId.TERMINAL_LOCATION, 64, 0, false,
|
||||
Jt808Header.ProtocolVersion.V2013, "123456789012", 1, 0, 0);
|
||||
var body = new Jt808Body.Location(
|
||||
0, 0, 116.397128, 39.916527, 50, 52.3, 90,
|
||||
Instant.parse("2024-01-02T03:04:05Z"),
|
||||
java.util.Map.ofEntries(
|
||||
java.util.Map.entry(0x01, new byte[]{0x00, 0x00, 0x30, 0x39}),
|
||||
java.util.Map.entry(0x02, new byte[]{0x00, 0x64}),
|
||||
java.util.Map.entry(0x03, new byte[]{0x01, (byte) 0xF4}),
|
||||
java.util.Map.entry(0x04, new byte[]{0x00, 0x02}),
|
||||
java.util.Map.entry(0x05, tirePressureBytes()),
|
||||
java.util.Map.entry(0x06, new byte[]{(byte) 0x80, 0x05}),
|
||||
java.util.Map.entry(0x11, new byte[]{0x01, 0x00, 0x00, 0x00, 0x2A}),
|
||||
java.util.Map.entry(0x12, new byte[]{0x04, 0x00, 0x00, 0x00, 0x2B, 0x01}),
|
||||
java.util.Map.entry(0x13, new byte[]{0x00, 0x00, 0x00, 0x2C, 0x00, 0x78, 0x01}),
|
||||
java.util.Map.entry(0x25, new byte[]{0x00, 0x00, 0x00, 0x05}),
|
||||
java.util.Map.entry(0x2A, new byte[]{0x00, 0x03}),
|
||||
java.util.Map.entry(0x2B, new byte[]{0x12, 0x34, 0x56, 0x78}),
|
||||
java.util.Map.entry(0x30, new byte[]{0x1F}),
|
||||
java.util.Map.entry(0x31, new byte[]{0x0C})));
|
||||
|
||||
VehicleEvent.Location event = (VehicleEvent.Location) mapper.toEvents(new Jt808Message(header, body)).getFirst();
|
||||
|
||||
assertThat(event.payload().totalMileageKm()).isEqualTo(1234.5);
|
||||
assertThat(event.metadata())
|
||||
.containsEntry("jt808.extra.mileage_km", "1234.5")
|
||||
.containsEntry("jt808.extra.fuel_l", "10")
|
||||
.containsEntry("jt808.extra.recorder_speed_kmh", "50")
|
||||
.containsEntry("jt808.extra.manual_alarm_event_id", "2")
|
||||
.containsEntry("jt808.extra.tire_pressure_pa_values", "32,33")
|
||||
.containsEntry("jt808.extra.tire_pressure_invalid_indexes", "2-29")
|
||||
.containsEntry("jt808.extra.compartment_temp_c", "-5")
|
||||
.containsEntry("jt808.extra.overspeed.position_type", "1")
|
||||
.containsEntry("jt808.extra.overspeed.area_route_id", "42")
|
||||
.containsEntry("jt808.extra.area_route_alarm.position_type", "4")
|
||||
.containsEntry("jt808.extra.area_route_alarm.area_route_id", "43")
|
||||
.containsEntry("jt808.extra.area_route_alarm.direction", "1")
|
||||
.containsEntry("jt808.extra.route_time.segment_id", "44")
|
||||
.containsEntry("jt808.extra.route_time.duration_seconds", "120")
|
||||
.containsEntry("jt808.extra.route_time.result", "1")
|
||||
.containsEntry("jt808.extra.vehicle_signal_raw", "5")
|
||||
.containsEntry("jt808.extra.vehicle_signal.low_beam", "true")
|
||||
.containsEntry("jt808.extra.vehicle_signal.high_beam", "false")
|
||||
.containsEntry("jt808.extra.vehicle_signal.right_turn", "true")
|
||||
.containsEntry("jt808.extra.io_status_raw", "3")
|
||||
.containsEntry("jt808.extra.io_status.deep_sleep", "true")
|
||||
.containsEntry("jt808.extra.io_status.sleep", "true")
|
||||
.containsEntry("jt808.extra.analog_raw", "305419896")
|
||||
.containsEntry("jt808.extra.analog_ad0", "22136")
|
||||
.containsEntry("jt808.extra.analog_ad1", "4660")
|
||||
.containsEntry("jt808.extra.network_signal_strength", "31")
|
||||
.containsEntry("jt808.extra.gnss_satellite_count", "12");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -249,4 +312,12 @@ class Jt808EventMapperTest {
|
||||
.containsEntry("parseError", "true");
|
||||
});
|
||||
}
|
||||
|
||||
private static byte[] tirePressureBytes() {
|
||||
byte[] bytes = new byte[30];
|
||||
java.util.Arrays.fill(bytes, (byte) 0xFF);
|
||||
bytes[0] = 32;
|
||||
bytes[1] = 33;
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,10 @@
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
|
||||
@@ -12,6 +12,10 @@ import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -27,6 +31,8 @@ import java.util.Map;
|
||||
@RequestMapping("/api/event-history")
|
||||
public class EventHistoryController {
|
||||
|
||||
private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
|
||||
private final EventFileStore store;
|
||||
|
||||
public EventHistoryController(EventFileStore store) {
|
||||
@@ -40,8 +46,13 @@ public class EventHistoryController {
|
||||
@RequestParam String dateTo,
|
||||
@RequestParam(defaultValue = "ASC") EventFileQuery.Order order,
|
||||
@RequestParam(defaultValue = "100") int limit,
|
||||
@RequestParam(required = false) String vin) throws IOException {
|
||||
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin).stream()
|
||||
@RequestParam(required = false) String vin,
|
||||
@RequestParam(required = false) String eventType,
|
||||
@RequestParam(required = false) String cursorEventTime,
|
||||
@RequestParam(required = false) String cursorIngestTime,
|
||||
@RequestParam(required = false) String cursorEventId) throws IOException {
|
||||
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin, eventType,
|
||||
cursorEventTime, cursorIngestTime, cursorEventId).stream()
|
||||
.map(RecordResponse::from)
|
||||
.toList();
|
||||
}
|
||||
@@ -51,7 +62,11 @@ public class EventHistoryController {
|
||||
String dateTo,
|
||||
EventFileQuery.Order order,
|
||||
int limit,
|
||||
String vin) throws IOException {
|
||||
String vin,
|
||||
String eventType,
|
||||
String cursorEventTime,
|
||||
String cursorIngestTime,
|
||||
String cursorEventId) throws IOException {
|
||||
QueryTimeRange range = QueryTimeRange.parse(dateFrom, dateTo);
|
||||
return store.query(new EventFileQuery(
|
||||
protocol,
|
||||
@@ -62,7 +77,22 @@ public class EventHistoryController {
|
||||
order,
|
||||
limit,
|
||||
vin,
|
||||
null));
|
||||
eventType,
|
||||
instantParam(cursorEventTime),
|
||||
instantParam(cursorIngestTime),
|
||||
cursorEventId));
|
||||
}
|
||||
|
||||
private static Instant instantParam(String raw) {
|
||||
if (raw == null || raw.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String value = raw.trim();
|
||||
try {
|
||||
return OffsetDateTime.parse(value).toInstant();
|
||||
} catch (RuntimeException ignored) {
|
||||
return LocalDateTime.parse(value).atZone(DEFAULT_ZONE).toInstant();
|
||||
}
|
||||
}
|
||||
|
||||
public record RecordResponse(
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.lingniu.ingest.eventhistory;
|
||||
|
||||
import com.google.protobuf.InvalidProtocolBufferException;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
|
||||
import com.lingniu.ingest.eventfilestore.EventFileRecord;
|
||||
import com.lingniu.ingest.eventfilestore.EventFileStore;
|
||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
||||
@@ -20,7 +20,7 @@ import java.util.List;
|
||||
* <p>当前 32960 本机运行配置没有启用 Kafka consumer;这个类保留给“其他服务把 envelope
|
||||
* 写回历史库”的解耦部署方式。失败时返回结构化结果,由 Kafka worker 决定是否进 DLQ。
|
||||
*/
|
||||
public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor {
|
||||
|
||||
private final EventFileStore store;
|
||||
private final TelemetryEnvelopeRecordMapper mapper;
|
||||
@@ -33,7 +33,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
|
||||
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
|
||||
TdengineHistoryWriter tdengineWriter) {
|
||||
this(mapper, tdengineWriter, true);
|
||||
this(mapper, tdengineWriter, false);
|
||||
}
|
||||
|
||||
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
|
||||
@@ -45,7 +45,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
|
||||
public EventHistoryEnvelopeIngestor(EventFileStore store,
|
||||
TelemetryEnvelopeRecordMapper mapper,
|
||||
TdengineHistoryWriter tdengineWriter) {
|
||||
this(store, mapper, tdengineWriter, true, true);
|
||||
this(store, mapper, tdengineWriter, true, false);
|
||||
}
|
||||
|
||||
public EventHistoryEnvelopeIngestor(EventFileStore store,
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package com.lingniu.ingest.eventhistory;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
|
||||
@@ -32,8 +30,6 @@ import java.util.List;
|
||||
@Tag(name = "jt-808-location-controller", description = "JT808 位置历史分页查询接口。")
|
||||
public final class Jt808LocationHistoryController {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private final TdengineHistoryReader reader;
|
||||
|
||||
public Jt808LocationHistoryController(TdengineHistoryReader reader) {
|
||||
@@ -127,8 +123,7 @@ public final class Jt808LocationHistoryController {
|
||||
long alarmFlag,
|
||||
long statusFlag,
|
||||
Double totalMileageKm,
|
||||
String rawUri,
|
||||
String metadataJson) {
|
||||
String rawUri) {
|
||||
|
||||
private static LocationResponse from(TdengineLocationRow row) {
|
||||
return new LocationResponse(
|
||||
@@ -147,23 +142,7 @@ public final class Jt808LocationHistoryController {
|
||||
row.alarmFlag(),
|
||||
row.statusFlag(),
|
||||
row.totalMileageKm(),
|
||||
rawUri(row),
|
||||
row.metadataJson());
|
||||
}
|
||||
|
||||
private static String rawUri(TdengineLocationRow row) {
|
||||
if (row.rawUri() != null && !row.rawUri().isBlank()) {
|
||||
return row.rawUri();
|
||||
}
|
||||
String metadataJson = row.metadataJson();
|
||||
if (metadataJson == null || metadataJson.isBlank()) {
|
||||
return row.rawUri();
|
||||
}
|
||||
try {
|
||||
return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
|
||||
} catch (JsonProcessingException ignored) {
|
||||
return row.rawUri();
|
||||
}
|
||||
row.rawUri());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package com.lingniu.ingest.eventhistory;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
|
||||
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
|
||||
@@ -32,8 +30,6 @@ import java.util.Locale;
|
||||
@Tag(name = "location-history-controller", description = "通用位置历史分页查询接口。")
|
||||
public final class LocationHistoryController {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private final TdengineHistoryReader reader;
|
||||
|
||||
public LocationHistoryController(TdengineHistoryReader reader) {
|
||||
@@ -161,7 +157,6 @@ public final class LocationHistoryController {
|
||||
long statusFlag,
|
||||
Double totalMileageKm,
|
||||
String rawUri,
|
||||
String metadataJson,
|
||||
String protocol) {
|
||||
|
||||
private static LocationResponse from(TdengineLocationRow row) {
|
||||
@@ -181,24 +176,8 @@ public final class LocationHistoryController {
|
||||
row.alarmFlag(),
|
||||
row.statusFlag(),
|
||||
row.totalMileageKm(),
|
||||
rawUri(row),
|
||||
row.metadataJson(),
|
||||
row.rawUri(),
|
||||
row.protocol());
|
||||
}
|
||||
|
||||
private static String rawUri(TdengineLocationRow row) {
|
||||
if (row.rawUri() != null && !row.rawUri().isBlank()) {
|
||||
return row.rawUri();
|
||||
}
|
||||
String metadataJson = row.metadataJson();
|
||||
if (metadataJson == null || metadataJson.isBlank()) {
|
||||
return row.rawUri();
|
||||
}
|
||||
try {
|
||||
return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
|
||||
} catch (JsonProcessingException ignored) {
|
||||
return row.rawUri();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,6 +184,7 @@ public final class RawFrameHistoryController {
|
||||
String parseError,
|
||||
String peer,
|
||||
String metadataJson,
|
||||
String parsedJson,
|
||||
Map<String, Object> metadata,
|
||||
Map<String, Object> parsedFields,
|
||||
String protocol,
|
||||
@@ -209,8 +210,9 @@ public final class RawFrameHistoryController {
|
||||
row.parseError(),
|
||||
row.peer(),
|
||||
row.metadataJson(),
|
||||
row.parsedJson(),
|
||||
metadata,
|
||||
parsedFields(reader, row, metadata, includeParsedFields),
|
||||
parsedFields(reader, row, metadata, parsed(row.parsedJson()), includeParsedFields),
|
||||
row.protocol(),
|
||||
row.vehicleKey(),
|
||||
row.vin(),
|
||||
@@ -231,18 +233,23 @@ public final class RawFrameHistoryController {
|
||||
private static Map<String, Object> parsedFields(TdengineHistoryReader reader,
|
||||
TdengineRawFrameRow row,
|
||||
Map<String, Object> metadata,
|
||||
Map<String, Object> parsed,
|
||||
boolean includeParsedFields) throws IOException {
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, Object> entry : metadata.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
if (key == null || key.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
if (key.startsWith("raw.") || key.startsWith("jt808.")) {
|
||||
out.put(key, entry.getValue());
|
||||
} else {
|
||||
out.put("raw." + key, entry.getValue());
|
||||
if (parsed.isEmpty()) {
|
||||
for (Map.Entry<String, Object> entry : metadata.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
if (key == null || key.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
if (key.startsWith("raw.") || key.startsWith("jt808.")) {
|
||||
out.put(key, entry.getValue());
|
||||
} else {
|
||||
out.put("raw." + key, entry.getValue());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.putAll(parsed);
|
||||
}
|
||||
if (!includeParsedFields) {
|
||||
return Map.copyOf(out);
|
||||
@@ -262,6 +269,17 @@ public final class RawFrameHistoryController {
|
||||
return Map.copyOf(out);
|
||||
}
|
||||
|
||||
private static Map<String, Object> parsed(String parsedJson) {
|
||||
if (parsedJson == null || parsedJson.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
return OBJECT_MAPPER.readValue(parsedJson, MAP_TYPE);
|
||||
} catch (Exception ignored) {
|
||||
return Map.of("_raw", parsedJson);
|
||||
}
|
||||
}
|
||||
|
||||
private static void addLocation(Map<String, Object> out, TdengineLocationRow row) {
|
||||
out.put("location.longitude", row.longitude());
|
||||
out.put("location.latitude", row.latitude());
|
||||
|
||||
@@ -150,6 +150,7 @@ public final class TelemetryEnvelopeRecordMapper {
|
||||
payload.put("infoType", metadata.get("infoType"));
|
||||
}
|
||||
payload.put("rawSizeBytes", rawArchive.getSizeBytes());
|
||||
putParsedJson(payload, rawArchive.getParsedJson());
|
||||
payload.put("metadata", metadata);
|
||||
try {
|
||||
return OBJECT_MAPPER.writeValueAsString(payload);
|
||||
@@ -157,4 +158,16 @@ public final class TelemetryEnvelopeRecordMapper {
|
||||
throw new IllegalArgumentException("failed to serialize raw_archive", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static void putParsedJson(Map<String, Object> payload, String parsedJson) {
|
||||
if (parsedJson == null || parsedJson.isBlank()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
payload.put("parsed", OBJECT_MAPPER.readTree(parsedJson));
|
||||
} catch (JsonProcessingException ex) {
|
||||
payload.put("parsed", parsedJson);
|
||||
payload.put("parsedJsonError", ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,11 @@ class EventHistoryControllerTest {
|
||||
"2026-06-22",
|
||||
EventFileQuery.Order.DESC,
|
||||
50,
|
||||
"VIN001");
|
||||
"VIN001",
|
||||
"RAW_ARCHIVE",
|
||||
"2026-06-22T08:00:00Z",
|
||||
"2026-06-22T08:00:01Z",
|
||||
"event-0");
|
||||
|
||||
assertThat(records).extracting(EventHistoryController.RecordResponse::eventId).containsExactly("event-1");
|
||||
assertThat(records.getFirst().eventTime()).isEqualTo("2026-06-22T08:00:00Z");
|
||||
@@ -37,6 +41,10 @@ class EventHistoryControllerTest {
|
||||
assertThat(store.lastQuery.order()).isEqualTo(EventFileQuery.Order.DESC);
|
||||
assertThat(store.lastQuery.limit()).isEqualTo(50);
|
||||
assertThat(store.lastQuery.vin()).isEqualTo("VIN001");
|
||||
assertThat(store.lastQuery.eventType()).isEqualTo("RAW_ARCHIVE");
|
||||
assertThat(store.lastQuery.cursorEventTime()).isEqualTo(Instant.parse("2026-06-22T08:00:00Z"));
|
||||
assertThat(store.lastQuery.cursorIngestTime()).isEqualTo(Instant.parse("2026-06-22T08:00:01Z"));
|
||||
assertThat(store.lastQuery.cursorEventId()).isEqualTo("event-0");
|
||||
}
|
||||
|
||||
private static EventFileRecord record(String eventId) {
|
||||
|
||||
@@ -126,7 +126,7 @@ class EventHistoryEnvelopeIngestorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void tryIngestWritesRawLocationAndTelemetryFactsToTdengineWhenWriterExists() throws Exception {
|
||||
void tryIngestWritesRawAndLocationFactsToTdengineWhenWriterExists() throws Exception {
|
||||
CapturingStore store = new CapturingStore();
|
||||
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
|
||||
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
|
||||
@@ -180,10 +180,7 @@ class EventHistoryEnvelopeIngestorTest {
|
||||
.extracting(TdengineLocationRow::factId)
|
||||
.containsExactly("jt808-location-1");
|
||||
assertThat(tdengineWriter.locations.getFirst().longitude()).isEqualTo(113.12);
|
||||
assertThat(tdengineWriter.telemetryFields)
|
||||
.extracting(TdengineTelemetryFieldRow::fieldKey)
|
||||
.containsExactly("location.speedKmh");
|
||||
assertThat(tdengineWriter.telemetryFields.getFirst().valueDouble()).isEqualTo(42.5);
|
||||
assertThat(tdengineWriter.telemetryFields).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -200,16 +197,14 @@ class EventHistoryEnvelopeIngestorTest {
|
||||
.containsExactly("frame-jt808-1");
|
||||
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
|
||||
.containsExactly("jt808-location-1");
|
||||
assertThat(tdengineWriter.telemetryFields)
|
||||
.extracting(TdengineTelemetryFieldRow::fieldKey)
|
||||
.containsExactly("location.speedKmh");
|
||||
assertThat(tdengineWriter.telemetryFields).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void tryIngestCanSkipTelemetryFieldFactsForHighThroughputRuntime() {
|
||||
void tryIngestCanWriteTelemetryFieldFactsWhenExplicitlyEnabled() {
|
||||
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
|
||||
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
|
||||
new TelemetryEnvelopeRecordMapper(), tdengineWriter, false);
|
||||
new TelemetryEnvelopeRecordMapper(), tdengineWriter, true);
|
||||
|
||||
EnvelopeIngestResult result = ingestor.tryIngest(
|
||||
jt808LocationEnvelope("jt808-location-1", "frame-jt808-1", "013800000001").toByteArray());
|
||||
@@ -219,7 +214,9 @@ class EventHistoryEnvelopeIngestorTest {
|
||||
.containsExactly("frame-jt808-1");
|
||||
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
|
||||
.containsExactly("jt808-location-1");
|
||||
assertThat(tdengineWriter.telemetryFields).isEmpty();
|
||||
assertThat(tdengineWriter.telemetryFields)
|
||||
.extracting(TdengineTelemetryFieldRow::fieldKey)
|
||||
.containsExactly("location.speedKmh");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -249,9 +246,7 @@ class EventHistoryEnvelopeIngestorTest {
|
||||
.containsExactly("frame-jt808-1", "frame-jt808-2");
|
||||
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
|
||||
.containsExactly("jt808-location-1", "jt808-location-2");
|
||||
assertThat(tdengineWriter.telemetryFields)
|
||||
.extracting(TdengineTelemetryFieldRow::fieldKey)
|
||||
.containsExactly("location.speedKmh", "location.speedKmh");
|
||||
assertThat(tdengineWriter.telemetryFields).isEmpty();
|
||||
}
|
||||
|
||||
private static VehicleEnvelope envelope(String eventId) {
|
||||
|
||||
@@ -314,6 +314,7 @@ class Gb32960DecodedFrameServiceTest {
|
||||
"",
|
||||
"127.0.0.1:32960",
|
||||
"{\"platformAccount\":\"Hyundai\"}",
|
||||
"",
|
||||
"GB32960",
|
||||
vin,
|
||||
vin,
|
||||
@@ -360,6 +361,7 @@ class Gb32960DecodedFrameServiceTest {
|
||||
"",
|
||||
"8.134.95.166:37302",
|
||||
"{\"platformAccount\":\"Hyundai\"}",
|
||||
"",
|
||||
"GB32960",
|
||||
"unknown:GB32960:platform-login",
|
||||
"",
|
||||
@@ -409,6 +411,7 @@ class Gb32960DecodedFrameServiceTest {
|
||||
"",
|
||||
"127.0.0.1:32960",
|
||||
"{\"platformAccount\":\"Hyundai\"}",
|
||||
"",
|
||||
"GB32960",
|
||||
vin,
|
||||
vin,
|
||||
|
||||
@@ -51,13 +51,12 @@ class Jt808LocationHistoryControllerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void fallsBackToRawArchiveUriInMetadataWhenLocationRawUriIsBlank() throws Exception {
|
||||
void returnsLocationRawUriFromCoreColumn() throws Exception {
|
||||
CapturingReader reader = new CapturingReader(new TdenginePage<>(
|
||||
List.of(row(
|
||||
"fact-blank-raw",
|
||||
"2026-06-29T05:00:01Z",
|
||||
"",
|
||||
"{\"rawArchiveUri\":\"archive://jt808/metadata-frame.bin\"}")),
|
||||
"archive://jt808/core-frame.bin")),
|
||||
Optional.empty()));
|
||||
Jt808LocationHistoryController controller = new Jt808LocationHistoryController(reader);
|
||||
|
||||
@@ -71,7 +70,7 @@ class Jt808LocationHistoryControllerTest {
|
||||
null);
|
||||
|
||||
assertThat(response.items().getFirst().rawUri())
|
||||
.isEqualTo("archive://jt808/metadata-frame.bin");
|
||||
.isEqualTo("archive://jt808/core-frame.bin");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -92,10 +91,10 @@ class Jt808LocationHistoryControllerTest {
|
||||
}
|
||||
|
||||
private static TdengineLocationRow row(String factId, String ts) {
|
||||
return row(factId, ts, "archive://jt808/frame-1.bin", "{\"messageId\":\"512\"}");
|
||||
return row(factId, ts, "archive://jt808/frame-1.bin");
|
||||
}
|
||||
|
||||
private static TdengineLocationRow row(String factId, String ts, String rawUri, String metadataJson) {
|
||||
private static TdengineLocationRow row(String factId, String ts, String rawUri) {
|
||||
Instant instant = Instant.parse(ts);
|
||||
return new TdengineLocationRow(
|
||||
instant,
|
||||
@@ -111,7 +110,6 @@ class Jt808LocationHistoryControllerTest {
|
||||
3L,
|
||||
null,
|
||||
rawUri,
|
||||
metadataJson,
|
||||
"JT808",
|
||||
"jt808:g7gps",
|
||||
"",
|
||||
|
||||
@@ -96,6 +96,7 @@ class Jt808RawFrameHistoryControllerTest {
|
||||
"",
|
||||
"222.66.200.68:41000",
|
||||
"{\"jt808.register.deviceId\":\"9963320\",\"jt808.register.plate\":\"沪A61559F\"}",
|
||||
"",
|
||||
"JT808",
|
||||
"jt808:13079963320",
|
||||
"unknown",
|
||||
|
||||
@@ -82,7 +82,6 @@ class LocationHistoryControllerTest {
|
||||
1,
|
||||
null,
|
||||
"",
|
||||
"{\"rawArchiveUri\":\"archive://raw.bin\"}",
|
||||
"MQTT_YUTONG",
|
||||
"LMRKH9AC2R1004087",
|
||||
"LMRKH9AC2R1004087",
|
||||
|
||||
@@ -91,7 +91,6 @@ class RawFrameHistoryControllerTest {
|
||||
524290,
|
||||
14504.446,
|
||||
raw.rawUri(),
|
||||
"{\"rawArchiveUri\":\"" + raw.rawUri() + "\"}",
|
||||
"JT808",
|
||||
"VIN-JT808-1",
|
||||
"VIN-JT808-1",
|
||||
@@ -183,6 +182,7 @@ class RawFrameHistoryControllerTest {
|
||||
"xinda",
|
||||
"{\"rawArchiveUri\":\"archive://2026/06/29/XINDA_PUSH/unknown/" + frameId
|
||||
+ ".bin\",\"plateNo\":\"粤BD12345\",\"rawJson\":\"{\\\"carName\\\":\\\"粤BD12345\\\"}\"}",
|
||||
"",
|
||||
"XINDA_PUSH",
|
||||
"LB9A32A24P0LS1270",
|
||||
"LB9A32A24P0LS1270",
|
||||
|
||||
@@ -2,9 +2,11 @@ package com.lingniu.ingest.eventhistory;
|
||||
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
import com.lingniu.ingest.eventfilestore.EventFileRecord;
|
||||
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
|
||||
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
|
||||
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
|
||||
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -12,6 +14,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
class TelemetryEnvelopeRecordMapperTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void mapsFullFieldEnvelopeToEventFileRecord() {
|
||||
VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
|
||||
@@ -91,4 +95,33 @@ class TelemetryEnvelopeRecordMapperTest {
|
||||
.containsEntry("tenant", "ln")
|
||||
.containsEntry("originalSource", "FUTURE_OEM");
|
||||
}
|
||||
|
||||
@Test
|
||||
void rawArchiveRecordStoresParsedJsonInPayloadOnly() throws Exception {
|
||||
VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
|
||||
.setSchemaVersion("1.0")
|
||||
.setEventId("raw-808-1")
|
||||
.setTraceId("trace-raw-1")
|
||||
.setVin("VIN808")
|
||||
.setSource("JT808")
|
||||
.setEventTimeMs(1_782_112_400_000L)
|
||||
.setIngestTimeMs(1_782_112_401_000L)
|
||||
.putMetadata("phone", "13307795425")
|
||||
.setRawArchive(RawArchiveRef.newBuilder()
|
||||
.setUri("archive://2026/06/30/JT808/VIN808/raw-808-1.bin")
|
||||
.setSizeBytes(64)
|
||||
.setParsedJson("{\"messageId\":\"0x0200\",\"body\":{\"speedKmh\":60.5}}")
|
||||
.build())
|
||||
.build();
|
||||
|
||||
EventFileRecord record = new TelemetryEnvelopeRecordMapper().toRecord(envelope);
|
||||
|
||||
assertThat(record.eventType()).isEqualTo("RAW_ARCHIVE");
|
||||
assertThat(record.metadata())
|
||||
.containsEntry("phone", "13307795425")
|
||||
.doesNotContainKey("rawArchiveParsedJson");
|
||||
var payload = objectMapper.readTree(record.payloadJson());
|
||||
assertThat(payload.path("parsed").path("messageId").asText()).isEqualTo("0x0200");
|
||||
assertThat(payload.path("parsed").path("body").path("speedKmh").asDouble()).isEqualTo(60.5);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,11 +35,14 @@ class EventHistoryAutoConfigurationTest {
|
||||
@Test
|
||||
void createsHistoryBeansWhenEnabled() {
|
||||
contextRunner
|
||||
.withPropertyValues("lingniu.ingest.event-history.enabled=true")
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.event-history.enabled=true",
|
||||
"lingniu.ingest.sink.mq.consumer.enabled=true")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class);
|
||||
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
|
||||
assertThat(context).hasSingleBean(EnvelopeConsumerProcessor.class);
|
||||
assertThat(context.getBeansOfType(EnvelopeConsumerProcessor.class))
|
||||
.containsOnlyKeys("eventHistoryEnvelopeConsumerProcessor");
|
||||
assertThat(context).hasSingleBean(EventHistoryController.class);
|
||||
assertThat(context).doesNotHaveBean(Gb32960DecodedFrameService.class);
|
||||
assertThat(context).doesNotHaveBean(Gb32960FrameController.class);
|
||||
|
||||
@@ -23,6 +23,9 @@ import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
|
||||
import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
@@ -36,7 +39,11 @@ import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.time.ZoneId;
|
||||
|
||||
@AutoConfiguration
|
||||
@AutoConfiguration(after = {
|
||||
DataSourceAutoConfiguration.class,
|
||||
JdbcTemplateAutoConfiguration.class,
|
||||
RedisAutoConfiguration.class
|
||||
})
|
||||
@EnableConfigurationProperties(VehicleStatProperties.class)
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.vehicle-stat", name = "enabled", havingValue = "true")
|
||||
public class VehicleStatAutoConfiguration {
|
||||
|
||||
@@ -4,13 +4,16 @@ import org.springframework.jdbc.core.JdbcTemplate;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
|
||||
public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageRepository {
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
|
||||
public JdbcJt808DailyMileageRepository(JdbcTemplate jdbc) {
|
||||
this.jdbc = jdbc;
|
||||
this.jdbc = Objects.requireNonNull(jdbc, "jdbc must not be null");
|
||||
initializeSchema();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -18,18 +21,23 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
|
||||
jdbc.update("""
|
||||
insert into vehicle_daily_mileage_jt808 (
|
||||
stat_date, vehicle_key, vin, phone, first_event_time, last_event_time,
|
||||
gps_mileage_km, speed_integral_km, odometer_mileage_km,
|
||||
daily_mileage_km, mileage_source, gps_mileage_km, speed_integral_km,
|
||||
odometer_mileage_km, first_total_mileage_km, last_total_mileage_km,
|
||||
accepted_points, bad_jump_segments, long_gap_segments, out_of_order_points,
|
||||
odometer_anomalies, data_quality
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
on duplicate key update
|
||||
vin = values(vin),
|
||||
phone = values(phone),
|
||||
first_event_time = values(first_event_time),
|
||||
last_event_time = values(last_event_time),
|
||||
daily_mileage_km = values(daily_mileage_km),
|
||||
mileage_source = values(mileage_source),
|
||||
gps_mileage_km = values(gps_mileage_km),
|
||||
speed_integral_km = values(speed_integral_km),
|
||||
odometer_mileage_km = values(odometer_mileage_km),
|
||||
first_total_mileage_km = values(first_total_mileage_km),
|
||||
last_total_mileage_km = values(last_total_mileage_km),
|
||||
accepted_points = values(accepted_points),
|
||||
bad_jump_segments = values(bad_jump_segments),
|
||||
long_gap_segments = values(long_gap_segments),
|
||||
@@ -44,9 +52,13 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
|
||||
result.phone(),
|
||||
timestamp(result.firstEventTime()),
|
||||
timestamp(result.lastEventTime()),
|
||||
result.dailyMileageKm(),
|
||||
result.mileageSource(),
|
||||
result.gpsMileageKm(),
|
||||
result.speedIntegralKm(),
|
||||
result.odometerMileageKm(),
|
||||
result.firstTotalMileageKm(),
|
||||
result.lastTotalMileageKm(),
|
||||
result.acceptedPoints(),
|
||||
result.badJumpSegments(),
|
||||
result.longGapSegments(),
|
||||
@@ -55,6 +67,59 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
|
||||
result.dataQuality());
|
||||
}
|
||||
|
||||
private void initializeSchema() {
|
||||
jdbc.execute("""
|
||||
create table if not exists vehicle_daily_mileage_jt808 (
|
||||
stat_date date not null,
|
||||
vehicle_key varchar(64) not null,
|
||||
vin varchar(32) null,
|
||||
phone varchar(32) null,
|
||||
first_event_time timestamp null,
|
||||
last_event_time timestamp null,
|
||||
daily_mileage_km decimal(12,3) not null default 0,
|
||||
mileage_source varchar(32) not null default 'INSUFFICIENT',
|
||||
gps_mileage_km decimal(12,3) not null,
|
||||
speed_integral_km decimal(12,3) not null,
|
||||
odometer_mileage_km decimal(12,3) null,
|
||||
first_total_mileage_km decimal(12,3) null,
|
||||
last_total_mileage_km decimal(12,3) null,
|
||||
accepted_points int not null,
|
||||
bad_jump_segments int not null,
|
||||
long_gap_segments int not null,
|
||||
out_of_order_points int not null,
|
||||
odometer_anomalies int not null,
|
||||
data_quality varchar(16) not null,
|
||||
updated_at timestamp not null default current_timestamp on update current_timestamp,
|
||||
primary key (stat_date, vehicle_key),
|
||||
key idx_vehicle_daily_mileage_jt808_vehicle_date (vehicle_key, stat_date),
|
||||
key idx_vehicle_daily_mileage_jt808_vin_date (vin, stat_date),
|
||||
key idx_vehicle_daily_mileage_jt808_phone_date (phone, stat_date)
|
||||
)
|
||||
""");
|
||||
ensureColumn("daily_mileage_km", "daily_mileage_km decimal(12,3) not null default 0");
|
||||
ensureColumn("mileage_source", "mileage_source varchar(32) not null default 'INSUFFICIENT'");
|
||||
ensureColumn("first_total_mileage_km", "first_total_mileage_km decimal(12,3) null");
|
||||
ensureColumn("last_total_mileage_km", "last_total_mileage_km decimal(12,3) null");
|
||||
}
|
||||
|
||||
private void ensureColumn(String columnName, String definition) {
|
||||
if (!columnExists(columnName)) {
|
||||
jdbc.execute("alter table vehicle_daily_mileage_jt808 add column " + definition);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean columnExists(String columnName) {
|
||||
Integer count = jdbc.queryForObject("""
|
||||
select count(*)
|
||||
from information_schema.columns
|
||||
where lower(table_name) = ?
|
||||
and lower(column_name) = ?
|
||||
""", Integer.class,
|
||||
"vehicle_daily_mileage_jt808",
|
||||
columnName.toLowerCase(Locale.ROOT));
|
||||
return count != null && count > 0;
|
||||
}
|
||||
|
||||
private static Timestamp timestamp(Instant instant) {
|
||||
return instant == null ? null : Timestamp.from(instant);
|
||||
}
|
||||
|
||||
@@ -10,9 +10,13 @@ public record Jt808DailyMileageResult(
|
||||
String phone,
|
||||
Instant firstEventTime,
|
||||
Instant lastEventTime,
|
||||
double dailyMileageKm,
|
||||
String mileageSource,
|
||||
double gpsMileageKm,
|
||||
double speedIntegralKm,
|
||||
Double odometerMileageKm,
|
||||
Double firstTotalMileageKm,
|
||||
Double lastTotalMileageKm,
|
||||
int acceptedPoints,
|
||||
int badJumpSegments,
|
||||
int longGapSegments,
|
||||
|
||||
@@ -12,6 +12,7 @@ public final class Jt808DailyMileageState {
|
||||
private Instant firstEventTime;
|
||||
private Instant lastEventTime;
|
||||
private Jt808LocationPoint lastPoint;
|
||||
private double firstTotalMileageKm = Double.NaN;
|
||||
private double lastTotalMileageKm = Double.NaN;
|
||||
private double gpsMileageKm;
|
||||
private double speedIntegralKm;
|
||||
@@ -47,6 +48,7 @@ public final class Jt808DailyMileageState {
|
||||
Instant firstEventTime,
|
||||
Instant lastEventTime,
|
||||
Jt808LocationPoint lastPoint,
|
||||
double firstTotalMileageKm,
|
||||
double lastTotalMileageKm,
|
||||
double gpsMileageKm,
|
||||
double speedIntegralKm,
|
||||
@@ -60,6 +62,7 @@ public final class Jt808DailyMileageState {
|
||||
state.firstEventTime = firstEventTime;
|
||||
state.lastEventTime = lastEventTime;
|
||||
state.lastPoint = lastPoint;
|
||||
state.firstTotalMileageKm = firstTotalMileageKm;
|
||||
state.lastTotalMileageKm = lastTotalMileageKm;
|
||||
state.gpsMileageKm = gpsMileageKm;
|
||||
state.speedIntegralKm = speedIntegralKm;
|
||||
@@ -82,7 +85,7 @@ public final class Jt808DailyMileageState {
|
||||
}
|
||||
if (lastPoint == null) {
|
||||
firstEventTime = point.eventTime();
|
||||
updateOdometer(point);
|
||||
updateOdometer(point, calculator);
|
||||
accept(point);
|
||||
return;
|
||||
}
|
||||
@@ -100,7 +103,7 @@ public final class Jt808DailyMileageState {
|
||||
gpsMileageKm += segment.gpsKm();
|
||||
speedIntegralKm += segment.speedIntegralKm();
|
||||
}
|
||||
updateOdometer(point);
|
||||
updateOdometer(point, calculator);
|
||||
accept(point);
|
||||
}
|
||||
|
||||
@@ -112,9 +115,13 @@ public final class Jt808DailyMileageState {
|
||||
phone,
|
||||
firstEventTime,
|
||||
lastEventTime,
|
||||
dailyMileageKm(),
|
||||
mileageSource(),
|
||||
gpsMileageKm,
|
||||
speedIntegralKm,
|
||||
odometerMileageKm > 0.0 ? odometerMileageKm : null,
|
||||
nullable(firstTotalMileageKm),
|
||||
nullable(lastTotalMileageKm),
|
||||
acceptedPoints,
|
||||
badJumpSegments,
|
||||
longGapSegments,
|
||||
@@ -129,11 +136,14 @@ public final class Jt808DailyMileageState {
|
||||
acceptedPoints++;
|
||||
}
|
||||
|
||||
private void updateOdometer(Jt808LocationPoint point) {
|
||||
private void updateOdometer(Jt808LocationPoint point, Jt808GpsMileageCalculator calculator) {
|
||||
if (point.totalMileageKm() == null || !Double.isFinite(point.totalMileageKm())) {
|
||||
return;
|
||||
}
|
||||
double current = point.totalMileageKm();
|
||||
if (!Double.isFinite(firstTotalMileageKm)) {
|
||||
firstTotalMileageKm = current;
|
||||
}
|
||||
if (!Double.isFinite(lastTotalMileageKm)) {
|
||||
lastTotalMileageKm = current;
|
||||
return;
|
||||
@@ -143,15 +153,56 @@ public final class Jt808DailyMileageState {
|
||||
lastTotalMileageKm = current;
|
||||
return;
|
||||
}
|
||||
odometerMileageKm += current - lastTotalMileageKm;
|
||||
double delta = current - lastTotalMileageKm;
|
||||
if (lastPoint != null && !calculator.plausibleOdometerDelta(lastPoint, point, delta)) {
|
||||
odometerAnomalies++;
|
||||
lastTotalMileageKm = current;
|
||||
return;
|
||||
}
|
||||
odometerMileageKm += delta;
|
||||
lastTotalMileageKm = current;
|
||||
}
|
||||
|
||||
private double dailyMileageKm() {
|
||||
if (hasUsableTotalMileage()) {
|
||||
return odometerMileageKm;
|
||||
}
|
||||
if (gpsMileageKm > 0.0) {
|
||||
return gpsMileageKm;
|
||||
}
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
private String mileageSource() {
|
||||
if (hasUsableTotalMileage()) {
|
||||
return odometerAnomalies > 0 ? "JT808_TOTAL_MILEAGE_PARTIAL" : "JT808_TOTAL_MILEAGE";
|
||||
}
|
||||
if (gpsMileageKm > 0.0) {
|
||||
return "GPS_DISTANCE";
|
||||
}
|
||||
return "INSUFFICIENT";
|
||||
}
|
||||
|
||||
private boolean hasUsableTotalMileage() {
|
||||
if (odometerMileageKm > 0.0) {
|
||||
return true;
|
||||
}
|
||||
return acceptedPoints >= 2
|
||||
&& odometerAnomalies == 0
|
||||
&& Double.isFinite(firstTotalMileageKm)
|
||||
&& Double.isFinite(lastTotalMileageKm)
|
||||
&& lastTotalMileageKm >= firstTotalMileageKm;
|
||||
}
|
||||
|
||||
private static Double nullable(double value) {
|
||||
return Double.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
private String dataQuality() {
|
||||
if (acceptedPoints < 2) {
|
||||
return "PARTIAL";
|
||||
}
|
||||
if (badJumpSegments > 0 || longGapSegments > 0 || vehicleKey.startsWith("jt808:")) {
|
||||
if (badJumpSegments > 0 || longGapSegments > 0 || odometerAnomalies > 0 || vehicleKey.startsWith("jt808:")) {
|
||||
return "PARTIAL";
|
||||
}
|
||||
return "GOOD";
|
||||
@@ -201,6 +252,10 @@ public final class Jt808DailyMileageState {
|
||||
return lastTotalMileageKm;
|
||||
}
|
||||
|
||||
public double firstTotalMileageKm() {
|
||||
return firstTotalMileageKm;
|
||||
}
|
||||
|
||||
public int acceptedPoints() {
|
||||
return acceptedPoints;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import java.time.Duration;
|
||||
public final class Jt808GpsMileageCalculator {
|
||||
|
||||
private static final double EARTH_RADIUS_KM = 6371.0088;
|
||||
private static final double ODOMETER_DELTA_TOLERANCE_KM = 0.1;
|
||||
|
||||
private final Duration maxSegmentGap;
|
||||
private final double maxImpliedSpeedKmh;
|
||||
@@ -61,6 +62,20 @@ public final class Jt808GpsMileageCalculator {
|
||||
return new SegmentResult(gpsKm, speedIntegralKm, true, false, false, false);
|
||||
}
|
||||
|
||||
public boolean plausibleOdometerDelta(Jt808LocationPoint previous,
|
||||
Jt808LocationPoint current,
|
||||
double deltaKm) {
|
||||
if (previous == null || current == null || !Double.isFinite(deltaKm) || deltaKm < 0.0) {
|
||||
return false;
|
||||
}
|
||||
Duration elapsed = Duration.between(previous.eventTime(), current.eventTime());
|
||||
if (elapsed.isZero() || elapsed.isNegative()) {
|
||||
return false;
|
||||
}
|
||||
double hours = elapsed.toMillis() / 3_600_000.0;
|
||||
return deltaKm <= maxImpliedSpeedKmh * hours + ODOMETER_DELTA_TOLERANCE_KM;
|
||||
}
|
||||
|
||||
private static double haversineKm(double lon1, double lat1, double lon2, double lat2) {
|
||||
double latRad1 = Math.toRadians(lat1);
|
||||
double latRad2 = Math.toRadians(lat2);
|
||||
|
||||
@@ -13,7 +13,8 @@ public final class Jt808LocationPointExtractor {
|
||||
public Optional<Jt808LocationPoint> extract(VehicleEnvelope envelope) {
|
||||
if (envelope == null
|
||||
|| !"JT808".equalsIgnoreCase(envelope.getSource())
|
||||
|| !envelope.hasTelemetrySnapshot()) {
|
||||
|| !envelope.hasTelemetrySnapshot()
|
||||
|| !"LOCATION".equalsIgnoreCase(envelope.getTelemetrySnapshot().getEventType())) {
|
||||
return Optional.empty();
|
||||
}
|
||||
Map<String, String> fields = fields(envelope);
|
||||
|
||||
@@ -64,7 +64,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
|
||||
String firstEventTime,
|
||||
String lastEventTime,
|
||||
PointSnapshot lastPoint,
|
||||
double lastTotalMileageKm,
|
||||
Double firstTotalMileageKm,
|
||||
Double lastTotalMileageKm,
|
||||
double gpsMileageKm,
|
||||
double speedIntegralKm,
|
||||
double odometerMileageKm,
|
||||
@@ -83,7 +84,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
|
||||
format(state.firstEventTime()),
|
||||
format(state.lastEventTime()),
|
||||
PointSnapshot.from(state.lastPoint()),
|
||||
state.lastTotalMileageKm(),
|
||||
finiteOrNull(state.firstTotalMileageKm()),
|
||||
finiteOrNull(state.lastTotalMileageKm()),
|
||||
state.gpsMileageKm(),
|
||||
state.speedIntegralKm(),
|
||||
state.odometerMileageKm(),
|
||||
@@ -103,7 +105,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
|
||||
parseInstant(firstEventTime),
|
||||
parseInstant(lastEventTime),
|
||||
lastPoint == null ? null : lastPoint.toPoint(),
|
||||
lastTotalMileageKm,
|
||||
nanIfNull(firstTotalMileageKm),
|
||||
nanIfNull(lastTotalMileageKm),
|
||||
gpsMileageKm,
|
||||
speedIntegralKm,
|
||||
odometerMileageKm,
|
||||
@@ -164,4 +167,12 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
|
||||
private static Instant parseInstant(String value) {
|
||||
return value == null || value.isBlank() ? null : Instant.parse(value);
|
||||
}
|
||||
|
||||
private static Double finiteOrNull(double value) {
|
||||
return Double.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
private static double nanIfNull(Double value) {
|
||||
return value == null ? Double.NaN : value;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,10 +21,13 @@ import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
@@ -117,6 +120,29 @@ class VehicleStatAutoConfigurationTest {
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void createsJt808MileageBeansAfterJdbcAutoConfiguration() {
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(
|
||||
JdbcTemplateAutoConfiguration.class,
|
||||
VehicleStatAutoConfiguration.class))
|
||||
.withBean(EnvelopeDeadLetterSink.class, () -> record -> {})
|
||||
.withBean(DataSource.class, () -> new DriverManagerDataSource(
|
||||
"jdbc:h2:mem:vehicle_stat_auto;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1",
|
||||
"sa",
|
||||
""))
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.vehicle-stat.enabled=true",
|
||||
"lingniu.ingest.vehicle-stat.jt808.enabled=true",
|
||||
"lingniu.ingest.vehicle-stat.jt808.state-store=memory")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(JdbcTemplate.class);
|
||||
assertThat(context).hasSingleBean(Jt808DailyMileageRepository.class);
|
||||
assertThat(context).hasSingleBean(Jt808MileageStreamProcessor.class);
|
||||
assertThat(context).hasSingleBean(VehicleStatEnvelopeIngestor.class);
|
||||
});
|
||||
}
|
||||
|
||||
private static final class InMemoryVehicleStatRepository implements VehicleStatRepository {
|
||||
@Override public void appendMileagePoint(String vin, MileagePoint point) {}
|
||||
@Override public List<MileagePoint> mileagePoints(String vin, LocalDate statDate) { return List.of(); }
|
||||
|
||||
@@ -24,30 +24,20 @@ class JdbcJt808DailyMileageRepositoryTest {
|
||||
"");
|
||||
jdbc = new JdbcTemplate(dataSource);
|
||||
jdbc.execute("drop table if exists vehicle_daily_mileage_jt808");
|
||||
jdbc.execute("""
|
||||
create table vehicle_daily_mileage_jt808 (
|
||||
stat_date date not null,
|
||||
vehicle_key varchar(64) not null,
|
||||
vin varchar(32) null,
|
||||
phone varchar(32) null,
|
||||
first_event_time timestamp null,
|
||||
last_event_time timestamp null,
|
||||
gps_mileage_km decimal(12,3) not null,
|
||||
speed_integral_km decimal(12,3) not null,
|
||||
odometer_mileage_km decimal(12,3) null,
|
||||
accepted_points int not null,
|
||||
bad_jump_segments int not null,
|
||||
long_gap_segments int not null,
|
||||
out_of_order_points int not null,
|
||||
odometer_anomalies int not null,
|
||||
data_quality varchar(16) not null,
|
||||
updated_at timestamp not null default current_timestamp,
|
||||
primary key (stat_date, vehicle_key)
|
||||
)
|
||||
""");
|
||||
repository = new JdbcJt808DailyMileageRepository(jdbc);
|
||||
}
|
||||
|
||||
@Test
|
||||
void initializesTableWhenMissing() {
|
||||
Integer count = jdbc.queryForObject("""
|
||||
select count(*)
|
||||
from information_schema.tables
|
||||
where table_name = 'vehicle_daily_mileage_jt808'
|
||||
""", Integer.class);
|
||||
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertsDailyMileageResult() {
|
||||
Jt808DailyMileageResult first = new Jt808DailyMileageResult(
|
||||
@@ -58,8 +48,12 @@ class JdbcJt808DailyMileageRepositoryTest {
|
||||
Instant.parse("2026-06-30T00:00:00Z"),
|
||||
Instant.parse("2026-06-30T00:01:00Z"),
|
||||
1.23456,
|
||||
"GPS_DISTANCE",
|
||||
1.23456,
|
||||
1.2,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
@@ -73,9 +67,13 @@ class JdbcJt808DailyMileageRepositoryTest {
|
||||
"13900000001",
|
||||
Instant.parse("2026-06-30T00:00:00Z"),
|
||||
Instant.parse("2026-06-30T00:02:00Z"),
|
||||
2.0,
|
||||
"JT808_TOTAL_MILEAGE",
|
||||
2.34567,
|
||||
2.3,
|
||||
2.0,
|
||||
100.0,
|
||||
102.0,
|
||||
3,
|
||||
1,
|
||||
0,
|
||||
@@ -87,14 +85,19 @@ class JdbcJt808DailyMileageRepositoryTest {
|
||||
repository.upsert(updated);
|
||||
|
||||
Map<String, Object> row = jdbc.queryForMap("""
|
||||
select vehicle_key, gps_mileage_km, odometer_mileage_km, accepted_points,
|
||||
bad_jump_segments, data_quality
|
||||
select vehicle_key, daily_mileage_km, mileage_source, gps_mileage_km,
|
||||
odometer_mileage_km, first_total_mileage_km, last_total_mileage_km,
|
||||
accepted_points, bad_jump_segments, data_quality
|
||||
from vehicle_daily_mileage_jt808
|
||||
where stat_date = '2026-06-30' and vehicle_key = 'VIN001'
|
||||
""");
|
||||
assertThat(row.get("vehicle_key")).isEqualTo("VIN001");
|
||||
assertThat(row.get("daily_mileage_km").toString()).isEqualTo("2.000");
|
||||
assertThat(row.get("mileage_source")).isEqualTo("JT808_TOTAL_MILEAGE");
|
||||
assertThat(row.get("gps_mileage_km").toString()).isEqualTo("2.346");
|
||||
assertThat(row.get("odometer_mileage_km").toString()).isEqualTo("2.000");
|
||||
assertThat(row.get("first_total_mileage_km").toString()).isEqualTo("100.000");
|
||||
assertThat(row.get("last_total_mileage_km").toString()).isEqualTo("102.000");
|
||||
assertThat(row.get("accepted_points")).isEqualTo(3);
|
||||
assertThat(row.get("bad_jump_segments")).isEqualTo(1);
|
||||
assertThat(row.get("data_quality")).isEqualTo("PARTIAL");
|
||||
|
||||
@@ -25,6 +25,10 @@ class Jt808DailyMileageStateTest {
|
||||
assertThat(state.gpsMileageKm()).isZero();
|
||||
assertThat(state.speedIntegralKm()).isZero();
|
||||
assertThat(state.odometerMileageKm()).isZero();
|
||||
assertThat(state.firstTotalMileageKm()).isEqualTo(100.0);
|
||||
assertThat(state.lastTotalMileageKm()).isEqualTo(100.0);
|
||||
assertThat(state.toResult().dailyMileageKm()).isZero();
|
||||
assertThat(state.toResult().mileageSource()).isEqualTo("INSUFFICIENT");
|
||||
assertThat(state.firstEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z"));
|
||||
assertThat(state.lastEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z"));
|
||||
}
|
||||
@@ -41,6 +45,12 @@ class Jt808DailyMileageStateTest {
|
||||
assertThat(state.gpsMileageKm()).isBetween(0.99, 1.01);
|
||||
assertThat(state.speedIntegralKm()).isBetween(0.99, 1.01);
|
||||
assertThat(state.odometerMileageKm()).isCloseTo(1.2, offset(0.000001));
|
||||
assertThat(state.firstTotalMileageKm()).isEqualTo(100.0);
|
||||
assertThat(state.lastTotalMileageKm()).isEqualTo(101.2);
|
||||
assertThat(state.toResult().dailyMileageKm()).isCloseTo(1.2, offset(0.000001));
|
||||
assertThat(state.toResult().mileageSource()).isEqualTo("JT808_TOTAL_MILEAGE");
|
||||
assertThat(state.toResult().firstTotalMileageKm()).isEqualTo(100.0);
|
||||
assertThat(state.toResult().lastTotalMileageKm()).isEqualTo(101.2);
|
||||
assertThat(state.badJumpSegments()).isZero();
|
||||
assertThat(state.longGapSegments()).isZero();
|
||||
}
|
||||
@@ -68,6 +78,36 @@ class Jt808DailyMileageStateTest {
|
||||
|
||||
assertThat(state.odometerMileageKm()).isZero();
|
||||
assertThat(state.odometerAnomalies()).isEqualTo(1);
|
||||
assertThat(state.toResult().dailyMileageKm()).isGreaterThan(0.9);
|
||||
assertThat(state.toResult().mileageSource()).isEqualTo("GPS_DISTANCE");
|
||||
}
|
||||
|
||||
@Test
|
||||
void odometerJumpAboveConfiguredSpeedIsIgnoredAndCounted() {
|
||||
Jt808DailyMileageState state = Jt808DailyMileageState.empty("VIN001", "VIN001", "13900000001",
|
||||
LocalDate.of(2026, 6, 30));
|
||||
state.apply(point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0, 100.0), calculator);
|
||||
|
||||
state.apply(point("2026-06-30T00:01:00Z", 120.000100, 30.000000, 36.0, 200.0), calculator);
|
||||
|
||||
assertThat(state.odometerMileageKm()).isZero();
|
||||
assertThat(state.odometerAnomalies()).isEqualTo(1);
|
||||
assertThat(state.toResult().dataQuality()).isEqualTo("PARTIAL");
|
||||
}
|
||||
|
||||
@Test
|
||||
void gpsDistanceIsFallbackWhenTotalMileageIsMissing() {
|
||||
Jt808DailyMileageState state = Jt808DailyMileageState.empty("VIN001", "VIN001", "13900000001",
|
||||
LocalDate.of(2026, 6, 30));
|
||||
state.apply(point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0, null), calculator);
|
||||
|
||||
state.apply(point("2026-06-30T00:01:40Z", 120.010370, 30.000000, 36.0, null), calculator);
|
||||
|
||||
assertThat(state.toResult().dailyMileageKm()).isBetween(0.99, 1.01);
|
||||
assertThat(state.toResult().mileageSource()).isEqualTo("GPS_DISTANCE");
|
||||
assertThat(state.toResult().odometerMileageKm()).isNull();
|
||||
assertThat(state.toResult().firstTotalMileageKm()).isNull();
|
||||
assertThat(state.toResult().lastTotalMileageKm()).isNull();
|
||||
}
|
||||
|
||||
private static Jt808LocationPoint point(String time,
|
||||
|
||||
@@ -71,6 +71,18 @@ class Jt808GpsMileageCalculatorTest {
|
||||
assertThat(result.gpsKm()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatesPlausibleOdometerDeltaByConfiguredSpeed() {
|
||||
assertThat(calculator.plausibleOdometerDelta(
|
||||
point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0),
|
||||
point("2026-06-30T00:01:00Z", 120.001000, 30.000000, 36.0),
|
||||
2.0)).isTrue();
|
||||
assertThat(calculator.plausibleOdometerDelta(
|
||||
point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0),
|
||||
point("2026-06-30T00:01:00Z", 120.001000, 30.000000, 36.0),
|
||||
20.0)).isFalse();
|
||||
}
|
||||
|
||||
private static Jt808LocationPoint point(String time, double longitude, double latitude, Double speedKmh) {
|
||||
return new Jt808LocationPoint(
|
||||
"VIN001",
|
||||
|
||||
@@ -57,6 +57,21 @@ class Jt808LocationPointExtractorTest {
|
||||
field("longitude", "118.901034")))).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void ignoresJt808NonLocationSnapshot() {
|
||||
VehicleEnvelope envelope = envelope("JT808", "VIN001", "13900000001",
|
||||
field("longitude", "118.901034"),
|
||||
field("latitude", "31.946325"))
|
||||
.toBuilder()
|
||||
.setTelemetrySnapshot(TelemetrySnapshot.newBuilder()
|
||||
.setEventType("REGISTER")
|
||||
.addFields(field("longitude", "118.901034"))
|
||||
.addFields(field("latitude", "31.946325")))
|
||||
.build();
|
||||
|
||||
assertThat(extractor.extract(envelope)).isEmpty();
|
||||
}
|
||||
|
||||
private static VehicleEnvelope envelope(String source, String vin, String phone, TelemetryField... fields) {
|
||||
TelemetrySnapshot.Builder snapshot = TelemetrySnapshot.newBuilder().setEventType("LOCATION");
|
||||
for (TelemetryField field : fields) {
|
||||
|
||||
@@ -30,6 +30,8 @@ class Jt808MileageStreamProcessorTest {
|
||||
assertThat(state.gpsMileageKm()).isGreaterThan(0.9);
|
||||
assertThat(repository.results).hasSize(2);
|
||||
assertThat(repository.results.getLast().vehicleKey()).isEqualTo("VIN001");
|
||||
assertThat(repository.results.getLast().dailyMileageKm()).isEqualTo(1.0);
|
||||
assertThat(repository.results.getLast().mileageSource()).isEqualTo("JT808_TOTAL_MILEAGE");
|
||||
assertThat(repository.results.getLast().odometerMileageKm()).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
|
||||
@@ -36,11 +36,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
private static final TypeReference<Map<String, String>> STRING_MAP =
|
||||
new TypeReference<>() {};
|
||||
private static final String HEADER_VERSION = "event-records-v1";
|
||||
private static final int VEHICLE_BUCKETS = 64;
|
||||
|
||||
private final Path root;
|
||||
private final Path indexPath;
|
||||
private final ZoneId partitionZone;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final boolean writeParquetEnabled;
|
||||
private volatile boolean indexInitialized;
|
||||
|
||||
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) {
|
||||
@@ -48,6 +50,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
}
|
||||
|
||||
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) {
|
||||
this(root, partitionZone, objectMapper, true);
|
||||
}
|
||||
|
||||
public DuckDbParquetEventFileStore(Path root,
|
||||
ZoneId partitionZone,
|
||||
ObjectMapper objectMapper,
|
||||
boolean writeParquetEnabled) {
|
||||
if (root == null) {
|
||||
throw new IllegalArgumentException("root must not be null");
|
||||
}
|
||||
@@ -55,6 +64,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
this.indexPath = this.root.resolve("events.duckdb");
|
||||
this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone;
|
||||
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
|
||||
this.writeParquetEnabled = writeParquetEnabled;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -67,16 +77,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
Partition partition = partition(record);
|
||||
byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
|
||||
}
|
||||
for (Map.Entry<Partition, List<EventFileRecord>> entry : byPartition.entrySet()) {
|
||||
writePartition(entry.getKey(), entry.getValue());
|
||||
appendIndex(entry.getKey(), entry.getValue());
|
||||
if (writeParquetEnabled) {
|
||||
writePartitions(byPartition);
|
||||
}
|
||||
appendIndex(byPartition);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EventFileRecord> query(EventFileQuery query) throws IOException {
|
||||
// 有 VIN 时直接扫该车分区 parquet,避免先查总索引再和车辆文件 join。
|
||||
if (query.vin() != null) {
|
||||
// 有 VIN 且启用 Parquet 副本时,可直接扫分区文件;索引-only 模式走 DuckDB 主索引。
|
||||
if (query.vin() != null && writeParquetEnabled) {
|
||||
return queryParquet(query);
|
||||
}
|
||||
// 无 VIN 的管理类查询走 DuckDB 索引,代价更高但不影响高频单车查询路径。
|
||||
@@ -130,6 +140,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
if (query.eventTimeTo() != null) {
|
||||
predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli());
|
||||
}
|
||||
if (query.cursorEventTime() != null) {
|
||||
predicates.add(cursorPredicateSql(query));
|
||||
}
|
||||
// 这里的 SQL 只拼接经过 escape 的内部值;外部 rawArchiveUri 查询使用 PreparedStatement。
|
||||
String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n";
|
||||
String sql = """
|
||||
@@ -182,6 +195,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
if (query.eventTimeTo() != null) {
|
||||
where.append(" AND event_time_ms <= ?\n");
|
||||
}
|
||||
if (query.cursorEventTime() != null) {
|
||||
where.append(cursorPredicatePrepared(query.order()));
|
||||
}
|
||||
String sql = """
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, ingest_time_ms,
|
||||
@@ -209,6 +225,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
if (query.eventTimeTo() != null) {
|
||||
ps.setLong(index++, query.eventTimeTo().toEpochMilli());
|
||||
}
|
||||
if (query.cursorEventTime() != null) {
|
||||
long eventTimeMs = query.cursorEventTime().toEpochMilli();
|
||||
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
|
||||
ps.setLong(index++, eventTimeMs);
|
||||
ps.setLong(index++, eventTimeMs);
|
||||
ps.setLong(index++, ingestTimeMs);
|
||||
ps.setLong(index++, eventTimeMs);
|
||||
ps.setLong(index++, ingestTimeMs);
|
||||
ps.setString(index++, query.cursorEventId());
|
||||
}
|
||||
ps.setInt(index, query.limit());
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
List<EventFileRecord> result = new ArrayList<>();
|
||||
@@ -222,16 +248,43 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
}
|
||||
}
|
||||
|
||||
private void writePartition(Partition partition, List<EventFileRecord> records) throws IOException {
|
||||
Path dir = partition.dir(root);
|
||||
Files.createDirectories(dir);
|
||||
Path partFile = dir.resolve("events.parquet");
|
||||
Path tempFile = dir.resolve("events-" + UUID.randomUUID() + ".tmp.parquet");
|
||||
private static String cursorPredicateSql(EventFileQuery query) {
|
||||
long eventTimeMs = query.cursorEventTime().toEpochMilli();
|
||||
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
|
||||
String eventId = escapeSql(query.cursorEventId());
|
||||
String cmp = query.order() == EventFileQuery.Order.DESC ? "<" : ">";
|
||||
return """
|
||||
(
|
||||
event_time_ms %s %d
|
||||
OR (event_time_ms = %d AND ingest_time_ms %s %d)
|
||||
OR (event_time_ms = %d AND ingest_time_ms = %d AND event_id %s '%s')
|
||||
)
|
||||
""".formatted(cmp, eventTimeMs, eventTimeMs, cmp, ingestTimeMs,
|
||||
eventTimeMs, ingestTimeMs, cmp, eventId);
|
||||
}
|
||||
|
||||
private static String cursorPredicatePrepared(EventFileQuery.Order order) {
|
||||
String cmp = order == EventFileQuery.Order.DESC ? "<" : ">";
|
||||
return """
|
||||
AND (
|
||||
event_time_ms %s ?
|
||||
OR (event_time_ms = ? AND ingest_time_ms %s ?)
|
||||
OR (event_time_ms = ? AND ingest_time_ms = ? AND event_id %s ?)
|
||||
)
|
||||
""".formatted(cmp, cmp, cmp);
|
||||
}
|
||||
|
||||
private void writePartitions(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
|
||||
if (recordsByPartition.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<Path> tempFiles = new ArrayList<>();
|
||||
|
||||
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute("""
|
||||
CREATE TEMPORARY TABLE event_records (
|
||||
partition_id INTEGER,
|
||||
event_id VARCHAR,
|
||||
protocol VARCHAR,
|
||||
event_type VARCHAR,
|
||||
@@ -246,54 +299,62 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
)
|
||||
""");
|
||||
try (PreparedStatement ps = connection.prepareStatement("""
|
||||
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""")) {
|
||||
for (EventFileRecord record : records) {
|
||||
ps.setString(1, record.eventId());
|
||||
ps.setString(2, record.protocol().name());
|
||||
ps.setString(3, record.eventType());
|
||||
ps.setString(4, record.vin());
|
||||
ps.setLong(5, record.eventTime().toEpochMilli());
|
||||
ps.setString(6, record.eventTime().toString());
|
||||
ps.setLong(7, record.ingestTime().toEpochMilli());
|
||||
ps.setString(8, record.ingestTime().toString());
|
||||
ps.setString(9, record.rawArchiveUri());
|
||||
ps.setString(10, objectMapper.writeValueAsString(record.metadata()));
|
||||
ps.setString(11, record.payloadJson());
|
||||
ps.addBatch();
|
||||
int partitionId = 0;
|
||||
for (List<EventFileRecord> records : recordsByPartition.values()) {
|
||||
for (EventFileRecord record : records) {
|
||||
ps.setInt(1, partitionId);
|
||||
ps.setString(2, record.eventId());
|
||||
ps.setString(3, record.protocol().name());
|
||||
ps.setString(4, record.eventType());
|
||||
ps.setString(5, record.vin());
|
||||
ps.setLong(6, record.eventTime().toEpochMilli());
|
||||
ps.setString(7, record.eventTime().toString());
|
||||
ps.setLong(8, record.ingestTime().toEpochMilli());
|
||||
ps.setString(9, record.ingestTime().toString());
|
||||
ps.setString(10, record.rawArchiveUri());
|
||||
ps.setString(11, objectMapper.writeValueAsString(record.metadata()));
|
||||
ps.setString(12, record.payloadJson());
|
||||
ps.addBatch();
|
||||
}
|
||||
partitionId++;
|
||||
}
|
||||
ps.executeBatch();
|
||||
}
|
||||
if (Files.exists(partFile)) {
|
||||
// DuckDB 不能原地追加 parquet;先 UNION 旧文件和本批数据,再原子替换分区文件。
|
||||
int partitionId = 0;
|
||||
for (Partition partition : recordsByPartition.keySet()) {
|
||||
Path dir = partition.dir(root);
|
||||
Files.createDirectories(dir);
|
||||
String fragmentId = UUID.randomUUID().toString();
|
||||
Path partFile = dir.resolve("events-" + fragmentId + ".parquet");
|
||||
Path tempFile = dir.resolve("events-" + fragmentId + ".tmp.parquet");
|
||||
tempFiles.add(tempFile);
|
||||
statement.execute("""
|
||||
CREATE TEMPORARY TABLE combined_records AS
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, event_time, ingest_time_ms, ingest_time,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM read_parquet('%s')
|
||||
UNION ALL
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, event_time, ingest_time_ms, ingest_time,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM event_records
|
||||
""".formatted(escapeSql(partFile)));
|
||||
statement.execute("COPY combined_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
|
||||
} else {
|
||||
statement.execute("COPY event_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
|
||||
COPY (
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, event_time, ingest_time_ms, ingest_time,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM event_records
|
||||
WHERE partition_id = %d
|
||||
) TO '%s' (FORMAT parquet)
|
||||
""".formatted(partitionId, escapeSql(tempFile)));
|
||||
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
partitionId++;
|
||||
}
|
||||
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("write parquet event store failed: " + partFile, e);
|
||||
throw new IOException("write parquet event store failed", e);
|
||||
} finally {
|
||||
Files.deleteIfExists(tempFile);
|
||||
for (Path tempFile : tempFiles) {
|
||||
Files.deleteIfExists(tempFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void appendIndex(Partition partition, List<EventFileRecord> records) throws IOException {
|
||||
private void appendIndex(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
|
||||
ensureIndexInitialized();
|
||||
try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) {
|
||||
insertIndex(connection, partition, records);
|
||||
insertIndex(connection, recordsByPartition);
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("write duckdb event index failed", e);
|
||||
}
|
||||
@@ -364,23 +425,26 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
""".formatted(fileList));
|
||||
}
|
||||
|
||||
private void insertIndex(Connection connection, Partition partition, List<EventFileRecord> records)
|
||||
private void insertIndex(Connection connection, Map<Partition, List<EventFileRecord>> recordsByPartition)
|
||||
throws SQLException, IOException {
|
||||
try (PreparedStatement ps = connection.prepareStatement("""
|
||||
INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""")) {
|
||||
for (EventFileRecord record : records) {
|
||||
ps.setString(1, record.eventId());
|
||||
ps.setString(2, record.protocol().name());
|
||||
ps.setString(3, record.eventType());
|
||||
ps.setString(4, record.vin());
|
||||
ps.setLong(5, record.eventTime().toEpochMilli());
|
||||
ps.setLong(6, record.ingestTime().toEpochMilli());
|
||||
ps.setString(7, partition.date().toString());
|
||||
ps.setString(8, record.rawArchiveUri());
|
||||
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
|
||||
ps.setString(10, record.payloadJson());
|
||||
ps.addBatch();
|
||||
for (Map.Entry<Partition, List<EventFileRecord>> entry : recordsByPartition.entrySet()) {
|
||||
Partition partition = entry.getKey();
|
||||
for (EventFileRecord record : entry.getValue()) {
|
||||
ps.setString(1, record.eventId());
|
||||
ps.setString(2, record.protocol().name());
|
||||
ps.setString(3, record.eventType());
|
||||
ps.setString(4, record.vin());
|
||||
ps.setLong(5, record.eventTime().toEpochMilli());
|
||||
ps.setLong(6, record.ingestTime().toEpochMilli());
|
||||
ps.setString(7, partition.date().toString());
|
||||
ps.setString(8, record.rawArchiveUri());
|
||||
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
|
||||
ps.setString(10, record.payloadJson());
|
||||
ps.addBatch();
|
||||
}
|
||||
}
|
||||
ps.executeBatch();
|
||||
}
|
||||
@@ -390,15 +454,22 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
List<Path> files = new ArrayList<>();
|
||||
LocalDate date = query.dateFrom();
|
||||
while (!date.isAfter(query.dateTo())) {
|
||||
Path dir = query.vin() == null
|
||||
? partitionDir(query.protocol(), date)
|
||||
: partitionDir(query.protocol(), date).resolve("vehicle=" + storageName(query.vin()));
|
||||
// 目录结构把 VIN 放在 date 下,单车多日查询只需要遍历目标日期内的目标车辆目录。
|
||||
if (Files.isDirectory(dir)) {
|
||||
try (Stream<Path> stream = Files.walk(dir)) {
|
||||
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
|
||||
.sorted()
|
||||
.forEach(files::add);
|
||||
Path dateDir = partitionDir(query.protocol(), date);
|
||||
List<Path> candidateDirs = query.vin() == null
|
||||
? List.of(dateDir)
|
||||
: List.of(
|
||||
// 新布局:高写入路径按协议/日期写一个批次文件,VIN 由 Parquet/索引字段过滤。
|
||||
dateDir.resolve("header=" + HEADER_VERSION),
|
||||
// 兼容已落盘的旧布局。
|
||||
dateDir.resolve("bucket=" + bucketName(query.vin())),
|
||||
dateDir.resolve("vehicle=" + storageName(query.vin())));
|
||||
for (Path dir : candidateDirs) {
|
||||
if (Files.isDirectory(dir)) {
|
||||
try (Stream<Path> stream = Files.walk(dir)) {
|
||||
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
|
||||
.sorted()
|
||||
.forEach(files::add);
|
||||
}
|
||||
}
|
||||
}
|
||||
date = date.plusDays(1);
|
||||
@@ -434,8 +505,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
private Partition partition(EventFileRecord record) {
|
||||
return new Partition(
|
||||
record.protocol(),
|
||||
LocalDate.ofInstant(record.eventTime(), partitionZone),
|
||||
storageName(record.vin()));
|
||||
LocalDate.ofInstant(record.eventTime(), partitionZone));
|
||||
}
|
||||
|
||||
private Path partitionDir(ProtocolId protocol, LocalDate date) {
|
||||
@@ -488,11 +558,18 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
return vin.replaceAll("[^A-Za-z0-9._-]", "_");
|
||||
}
|
||||
|
||||
private record Partition(ProtocolId protocol, LocalDate date, String vehicle) {
|
||||
private static String bucketName(String vin) {
|
||||
String storageName = storageName(vin);
|
||||
if ("_unknown".equals(storageName)) {
|
||||
return "_unknown";
|
||||
}
|
||||
return "%02d".formatted(Math.floorMod(storageName.hashCode(), VEHICLE_BUCKETS));
|
||||
}
|
||||
|
||||
private record Partition(ProtocolId protocol, LocalDate date) {
|
||||
private Path dir(Path root) {
|
||||
return root.resolve("protocol=" + protocol.name())
|
||||
.resolve("date=" + date)
|
||||
.resolve("vehicle=" + vehicle)
|
||||
.resolve("header=" + HEADER_VERSION);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,10 @@ public record EventFileQuery(
|
||||
Order order,
|
||||
int limit,
|
||||
String vin,
|
||||
String eventType
|
||||
String eventType,
|
||||
Instant cursorEventTime,
|
||||
Instant cursorIngestTime,
|
||||
String cursorEventId
|
||||
) {
|
||||
public enum Order {
|
||||
ASC,
|
||||
@@ -45,6 +48,24 @@ public record EventFileQuery(
|
||||
limit = limit <= 0 ? 100 : limit;
|
||||
vin = vin == null || vin.isBlank() ? null : vin.trim();
|
||||
eventType = eventType == null || eventType.isBlank() ? null : eventType.trim();
|
||||
cursorEventId = cursorEventId == null || cursorEventId.isBlank() ? null : cursorEventId.trim();
|
||||
if ((cursorEventTime == null || cursorIngestTime == null || cursorEventId == null)
|
||||
&& !(cursorEventTime == null && cursorIngestTime == null && cursorEventId == null)) {
|
||||
throw new IllegalArgumentException("cursor eventTime, ingestTime and eventId must be provided together");
|
||||
}
|
||||
}
|
||||
|
||||
public EventFileQuery(ProtocolId protocol,
|
||||
LocalDate dateFrom,
|
||||
LocalDate dateTo,
|
||||
Instant eventTimeFrom,
|
||||
Instant eventTimeTo,
|
||||
Order order,
|
||||
int limit,
|
||||
String vin,
|
||||
String eventType) {
|
||||
this(protocol, dateFrom, dateTo, eventTimeFrom, eventTimeTo, order, limit,
|
||||
vin, eventType, null, null, null);
|
||||
}
|
||||
|
||||
public EventFileQuery(ProtocolId protocol,
|
||||
|
||||
@@ -226,9 +226,9 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
|
||||
return payload;
|
||||
}
|
||||
|
||||
private static Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
|
||||
String rawArchiveKey,
|
||||
String rawArchiveUri) {
|
||||
private Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
|
||||
String rawArchiveKey,
|
||||
String rawArchiveUri) throws IOException {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("eventId", raw.eventId());
|
||||
payload.put("vin", raw.vin());
|
||||
@@ -241,10 +241,18 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
|
||||
payload.put("command", hex(raw.command()));
|
||||
payload.put("infoType", hex(raw.infoType()));
|
||||
payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length);
|
||||
putParsedJson(payload, raw.parsedJson());
|
||||
payload.put("metadata", raw.metadata());
|
||||
return payload;
|
||||
}
|
||||
|
||||
private void putParsedJson(Map<String, Object> payload, String parsedJson) throws IOException {
|
||||
if (parsedJson == null || parsedJson.isBlank()) {
|
||||
return;
|
||||
}
|
||||
payload.put("parsed", objectMapper.readTree(parsedJson));
|
||||
}
|
||||
|
||||
private static String hex(int value) {
|
||||
return "0x" + String.format("%04X", value);
|
||||
}
|
||||
|
||||
@@ -44,7 +44,8 @@ public class EventFileStoreAutoConfiguration {
|
||||
return new DuckDbParquetEventFileStore(
|
||||
Path.of(properties.getPath()),
|
||||
ZoneId.of(properties.getZoneId()),
|
||||
mapper);
|
||||
mapper,
|
||||
properties.isWriteParquetEnabled());
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
||||
@@ -11,10 +11,10 @@ public class EventFileStoreProperties {
|
||||
private boolean enabled = false;
|
||||
|
||||
/**
|
||||
* Parquet 文件库根路径。
|
||||
* 历史库根路径。
|
||||
*
|
||||
* <p>实际结构为 {@code protocol=GB32960/date=yyyy-MM-dd/vehicle=VIN/header=event-records-v1/events.parquet},
|
||||
* 另有 {@code events.duckdb} sidecar index 放在根路径下。
|
||||
* <p>{@code events.duckdb} 是可查询主索引;开启 Parquet 副本时,还会在该根目录下按
|
||||
* {@code protocol/date/header} 保存追加文件。
|
||||
*/
|
||||
private String path = "./event-store";
|
||||
|
||||
@@ -39,6 +39,14 @@ public class EventFileStoreProperties {
|
||||
*/
|
||||
private long flushIntervalMillis = 1000;
|
||||
|
||||
/**
|
||||
* 是否同步写 Parquet 副本。
|
||||
*
|
||||
* <p>高写入生产链路可以关闭,只保留 DuckDB JSON 主索引和 raw archive 引用;需要离线副本或
|
||||
* 索引重建能力时再开启。
|
||||
*/
|
||||
private boolean writeParquetEnabled = true;
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
}
|
||||
@@ -78,4 +86,12 @@ public class EventFileStoreProperties {
|
||||
public void setFlushIntervalMillis(long flushIntervalMillis) {
|
||||
this.flushIntervalMillis = flushIntervalMillis;
|
||||
}
|
||||
|
||||
public boolean isWriteParquetEnabled() {
|
||||
return writeParquetEnabled;
|
||||
}
|
||||
|
||||
public void setWriteParquetEnabled(boolean writeParquetEnabled) {
|
||||
this.writeParquetEnabled = writeParquetEnabled;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.lingniu.ingest.eventfilestore;
|
||||
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
@@ -11,6 +12,7 @@ import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -28,7 +30,7 @@ class DuckDbParquetEventFileStoreTest {
|
||||
store.appendAll(List.of(first, second));
|
||||
|
||||
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
|
||||
assertThat(parquetFiles(dateDir)).hasSize(2);
|
||||
assertThat(parquetFiles(dateDir)).hasSize(1);
|
||||
|
||||
EventFileQuery ascending = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
@@ -135,16 +137,13 @@ class DuckDbParquetEventFileStoreTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void appendsSameVehicleDayHeaderToOneParquetFile() throws Exception {
|
||||
void appendsSameVehicleDayHeaderAsAppendOnlyParquetFragments() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME"));
|
||||
store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME"));
|
||||
|
||||
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
|
||||
.resolve("date=2026-06-22")
|
||||
.resolve("vehicle=VIN-SAME")
|
||||
.resolve("header=event-records-v1");
|
||||
assertThat(parquetFiles(vehicleDayDir)).containsExactly(vehicleDayDir.resolve("events.parquet"));
|
||||
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
|
||||
assertThat(parquetFiles(dateDir)).hasSize(2);
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
@@ -159,19 +158,54 @@ class DuckDbParquetEventFileStoreTest {
|
||||
.containsExactly("first", "second");
|
||||
}
|
||||
|
||||
@Test
|
||||
void writesManyVehiclesIntoBoundedDailyLayoutInsteadOfVehicleDirectories() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
List<EventFileRecord> records = IntStream.range(0, 80)
|
||||
.mapToObj(index -> rawRecord(
|
||||
"raw-" + index,
|
||||
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
|
||||
"VIN-" + index))
|
||||
.toList();
|
||||
|
||||
store.appendAll(records);
|
||||
|
||||
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
|
||||
try (var files = Files.list(dateDir)) {
|
||||
List<String> partitionDirs = files
|
||||
.filter(Files::isDirectory)
|
||||
.map(path -> path.getFileName().toString())
|
||||
.sorted()
|
||||
.toList();
|
||||
assertThat(partitionDirs).noneMatch(name -> name.startsWith("vehicle="));
|
||||
assertThat(partitionDirs).hasSizeLessThan(records.size());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void writesOneDailyParquetFragmentForOneBatchAcrossManyVehicles() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
List<EventFileRecord> records = IntStream.range(0, 80)
|
||||
.mapToObj(index -> rawRecord(
|
||||
"daily-raw-" + index,
|
||||
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
|
||||
"VIN-DAILY-" + index))
|
||||
.toList();
|
||||
|
||||
store.appendAll(records);
|
||||
|
||||
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
|
||||
assertThat(parquetFiles(dateDir)).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED"));
|
||||
|
||||
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
|
||||
.resolve("date=2026-06-22")
|
||||
.resolve("vehicle=VIN-INDEXED")
|
||||
.resolve("header=event-records-v1");
|
||||
try (var files = Files.list(vehicleDayDir)) {
|
||||
for (Path file : files.filter(p -> p.getFileName().toString().endsWith(".parquet")).toList()) {
|
||||
Files.delete(file);
|
||||
}
|
||||
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
|
||||
for (Path file : parquetFiles(dateDir)) {
|
||||
Files.delete(file);
|
||||
}
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
@@ -219,6 +253,31 @@ class DuckDbParquetEventFileStoreTest {
|
||||
assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri());
|
||||
}
|
||||
|
||||
@Test
|
||||
void canQueryVinRecordsFromIndexWhenParquetWritingIsDisabled() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(
|
||||
tempDir,
|
||||
ZoneId.of("Asia/Shanghai"),
|
||||
new ObjectMapper(),
|
||||
false);
|
||||
EventFileRecord record = rawRecord("index-only", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEX-ONLY");
|
||||
|
||||
store.append(record);
|
||||
|
||||
assertThat(parquetFiles(tempDir)).isEmpty();
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.DESC,
|
||||
10,
|
||||
"VIN-INDEX-ONLY",
|
||||
"RAW_ARCHIVE");
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("index-only");
|
||||
}
|
||||
|
||||
private static List<Path> parquetFiles(Path root) throws Exception {
|
||||
try (var files = Files.walk(root)) {
|
||||
return files.filter(p -> p.getFileName().toString().endsWith(".parquet"))
|
||||
|
||||
@@ -20,12 +20,15 @@ class EventFileStoreAutoConfigurationTest {
|
||||
contextRunner
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.event-file-store.enabled=true",
|
||||
"lingniu.ingest.event-file-store.path=target/test-event-store")
|
||||
"lingniu.ingest.event-file-store.path=target/test-event-store",
|
||||
"lingniu.ingest.event-file-store.write-parquet-enabled=false")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(EventFileStore.class);
|
||||
assertThat(context).hasSingleBean(EventFileStoreSink.class);
|
||||
assertThat(context.getBean(EventFileStoreProperties.class).getPath())
|
||||
.isEqualTo(Path.of("target/test-event-store").toString());
|
||||
assertThat(context.getBean(EventFileStoreProperties.class).isWriteParquetEnabled())
|
||||
.isFalse();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -92,6 +92,7 @@ public final class EnvelopeMapper {
|
||||
.setUri(uri)
|
||||
.setChecksum(checksum)
|
||||
.setSizeBytes(size)
|
||||
.setParsedJson(nullToEmpty(ra.parsedJson()))
|
||||
.build());
|
||||
b.setRawFrameFact(com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload.newBuilder()
|
||||
.setFrameId(frameId)
|
||||
|
||||
@@ -10,17 +10,19 @@ import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCloseable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(KafkaEnvelopeConsumerRunner.class);
|
||||
|
||||
private final List<KafkaEnvelopeConsumerWorker> workers;
|
||||
private final Supplier<List<KafkaEnvelopeConsumerWorker>> workersSupplier;
|
||||
private final Duration pollTimeout;
|
||||
private final Duration loopBackoff;
|
||||
private final boolean autoStartup;
|
||||
private final AtomicBoolean running = new AtomicBoolean(false);
|
||||
private final AtomicBoolean closed = new AtomicBoolean(false);
|
||||
private volatile List<KafkaEnvelopeConsumerWorker> workers;
|
||||
private ExecutorService executor;
|
||||
|
||||
public KafkaEnvelopeConsumerRunner(List<KafkaEnvelopeConsumerWorker> workers,
|
||||
@@ -30,14 +32,28 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
|
||||
if (workers == null || workers.isEmpty()) {
|
||||
throw new IllegalArgumentException("workers must not be empty");
|
||||
}
|
||||
this.workersSupplier = () -> List.copyOf(workers);
|
||||
this.workers = List.copyOf(workers);
|
||||
this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout;
|
||||
this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff;
|
||||
this.autoStartup = autoStartup;
|
||||
}
|
||||
|
||||
public KafkaEnvelopeConsumerRunner(Supplier<List<KafkaEnvelopeConsumerWorker>> workersSupplier,
|
||||
Duration pollTimeout,
|
||||
Duration loopBackoff,
|
||||
boolean autoStartup) {
|
||||
if (workersSupplier == null) {
|
||||
throw new IllegalArgumentException("workersSupplier must not be null");
|
||||
}
|
||||
this.workersSupplier = workersSupplier;
|
||||
this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout;
|
||||
this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff;
|
||||
this.autoStartup = autoStartup;
|
||||
}
|
||||
|
||||
public List<KafkaEnvelopeConsumerWorker> workers() {
|
||||
return workers;
|
||||
return workersOrCreate();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -45,17 +61,35 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
|
||||
if (!running.compareAndSet(false, true)) {
|
||||
return;
|
||||
}
|
||||
List<KafkaEnvelopeConsumerWorker> activeWorkers = workersOrCreate();
|
||||
// 每个 worker 一条后台线程,避免某个处理器阻塞时拖慢其他消费组。
|
||||
executor = Executors.newFixedThreadPool(workers.size(), r -> {
|
||||
executor = Executors.newFixedThreadPool(activeWorkers.size(), r -> {
|
||||
Thread thread = new Thread(r, "kafka-envelope-consumer");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
for (KafkaEnvelopeConsumerWorker worker : workers) {
|
||||
for (KafkaEnvelopeConsumerWorker worker : activeWorkers) {
|
||||
executor.submit(() -> pollLoop(worker));
|
||||
}
|
||||
}
|
||||
|
||||
private List<KafkaEnvelopeConsumerWorker> workersOrCreate() {
|
||||
List<KafkaEnvelopeConsumerWorker> current = workers;
|
||||
if (current != null) {
|
||||
return current;
|
||||
}
|
||||
synchronized (this) {
|
||||
if (workers == null) {
|
||||
List<KafkaEnvelopeConsumerWorker> created = workersSupplier.get();
|
||||
if (created == null || created.isEmpty()) {
|
||||
throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
|
||||
}
|
||||
workers = List.copyOf(created);
|
||||
}
|
||||
return workers;
|
||||
}
|
||||
}
|
||||
|
||||
private void pollLoop(KafkaEnvelopeConsumerWorker worker) {
|
||||
while (running.get()) {
|
||||
try {
|
||||
@@ -121,7 +155,11 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
|
||||
if (!closed.compareAndSet(false, true)) {
|
||||
return;
|
||||
}
|
||||
for (KafkaEnvelopeConsumerWorker worker : workers) {
|
||||
List<KafkaEnvelopeConsumerWorker> current = workers;
|
||||
if (current == null) {
|
||||
return;
|
||||
}
|
||||
for (KafkaEnvelopeConsumerWorker worker : current) {
|
||||
worker.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,26 +36,26 @@ public final class KafkaEnvelopeConsumerWorker implements AutoCloseable {
|
||||
|
||||
public int pollOnce(Duration timeout) {
|
||||
ConsumerRecords<String, byte[]> records = consumer.poll(timeout == null ? Duration.ZERO : timeout);
|
||||
Map<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> batches = new LinkedHashMap<>();
|
||||
Map<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> byProcessor = new LinkedHashMap<>();
|
||||
int processed = 0;
|
||||
for (ConsumerRecord<String, byte[]> record : records) {
|
||||
EnvelopeConsumerProcessor processor = processorsByTopic.get(record.topic());
|
||||
if (processor == null) {
|
||||
// worker 可能订阅多个 topic;没有显式绑定处理器的 topic 不参与提交语义。
|
||||
continue;
|
||||
}
|
||||
batches.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord(
|
||||
byProcessor.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord(
|
||||
record.topic(),
|
||||
record.partition(),
|
||||
record.offset(),
|
||||
record.key(),
|
||||
record.value()));
|
||||
processed++;
|
||||
}
|
||||
int processed = 0;
|
||||
for (Map.Entry<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> entry : batches.entrySet()) {
|
||||
for (Map.Entry<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> entry : byProcessor.entrySet()) {
|
||||
// EnvelopeConsumerProcessor 内部会把解析或业务错误转成 DLQ 记录,
|
||||
// 这里保持 Kafka worker 的职责单一:轮询、批量分发、成功后提交 offset。
|
||||
entry.getKey().processAll(entry.getValue());
|
||||
processed += entry.getValue().size();
|
||||
// 这里保持 Kafka worker 的职责单一:轮询、分发、成功后提交 offset。
|
||||
entry.getKey().processBatch(entry.getValue());
|
||||
}
|
||||
if (processed > 0) {
|
||||
// commitSync 放在批次末尾,保证同一个 poll 批次内的消息按 Kafka offset 一起确认。
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.lingniu.ingest.sink.mq;
|
||||
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
|
||||
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
@@ -8,15 +7,12 @@ import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.common.serialization.ByteArraySerializer;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
@@ -100,21 +96,4 @@ public class SinkMqAutoConfiguration {
|
||||
return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
@ConditionalOnBean(EnvelopeConsumerProcessor.class)
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
|
||||
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(Map<String, EnvelopeConsumerProcessor> processors,
|
||||
SinkMqProperties props) {
|
||||
// Consumer runner 根据 processor bean 名和配置 binding 生成 worker;没有 binding 时直接失败,避免静默不消费。
|
||||
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
|
||||
if (workers.isEmpty()) {
|
||||
throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
|
||||
}
|
||||
return new KafkaEnvelopeConsumerRunner(
|
||||
workers,
|
||||
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
|
||||
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
|
||||
props.getConsumer().isAutoStartup());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package com.lingniu.ingest.sink.mq;
|
||||
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||
import org.springframework.beans.factory.ListableBeanFactory;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@AutoConfiguration(after = SinkMqAutoConfiguration.class)
|
||||
@AutoConfigureAfter(name = {
|
||||
"com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration",
|
||||
"com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration",
|
||||
"com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration"
|
||||
})
|
||||
@EnableConfigurationProperties(SinkMqProperties.class)
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class SinkMqConsumerAutoConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
|
||||
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory,
|
||||
SinkMqProperties props) {
|
||||
return new KafkaEnvelopeConsumerRunner(
|
||||
() -> createWorkers(beanFactory, props),
|
||||
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
|
||||
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
|
||||
props.getConsumer().isAutoStartup());
|
||||
}
|
||||
|
||||
private List<KafkaEnvelopeConsumerWorker> createWorkers(ListableBeanFactory beanFactory, SinkMqProperties props) {
|
||||
Map<String, EnvelopeConsumerProcessor> processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class);
|
||||
return new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,7 @@ message RawArchiveRef {
|
||||
string uri = 1;
|
||||
string checksum = 2;
|
||||
int64 size_bytes = 3;
|
||||
string parsed_json = 4;
|
||||
}
|
||||
|
||||
message TelemetrySnapshot {
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration
|
||||
com.lingniu.ingest.sink.mq.SinkMqConsumerAutoConfiguration
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.lingniu.ingest.sink.mq;
|
||||
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
|
||||
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
|
||||
@@ -14,6 +15,7 @@ import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -44,4 +46,39 @@ class KafkaEnvelopeConsumerWorkerTest {
|
||||
assertThat(record.payload()).containsExactly(0x01, 0x02);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void pollsKafkaRecordsAndDispatchesBatchWhenProcessorSupportsIt() {
|
||||
MockConsumer<String, byte[]> consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST);
|
||||
TopicPartition partition = new TopicPartition("vehicle.raw", 0);
|
||||
consumer.assign(List.of(partition));
|
||||
consumer.updateBeginningOffsets(Map.of(partition, 0L));
|
||||
consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 12L, "VIN001", new byte[]{0x01}));
|
||||
consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 13L, "VIN002", new byte[]{0x02}));
|
||||
AtomicInteger batchCalls = new AtomicInteger();
|
||||
EnvelopeConsumerProcessor processor = new EnvelopeConsumerProcessor(
|
||||
"event-history",
|
||||
new EnvelopeBatchIngestor() {
|
||||
@Override
|
||||
public EnvelopeIngestResult tryIngest(byte[] kafkaValue) {
|
||||
throw new AssertionError("single-record ingest should not be used for a batch-capable processor");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues) {
|
||||
batchCalls.incrementAndGet();
|
||||
assertThat(kafkaValues).hasSize(2);
|
||||
return List.of(
|
||||
EnvelopeIngestResult.stored("event-1", "VIN001"),
|
||||
EnvelopeIngestResult.invalid("bad envelope"));
|
||||
}
|
||||
},
|
||||
record -> {});
|
||||
|
||||
int processed = new KafkaEnvelopeConsumerWorker(
|
||||
consumer, Map.of("vehicle.raw", processor)).pollOnce(Duration.ZERO);
|
||||
|
||||
assertThat(processed).isEqualTo(2);
|
||||
assertThat(batchCalls).hasValue(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
class SinkMqConsumerAutoConfigurationTest {
|
||||
|
||||
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
|
||||
.withUserConfiguration(SinkMqAutoConfiguration.class)
|
||||
.withUserConfiguration(SinkMqAutoConfiguration.class, SinkMqConsumerAutoConfiguration.class)
|
||||
.withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
|
||||
() -> new EnvelopeConsumerProcessor(
|
||||
"vehicle-state",
|
||||
|
||||
@@ -43,6 +43,7 @@ public final class TdengineEnvelopeRows {
|
||||
raw.getParseError(),
|
||||
raw.getPeer(),
|
||||
json(metadata),
|
||||
parsedJson(envelope, metadata),
|
||||
protocol(envelope),
|
||||
vehicleKey(envelope, raw.getVehicleKey(), raw.getPhone()),
|
||||
firstNonBlank(raw.getVin(), envelope.getVin()),
|
||||
@@ -73,7 +74,6 @@ public final class TdengineEnvelopeRows {
|
||||
location.getStatusFlag(),
|
||||
totalMileage(envelope),
|
||||
rawUri,
|
||||
json(envelope.getMetadataMap()),
|
||||
protocol(envelope),
|
||||
vehicleKey(envelope, envelope.getMetadataOrDefault("vehicle_key", ""),
|
||||
envelope.getMetadataOrDefault("phone", "")),
|
||||
@@ -188,6 +188,13 @@ public final class TdengineEnvelopeRows {
|
||||
return Double.parseDouble(value);
|
||||
}
|
||||
|
||||
private static String parsedJson(VehicleEnvelope envelope, Map<String, String> metadata) {
|
||||
if (envelope.hasRawArchive() && !envelope.getRawArchive().getParsedJson().isBlank()) {
|
||||
return envelope.getRawArchive().getParsedJson();
|
||||
}
|
||||
return firstNonBlank(metadata.get("parsedJson"), metadata.get("parsed_json"));
|
||||
}
|
||||
|
||||
private static Double valueDouble(TelemetryField field) {
|
||||
String valueType = field.getValueType();
|
||||
if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) {
|
||||
|
||||
@@ -6,11 +6,11 @@ import java.util.List;
|
||||
public final class TdengineHistoryQueries {
|
||||
|
||||
private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
|
||||
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, "
|
||||
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json, "
|
||||
+ "protocol, vehicle_key, vin, phone";
|
||||
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
|
||||
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, "
|
||||
+ "metadata_json, protocol, vehicle_key, vin, phone";
|
||||
+ "protocol, vehicle_key, vin, phone";
|
||||
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, "
|
||||
+ "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, "
|
||||
+ "metadata_json, protocol, vehicle_key, vin, phone";
|
||||
|
||||
@@ -64,7 +64,8 @@ public final class TdengineHistorySchema {
|
||||
parse_status NCHAR(16),
|
||||
parse_error NCHAR(512),
|
||||
peer NCHAR(128),
|
||||
metadata_json NCHAR(4096)
|
||||
metadata_json NCHAR(4096),
|
||||
parsed_json NCHAR(16374)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
@@ -88,8 +89,7 @@ public final class TdengineHistorySchema {
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE,
|
||||
raw_uri NCHAR(512),
|
||||
metadata_json NCHAR(4096)
|
||||
raw_uri NCHAR(512)
|
||||
) TAGS (
|
||||
protocol NCHAR(16),
|
||||
vehicle_key NCHAR(128),
|
||||
|
||||
@@ -6,12 +6,12 @@ import java.util.Arrays;
|
||||
public final class TdengineHistoryStatements {
|
||||
|
||||
private static final String RAW_FRAME_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
|
||||
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json";
|
||||
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
|
||||
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json";
|
||||
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
|
||||
|
||||
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
|
||||
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, metadata_json";
|
||||
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
|
||||
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri";
|
||||
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
|
||||
|
||||
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, "
|
||||
+ "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json";
|
||||
@@ -35,7 +35,7 @@ public final class TdengineHistoryStatements {
|
||||
values(
|
||||
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
|
||||
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
|
||||
row.parseError(), row.peer(), row.metadataJson()
|
||||
row.parseError(), row.peer(), row.metadataJson(), row.parsedJson()
|
||||
)
|
||||
);
|
||||
}
|
||||
@@ -49,7 +49,7 @@ public final class TdengineHistoryStatements {
|
||||
values(
|
||||
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
|
||||
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
|
||||
row.totalMileageKm(), row.rawUri(), row.metadataJson()
|
||||
row.totalMileageKm(), row.rawUri()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -157,6 +157,7 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
|
||||
rs.getString("parse_error"),
|
||||
rs.getString("peer"),
|
||||
rs.getString("metadata_json"),
|
||||
rs.getString("parsed_json"),
|
||||
rs.getString("protocol"),
|
||||
rs.getString("vehicle_key"),
|
||||
rs.getString("vin"),
|
||||
@@ -179,7 +180,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
|
||||
rs.getLong("status_flag"),
|
||||
nullableDouble(rs, "total_mileage_km"),
|
||||
rs.getString("raw_uri"),
|
||||
rs.getString("metadata_json"),
|
||||
rs.getString("protocol"),
|
||||
rs.getString("vehicle_key"),
|
||||
rs.getString("vin"),
|
||||
|
||||
@@ -16,7 +16,6 @@ public record TdengineLocationRow(
|
||||
long statusFlag,
|
||||
Double totalMileageKm,
|
||||
String rawUri,
|
||||
String metadataJson,
|
||||
String protocol,
|
||||
String vehicleKey,
|
||||
String vin,
|
||||
|
||||
@@ -16,6 +16,7 @@ public record TdengineRawFrameRow(
|
||||
String parseError,
|
||||
String peer,
|
||||
String metadataJson,
|
||||
String parsedJson,
|
||||
String protocol,
|
||||
String vehicleKey,
|
||||
String vin,
|
||||
|
||||
@@ -27,6 +27,7 @@ class TdengineHistoryStatementsTest {
|
||||
"",
|
||||
"10.0.0.1:808",
|
||||
"{\"auth\":\"passed\"}",
|
||||
"{\"body\":{\"speedKmh\":42.0}}",
|
||||
"JT808",
|
||||
"jt808:g7gps",
|
||||
"VIN123",
|
||||
@@ -39,13 +40,13 @@ class TdengineHistoryStatementsTest {
|
||||
.contains(" USING raw_frames TAGS ('JT808', 'jt808:g7gps', 'VIN123', '013800000000')");
|
||||
assertThat(batch.insertSql())
|
||||
.startsWith("INSERT INTO raw_jt808_")
|
||||
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json)")
|
||||
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json)")
|
||||
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
assertThat(batch.values())
|
||||
.containsExactly(
|
||||
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
|
||||
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
|
||||
row.parseError(), row.peer(), row.metadataJson());
|
||||
row.parseError(), row.peer(), row.metadataJson(), row.parsedJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -64,7 +65,6 @@ class TdengineHistoryStatementsTest {
|
||||
3,
|
||||
null,
|
||||
"archive://gb32960/frame-1.bin",
|
||||
"{}",
|
||||
"GB32960",
|
||||
"vin:O'HARE",
|
||||
"O'HARE",
|
||||
@@ -79,7 +79,7 @@ class TdengineHistoryStatementsTest {
|
||||
.containsExactly(
|
||||
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
|
||||
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
|
||||
row.totalMileageKm(), row.rawUri(), row.metadataJson());
|
||||
row.totalMileageKm(), row.rawUri());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -74,7 +74,6 @@ class TdengineJdbcHistoryWriterTest {
|
||||
3,
|
||||
null,
|
||||
"archive://gb32960/frame-1.bin",
|
||||
"{}",
|
||||
"GB32960",
|
||||
"vin:VIN123",
|
||||
"VIN123",
|
||||
@@ -217,6 +216,7 @@ class TdengineJdbcHistoryWriterTest {
|
||||
parseError,
|
||||
"10.0.0.1:808",
|
||||
"{}",
|
||||
"{\"frame\":\"" + frameId + "\"}",
|
||||
"JT808",
|
||||
"jt808:g7gps",
|
||||
"VIN123",
|
||||
|
||||
6
pom.xml
6
pom.xml
@@ -65,6 +65,7 @@
|
||||
<resilience4j.version>2.2.0</resilience4j.version>
|
||||
<micrometer.version>1.13.6</micrometer.version>
|
||||
<otel.version>1.42.1</otel.version>
|
||||
<paho-mqtt.version>1.2.5</paho-mqtt.version>
|
||||
<fusesource-mqtt.version>1.12</fusesource-mqtt.version>
|
||||
<duckdb.version>1.1.3</duckdb.version>
|
||||
<taos-jdbcdriver.version>3.8.4</taos-jdbcdriver.version>
|
||||
@@ -306,6 +307,11 @@
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<version>${kafka.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.paho</groupId>
|
||||
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
|
||||
<version>${paho-mqtt.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.fusesource.mqtt-client</groupId>
|
||||
<artifactId>mqtt-client</artifactId>
|
||||
|
||||
Reference in New Issue
Block a user