feat: productionize raw history ingestion

This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions

View File

@@ -10,6 +10,6 @@ ENV JAVA_OPTS=""
WORKDIR /app
COPY modules/apps/${APP_NAME}/target/${APP_NAME}.jar /app/app.jar
EXPOSE 20100 20200 20300 20400 32960 10808
EXPOSE 20100 20200 20300 20400 20500 20600 32960 10808
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app/app.jar"]

View File

@@ -4,7 +4,7 @@ version: "3.8"
# main-0.1.0-SNAPSHOT-22468e7
x-common-env: &common-env
TZ: Asia/Shanghai
JAVA_OPTS: ${JAVA_OPTS:--Xms128m -Xmx768m -Duser.timezone=Asia/Shanghai}
JAVA_OPTS: ${JAVA_OPTS:--XX:+UseZGC -XX:MaxRAMPercentage=75}
NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true}
NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848}
NACOS_NAMESPACE: ${NACOS_NAMESPACE:-}
@@ -13,32 +13,33 @@ x-common-env: &common-env
NACOS_REFRESH_ENABLED: ${NACOS_REFRESH_ENABLED:-true}
NACOS_USERNAME: ${NACOS_USERNAME:-}
NACOS_PASSWORD: ${NACOS_PASSWORD:-}
KAFKA_BROKERS: ${KAFKA_BROKERS:-114.55.58.251:9092}
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
KAFKA_ENABLED: ${KAFKA_ENABLED:-true}
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true}
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:-900000}
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
KAFKA_TOPIC_XINDA_PUSH_EVENT: ${KAFKA_TOPIC_XINDA_PUSH_EVENT:-vehicle.event.xinda-push.v1}
KAFKA_TOPIC_XINDA_PUSH_RAW: ${KAFKA_TOPIC_XINDA_PUSH_RAW:-vehicle.raw.xinda-push.v1}
KAFKA_TOPIC_XINDA_PUSH_DLQ: ${KAFKA_TOPIC_XINDA_PUSH_DLQ:-vehicle.dlq.xinda-push.v1}
KAFKA_TOPIC_MQTT_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_MQTT_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_MQTT_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
KAFKA_TOPIC_XINDA_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1}
KAFKA_TOPIC_XINDA_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1}
KAFKA_TOPIC_XINDA_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1}
MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
services:
gb32960-ingest-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: gb32960-ingest-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_GB32960:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20100
GB32960_PORT: 32960
KAFKA_CONSUMER_ENABLED: "false"
@@ -48,13 +49,11 @@ services:
GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}
GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}
GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
ports:
- "${GB32960_HTTP_PORT:-20100}:20100"
- "${GB32960_TCP_PORT:-32960}:32960"
volumes:
- vehicle-history-archive:/archive
- gb32960-ingest-data:/data
networks:
- vehicle-ingest
@@ -63,109 +62,82 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: jt808-ingest-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_JT808:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20482
HTTP_PORT: 20400
JT808_PORT: 808
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
ports:
- "${JT808_HTTP_PORT:-20482}:20482"
- "${JT808_HTTP_PORT:-20400}:20400"
- "${JT808_TCP_PORT:-808}:808"
volumes:
- vehicle-history-archive:/archive
- jt808-ingest-data:/data
networks:
- vehicle-ingest
yutong-mqtt-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: yutong-mqtt-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_YUTONG_MQTT:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20500
MQTT_ENABLED: "true"
XINDA_PUSH_ENABLED: "false"
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_YUTONG_MQTT:-yutong-mqtt-portainer}
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
YUTONG_MQTT_ENDPOINT_NAME: ${YUTONG_MQTT_ENDPOINT_NAME:-yutong}
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
YUTONG_MQTT_TOPIC: ${YUTONG_MQTT_TOPIC:-#}
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-1}
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-yutong-mqtt}
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
KAFKA_NODE_ID: ${KAFKA_NODE_ID_MQTT:-yutong-mqtt-portainer}
KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
MQTT_ENDPOINT_NAME: ${MQTT_ENDPOINT_NAME:-yutong}
MQTT_URI: ${MQTT_URI:-tcp://127.0.0.1:1883}
MQTT_TOPIC: "${MQTT_TOPIC:-#}"
MQTT_QOS: ${MQTT_QOS:-1}
MQTT_CLIENT_ID: ${MQTT_CLIENT_ID:-lingniu-yutong-mqtt}
MQTT_USERNAME: ${MQTT_USERNAME:-}
MQTT_PASSWORD: ${MQTT_PASSWORD:-}
MQTT_CLEAN_SESSION: ${MQTT_CLEAN_SESSION:-true}
MQTT_PROFILE: ${MQTT_PROFILE:-yutong}
ports:
- "${YUTONG_MQTT_HTTP_PORT:-20500}:20500"
volumes:
- vehicle-history-archive:/archive
- yutong-mqtt-data:/data
networks:
- vehicle-ingest
xinda-push-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/xinda-push-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: xinda-push-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_XINDA_PUSH:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20600
MQTT_ENABLED: "false"
XINDA_PUSH_ENABLED: "true"
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA_PUSH:-xinda-push-portainer}
XINDA_PUSH_ENABLED: ${XINDA_PUSH_ENABLED:-false}
KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA:-xinda-push-portainer}
KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1}
KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1}
KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
XINDA_PUSH_HOST: ${XINDA_PUSH_HOST:-}
XINDA_PUSH_PORT: ${XINDA_PUSH_PORT:-10100}
XINDA_PUSH_USERNAME: ${XINDA_PUSH_USERNAME:-}
XINDA_PUSH_PASSWORD: ${XINDA_PUSH_PASSWORD:-}
XINDA_PUSH_CLIENT_DESC: ${XINDA_PUSH_CLIENT_DESC:-lingniu-ingest}
XINDA_PUSH_MSG_ID_LOCATION: ${XINDA_PUSH_MSG_ID_LOCATION:-0200}
XINDA_PUSH_MSG_ID_ALARM: ${XINDA_PUSH_MSG_ID_ALARM:-0300}
XINDA_PUSH_MSG_ID_PASSTHROUGH: ${XINDA_PUSH_MSG_ID_PASSTHROUGH:-0401}
XINDA_PUSH_RECONNECT_INTERVAL_SEC: ${XINDA_PUSH_RECONNECT_INTERVAL_SEC:-5}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
XINDA_PUSH_SUBSCRIBE_MSG_IDS: "${XINDA_PUSH_SUBSCRIBE_MSG_IDS:-0200,0300,0401}"
ports:
- "${XINDA_PUSH_HTTP_PORT:-20600}:20600"
volumes:
- vehicle-history-archive:/archive
- xinda-push-data:/data
networks:
- vehicle-ingest
@@ -173,8 +145,6 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: vehicle-history-app
restart: unless-stopped
mem_limit: 2560m
memswap_limit: 2560m
depends_on:
- gb32960-ingest-app
- jt808-ingest-app
@@ -182,29 +152,28 @@ services:
- xinda-push-app
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_HISTORY:--Xms512m -Xmx1536m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20200
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history}
KAFKA_CONSUMER_MAX_POLL_RECORDS: ${KAFKA_CONSUMER_MAX_POLL_RECORDS_HISTORY:-2000}
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
KAFKA_CONSUMER_CONCURRENCY: ${KAFKA_CONSUMER_CONCURRENCY:-3}
KAFKA_GROUP_HISTORY_GB32960: ${KAFKA_GROUP_HISTORY_GB32960:-vehicle-history}
KAFKA_GROUP_HISTORY_JT808: ${KAFKA_GROUP_HISTORY_JT808:-vehicle-history}
KAFKA_GROUP_HISTORY_MQTT_YUTONG: ${KAFKA_GROUP_HISTORY_MQTT_YUTONG:-vehicle-history}
KAFKA_GROUP_HISTORY_XINDA: ${KAFKA_GROUP_HISTORY_XINDA:-vehicle-history}
SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true}
SINK_ARCHIVE_PATH: /archive/
# 高吞吐生产热路径默认只写 TDengine。需要 GB32960 snapshot/fields 旧查询时再显式开启。
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-false}
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true}
EVENT_FILE_STORE_PATH: /event-store/
EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai}
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://tdengine:6041/vehicle_ts}
TDENGINE_DRIVER_CLASS_NAME: ${TDENGINE_DRIVER_CLASS_NAME:-com.taosdata.jdbc.ws.WebSocketDriver}
EVENT_FILE_STORE_BATCH_SIZE: ${EVENT_FILE_STORE_BATCH_SIZE:-500}
EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS: ${EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS:-1000}
EVENT_FILE_STORE_WRITE_PARQUET_ENABLED: ${EVENT_FILE_STORE_WRITE_PARQUET_ENABLED:-false}
TDENGINE_ENABLED: ${TDENGINE_ENABLED:-true}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts}
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-vehicle_ts}
TDENGINE_JT808_LOCATION_STABLE: ${TDENGINE_JT808_LOCATION_STABLE:-jt808_locations}
TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root}
TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata}
TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32}
TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-0}
TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-5000}
TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:--1}
# 默认关闭逐字段宽表,避免 telemetry_fields 写放大;需要字段趋势查询时显式打开。
TDENGINE_TELEMETRY_FIELDS_ENABLED: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:-false}
ports:
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
volumes:
@@ -217,16 +186,11 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: vehicle-analytics-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
depends_on:
- gb32960-ingest-app
- jt808-ingest-app
- yutong-mqtt-app
- xinda-push-app
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_ANALYTICS:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20300
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics}
KAFKA_GROUP_STATE: ${KAFKA_GROUP_STATE:-vehicle-state}
@@ -244,6 +208,7 @@ services:
REDIS_HOST: ${REDIS_HOST:-}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DATABASE: ${REDIS_DATABASE:-50}
REDIS_USERNAME: ${REDIS_USERNAME:-}
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
ports:
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300"
@@ -259,8 +224,6 @@ networks:
volumes:
gb32960-ingest-data:
jt808-ingest-data:
yutong-mqtt-data:
xinda-push-data:
vehicle-history-archive:
vehicle-history-event-store:
vehicle-analytics-stat:

View File

@@ -0,0 +1,92 @@
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.admin.NewPartitions;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.admin.TopicDescription;
public final class EnsureKafkaTopics {
private static final short REPLICATION_FACTOR = 1;
private record TopicSpec(String name, int partitions) {}
private EnsureKafkaTopics() {}
public static void main(String[] args) throws Exception {
String brokers = args.length > 0 ? args[0] : "172.17.111.56:9092";
List<TopicSpec> specs =
List.of(
topic("vehicle.event.gb32960.v1", 12),
topic("vehicle.raw.gb32960.v1", 12),
topic("vehicle.dlq.gb32960.v1", 3),
topic("vehicle.event.jt808.v1", 12),
topic("vehicle.raw.jt808.v1", 12),
topic("vehicle.dlq.jt808.v1", 3),
topic("vehicle.event.mqtt-yutong.v1", 12),
topic("vehicle.raw.mqtt-yutong.v1", 12),
topic("vehicle.dlq.mqtt-yutong.v1", 3),
topic("vehicle.event.xinda.v1", 12),
topic("vehicle.raw.xinda.v1", 12),
topic("vehicle.dlq.xinda.v1", 3),
topic("vehicle.event.xinda-push.v1", 12),
topic("vehicle.raw.xinda-push.v1", 12),
topic("vehicle.dlq.xinda-push.v1", 3));
Properties props = new Properties();
props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);
props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "15000");
props.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "30000");
try (AdminClient admin = AdminClient.create(props)) {
Set<String> existing = admin.listTopics().names().get();
List<NewTopic> missing =
specs.stream()
.filter(spec -> !existing.contains(spec.name()))
.map(spec -> new NewTopic(spec.name(), spec.partitions(), REPLICATION_FACTOR))
.toList();
if (!missing.isEmpty()) {
try {
admin.createTopics(missing).all().get();
} catch (ExecutionException e) {
System.out.println("createTopics result=" + e.getCause());
}
}
Map<String, TopicSpec> byName =
specs.stream().collect(Collectors.toMap(TopicSpec::name, spec -> spec));
Map<String, TopicDescription> descriptions =
admin.describeTopics(byName.keySet()).allTopicNames().get();
Map<String, NewPartitions> increases =
descriptions.entrySet().stream()
.filter(entry -> entry.getValue().partitions().size() < byName.get(entry.getKey()).partitions())
.collect(
Collectors.toMap(
Map.Entry::getKey,
entry -> NewPartitions.increaseTo(byName.get(entry.getKey()).partitions())));
if (!increases.isEmpty()) {
admin.createPartitions(increases).all().get();
Thread.sleep(Duration.ofSeconds(2).toMillis());
descriptions = admin.describeTopics(byName.keySet()).allTopicNames().get();
}
List<String> lines = new ArrayList<>();
descriptions.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.forEach(entry -> lines.add(entry.getKey() + " partitions=" + entry.getValue().partitions().size()));
System.out.println("bootstrap=" + brokers);
lines.forEach(System.out::println);
}
}
private static TopicSpec topic(String name, int partitions) {
return new TopicSpec(name, partitions);
}
}

View File

@@ -52,12 +52,11 @@ lingniu:
file:
path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl}
mysql:
table: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:}
jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}
username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root}
password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:}
driver-class-name: ${VEHICLE_IDENTITY_MYSQL_DRIVER_CLASS_NAME:com.mysql.cj.jdbc.Driver}
refresh-interval: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}
table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}
initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}
sink:
mq:
enabled: ${KAFKA_ENABLED:true}
@@ -79,10 +78,6 @@ lingniu:
dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}
consumer:
enabled: false
archive:
enabled: ${SINK_ARCHIVE_ENABLED:true}
type: local
path: ${SINK_ARCHIVE_PATH:./archive/}
event-file-store:
enabled: false
event-history:

View File

@@ -29,12 +29,14 @@ class Jt808IngestAppDefaultsTest {
.containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}")
.containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}")
.containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}")
.containsEntry("lingniu.ingest.identity.mysql.table", "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}")
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url", "${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}")
.containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:}")
.containsEntry("lingniu.ingest.identity.mysql.refresh-interval", "${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}")
.containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}")
.containsEntry("lingniu.ingest.identity.mysql.table-name",
"${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}")
.containsEntry("lingniu.ingest.identity.mysql.jdbc-url",
"${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}")
.containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:root}")
.containsEntry("lingniu.ingest.identity.mysql.password", "${VEHICLE_IDENTITY_MYSQL_PASSWORD:}")
.containsEntry("lingniu.ingest.identity.mysql.initialize-schema",
"${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}")
.containsEntry("lingniu.ingest.event-file-store.enabled", false)
.containsEntry("lingniu.ingest.event-history.enabled", false)
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)

View File

@@ -33,6 +33,10 @@
<groupId>com.lingniu.ingest</groupId>
<artifactId>vehicle-stat-service</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>

View File

@@ -26,6 +26,7 @@ spring:
redis:
host: ${REDIS_HOST:}
port: ${REDIS_PORT:6379}
username: ${REDIS_USERNAME:}
password: ${REDIS_PASSWORD:}
database: ${REDIS_DATABASE:50}
@@ -55,6 +56,7 @@ lingniu:
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:500}
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:900000}
bindings:
vehicleStateEnvelopeConsumerProcessor:
enabled: ${VEHICLE_STATE_ENABLED:false}

View File

@@ -46,7 +46,7 @@ lingniu:
enabled: ${KAFKA_CONSUMER_ENABLED:true}
client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history}
auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}
max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}
max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000}
concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3}
bindings:

View File

@@ -43,7 +43,7 @@ class VehicleHistoryAppDefaultsTest {
.containsEntry("lingniu.ingest.vehicle-state.enabled", false)
.containsEntry("lingniu.ingest.vehicle-stat.enabled", false)
.containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}")
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}")
.containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}")
.containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}")
.containsEntry(
"lingniu.ingest.sink.mq.consumer.max-poll-interval-millis",

View File

@@ -0,0 +1,12 @@
package com.lingniu.ingest.api.consumer;
import java.util.List;
/**
* Optional extension for consumers that can persist a Kafka poll batch in one
* storage operation.
*/
public interface EnvelopeBatchIngestor extends EnvelopeIngestor {
List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues);
}

View File

@@ -70,6 +70,32 @@ public final class EnvelopeConsumerProcessor {
}
}
public List<EnvelopeIngestResult> processBatch(List<EnvelopeConsumerRecord> records) {
if (records == null || records.isEmpty()) {
return List.of();
}
if (!(ingestor instanceof EnvelopeBatchIngestor batchIngestor)) {
return records.stream().map(this::process).toList();
}
List<byte[]> payloads = records.stream()
.map(EnvelopeConsumerRecord::payload)
.toList();
List<EnvelopeIngestResult> results = batchIngestor.tryIngestAll(payloads);
if (results.size() != records.size()) {
throw new IllegalStateException("batch ingestor returned " + results.size()
+ " results for " + records.size() + " records");
}
List<EnvelopeIngestResult> copy = new ArrayList<>(results.size());
for (int i = 0; i < records.size(); i++) {
EnvelopeIngestResult result = results.get(i);
if (DEAD_LETTER_STATUSES.contains(result.status())) {
deadLetterSink.publish(toDeadLetter(records.get(i), result));
}
copy.add(result);
}
return List.copyOf(copy);
}
private EnvelopeDeadLetterRecord toDeadLetter(EnvelopeConsumerRecord record, EnvelopeIngestResult result) {
return new EnvelopeDeadLetterRecord(
service,

View File

@@ -19,6 +19,7 @@ public final class RawArchiveKeys {
public static final String META_EVENT_ID = "rawArchiveEventId";
public static final String META_KEY = "rawArchiveKey";
public static final String META_URI = "rawArchiveUri";
public static final String META_PARSED_JSON = "rawArchiveParsedJson";
private static final DateTimeFormatter DATE_KEY =
DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneId.of("Asia/Shanghai"));

View File

@@ -150,6 +150,7 @@ public sealed interface VehicleEvent
* @param command 协议主命令码(如 32960 0x02/0x03 等),冗余在事件里便于按命令分片归档
* @param infoType 协议子类型(可为 0同上
* @param rawBytes 原始入站字节,不可为 null
* @param parsedJson 已解析结构化 JSON。只用于 RAW 明细查询,不应复制到位置/字段等派生表。
*/
record RawArchive(
String eventId,
@@ -161,6 +162,25 @@ public sealed interface VehicleEvent
Map<String, String> metadata,
int command,
int infoType,
byte[] rawBytes
) implements VehicleEvent {}
byte[] rawBytes,
String parsedJson
) implements VehicleEvent {
public RawArchive(String eventId,
String vin,
ProtocolId source,
Instant eventTime,
Instant ingestTime,
String traceId,
Map<String, String> metadata,
int command,
int infoType,
byte[] rawBytes) {
this(eventId, vin, source, eventTime, ingestTime, traceId, metadata,
command, infoType, rawBytes, "");
}
public RawArchive {
parsedJson = parsedJson == null ? "" : parsedJson;
}
}
}

View File

@@ -24,6 +24,14 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>

View File

@@ -1,5 +1,8 @@
package com.lingniu.ingest.core.dispatcher;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.api.pipeline.IngestContext;
@@ -36,6 +39,9 @@ public final class Dispatcher {
private static final Logger log = LoggerFactory.getLogger(Dispatcher.class);
private static final AtomicLong RAW_ARCHIVE_SEQUENCE = new AtomicLong();
private static final String REQUIRED_DURABLE_SINK = "kafka";
private static final ObjectMapper RAW_ARCHIVE_JSON = JsonMapper.builder()
.findAndAddModules()
.build();
private final HandlerRegistry registry;
private final InterceptorChain interceptors;
@@ -167,7 +173,10 @@ public final class Dispatcher {
byte[] bytes = frame.rawBytes();
if (bytes == null || bytes.length == 0) return RawArchiveLookup.empty();
Map<String, String> meta = frame.sourceMeta() == null ? Map.of() : frame.sourceMeta();
Map<String, String> meta = frame.sourceMeta() == null
? Map.of()
: new LinkedHashMap<>(frame.sourceMeta());
String parsedJson = parsedJson(frame.payload(), meta.remove(RawArchiveKeys.META_PARSED_JSON));
String vin = meta.getOrDefault("vin", "");
Instant ingestTime = frame.receivedAt() != null ? frame.receivedAt() : Instant.now();
String eventId = nextRawArchiveEventId(ingestTime);
@@ -184,11 +193,34 @@ public final class Dispatcher {
archiveMeta,
frame.command(),
frame.infoType(),
bytes);
bytes,
parsedJson);
out.add(raw);
return new RawArchiveLookup(eventId, key, RawArchiveKeys.logicalUri(key));
}
private static String parsedJson(Object payload, String explicitParsedJson) {
if (explicitParsedJson != null && !explicitParsedJson.isBlank()) {
return explicitParsedJson;
}
if (payload == null) {
return "{}";
}
try {
return RAW_ARCHIVE_JSON.writeValueAsString(payload);
} catch (JsonProcessingException ex) {
return "{\"serializationError\":\"" + escapeJson(ex.getMessage()) + "\",\"payloadType\":\""
+ escapeJson(payload.getClass().getName()) + "\"}";
}
}
private static String escapeJson(String value) {
if (value == null) {
return "";
}
return value.replace("\\", "\\\\").replace("\"", "\\\"");
}
private static String nextRawArchiveEventId(Instant ingestTime) {
// 文件名需要可排序且单进程内唯一:毫秒时间放大到微秒量级,再用 AtomicLong 递增兜底。
long base = (ingestTime == null ? Instant.now() : ingestTime).toEpochMilli() * 1000L;

View File

@@ -42,7 +42,7 @@ class DispatcherRawArchiveMetadataTest {
ProtocolId.JT808,
0x0200,
0,
new Object(),
new SamplePayload("payload-0200", 7),
new byte[]{0x7e, 0x01, 0x7e},
Map.of("vin", "VIN001", "peer", "127.0.0.1:10001"),
Instant.parse("2026-06-22T08:00:00Z")));
@@ -52,6 +52,8 @@ class DispatcherRawArchiveMetadataTest {
String rawArchiveKey = raw.metadata().get("rawArchiveKey");
assertThat(rawArchiveKey).isNotBlank();
assertThat(raw.parsedJson()).contains("\"name\":\"payload-0200\"");
assertThat(raw.metadata()).doesNotContainKey("rawArchiveParsedJson");
assertThat(raw.eventId()).containsOnlyDigits();
assertThat(rawArchiveKey).endsWith("/" + raw.eventId() + ".bin");
assertThat(location.metadata())
@@ -102,7 +104,7 @@ class DispatcherRawArchiveMetadataTest {
private static HandlerRegistry registryWithLocationHandler() throws NoSuchMethodException {
LocationHandler bean = new LocationHandler();
Method method = LocationHandler.class.getDeclaredMethod("onLocation", Object.class);
Method method = LocationHandler.class.getDeclaredMethod("onLocation", SamplePayload.class);
HandlerRegistry registry = new HandlerRegistry();
registry.register(new HandlerDefinition(
ProtocolId.JT808,
@@ -169,7 +171,7 @@ class DispatcherRawArchiveMetadataTest {
@ProtocolHandler(protocol = ProtocolId.JT808)
static final class LocationHandler {
@MessageMapping(command = 0x0200)
VehicleEvent.Location onLocation(Object ignored) {
VehicleEvent.Location onLocation(SamplePayload ignored) {
return new VehicleEvent.Location(
"location-1",
"VIN001",
@@ -182,6 +184,9 @@ class DispatcherRawArchiveMetadataTest {
}
}
record SamplePayload(String name, int value) {
}
@ProtocolHandler(protocol = ProtocolId.JT808)
public static final class AsyncLocationHandler {
@MessageMapping(command = 0x0201)

View File

@@ -24,6 +24,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>

View File

@@ -1,70 +1,64 @@
package com.lingniu.ingest.identity;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
import org.springframework.jdbc.core.ConnectionCallback;
import org.springframework.jdbc.core.JdbcTemplate;
import javax.sql.DataSource;
import java.time.Duration;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.Map;
public final class MySqlVehicleIdentityService implements VehicleIdentityResolver, VehicleIdentityRegistry, AutoCloseable {
public final class MySqlVehicleIdentityService implements VehicleIdentityResolver, VehicleIdentityRegistry {
private static final Logger log = LoggerFactory.getLogger(MySqlVehicleIdentityService.class);
private final JdbcTemplate jdbc;
private final ObjectMapper mapper;
private final String tableName;
private final DataSource dataSource;
private final String table;
private final String registrationTable;
private final ScheduledExecutorService refresher;
private volatile InMemoryVehicleIdentityService index;
public MySqlVehicleIdentityService(DataSource dataSource, String table) {
this(dataSource, table, Duration.ZERO);
}
public MySqlVehicleIdentityService(DataSource dataSource, String table, Duration refreshInterval) {
if (dataSource == null) {
throw new IllegalArgumentException("dataSource must not be null");
public MySqlVehicleIdentityService(DataSource dataSource,
VehicleIdentityProperties.Mysql properties,
ObjectMapper mapper) {
this.jdbc = new JdbcTemplate(dataSource);
this.mapper = mapper == null ? new ObjectMapper() : mapper;
VehicleIdentityProperties.Mysql mysql = properties == null ? new VehicleIdentityProperties.Mysql() : properties;
this.tableName = safeTableName(mysql.getTableName());
if (mysql.isInitializeSchema()) {
initializeSchema();
}
this.dataSource = dataSource;
this.table = sanitizeTable(table);
this.registrationTable = this.table + "_registration";
initializeSchema();
this.index = loadIndex();
this.refresher = startRefresher(refreshInterval);
}
@Override
public void bind(VehicleIdentityBinding binding) {
if (binding == null || !binding.hasResolvedVin()) {
return;
}
upsert(binding.plate(), binding.vin());
index.bind(binding);
}
@Override
public void register(VehicleRegistrationBinding registration) {
if (registration == null) {
return;
}
upsertRegistration(registration);
if (registration.hasResolvedVin()) {
bind(new VehicleIdentityBinding(
registration.protocol(),
registration.vin(),
registration.phone(),
registration.deviceId(),
registration.plate()));
}
String metadataJson = metadataJson(binding.metadata());
jdbc.update("""
INSERT INTO %s
(protocol, vin, phone, device_id, plate, province, city, maker, device_type, plate_color, metadata_json,
created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON DUPLICATE KEY UPDATE
vin = VALUES(vin),
device_id = VALUES(device_id),
province = VALUES(province),
city = VALUES(city),
maker = VALUES(maker),
device_type = VALUES(device_type),
plate_color = VALUES(plate_color),
metadata_json = VALUES(metadata_json),
updated_at = CURRENT_TIMESTAMP
""".formatted(tableName),
protocol(binding.protocol()), binding.vin(), binding.phone(), binding.deviceId(), binding.plate(),
binding.metadata().getOrDefault("province", ""),
binding.metadata().getOrDefault("city", ""),
binding.metadata().getOrDefault("maker", ""),
binding.metadata().getOrDefault("deviceType", ""),
binding.metadata().getOrDefault("plateColor", ""),
metadataJson);
}
@Override
@@ -72,223 +66,167 @@ public final class MySqlVehicleIdentityService implements VehicleIdentityResolve
if (lookup == null) {
return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN);
}
return index.resolve(lookup);
}
public void refresh() {
index = loadIndex();
}
@Override
public void close() {
if (refresher != null) {
refresher.shutdownNow();
if (!lookup.vin().isBlank()) {
return new VehicleIdentity(lookup.vin(), true, VehicleIdentitySource.EXPLICIT_VIN);
}
VehicleIdentity byPhone = resolveBy("phone", lookup.protocol(), lookup.phone(), VehicleIdentitySource.BOUND_PHONE);
if (byPhone != null) {
return byPhone;
}
VehicleIdentity byDevice = resolveBy("device_id", lookup.protocol(), lookup.deviceId(), VehicleIdentitySource.BOUND_DEVICE_ID);
if (byDevice != null) {
return byDevice;
}
VehicleIdentity byPlate = resolveBy("plate", lookup.protocol(), lookup.plate(), VehicleIdentitySource.BOUND_PLATE);
if (byPlate != null) {
return byPlate;
}
if (!lookup.deviceId().isBlank()) {
return new VehicleIdentity(lookup.deviceId(), false, VehicleIdentitySource.FALLBACK_DEVICE_ID);
}
if (!lookup.phone().isBlank()) {
return new VehicleIdentity(lookup.phone(), false, VehicleIdentitySource.FALLBACK_PHONE);
}
if (!lookup.plate().isBlank()) {
return new VehicleIdentity(lookup.plate(), false, VehicleIdentitySource.FALLBACK_PLATE);
}
return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN);
}
private VehicleIdentity resolveBy(String column, ProtocolId protocol, String externalId, VehicleIdentitySource source) {
if (externalId == null || externalId.isBlank()) {
return null;
}
String safeColumn = switch (column) {
case "phone", "device_id", "plate" -> column;
default -> throw new IllegalArgumentException("unsupported identity lookup column: " + column);
};
List<VehicleIdentity> matches = jdbc.query("""
SELECT vin
FROM %s
WHERE protocol = ? AND %s = ?
ORDER BY updated_at DESC
LIMIT 1
""".formatted(tableName, safeColumn),
(rs, rowNum) -> identity(rs, source),
protocol(protocol), externalId.trim());
return matches.isEmpty() ? null : matches.getFirst();
}
private VehicleIdentity identity(ResultSet rs, VehicleIdentitySource source) throws SQLException {
return new VehicleIdentity(rs.getString("vin"), true, source);
}
private void initializeSchema() {
String sql = """
jdbc.execute("""
CREATE TABLE IF NOT EXISTS %s (
plate VARCHAR(128) NOT NULL,
vin VARCHAR(64) NOT NULL,
PRIMARY KEY (plate),
KEY idx_vehicle_identity_vin (vin)
id BIGINT AUTO_INCREMENT PRIMARY KEY,
protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL,
phone VARCHAR(64) NOT NULL DEFAULT '',
device_id VARCHAR(64) NOT NULL DEFAULT '',
plate VARCHAR(64) NOT NULL DEFAULT '',
province VARCHAR(32) NOT NULL DEFAULT '',
city VARCHAR(32) NOT NULL DEFAULT '',
maker VARCHAR(64) NOT NULL DEFAULT '',
device_type VARCHAR(128) NOT NULL DEFAULT '',
plate_color VARCHAR(32) NOT NULL DEFAULT '',
metadata_json TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uk_vehicle_identity_protocol_phone (protocol, phone),
KEY idx_vehicle_identity_protocol_device (protocol, device_id),
KEY idx_vehicle_identity_protocol_plate (protocol, plate)
)
""".formatted(table);
String registrationSql = """
CREATE TABLE IF NOT EXISTS %s (
protocol VARCHAR(32) NOT NULL,
phone VARCHAR(64) NOT NULL,
vin VARCHAR(64) NOT NULL DEFAULT 'unknown',
device_id VARCHAR(128) NOT NULL DEFAULT '',
plate VARCHAR(128) NOT NULL DEFAULT '',
province INT NULL,
city INT NULL,
maker VARCHAR(64) NOT NULL DEFAULT '',
device_type VARCHAR(128) NOT NULL DEFAULT '',
plate_color INT NULL,
last_registered_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (protocol, phone),
KEY idx_vehicle_registration_vin (vin),
KEY idx_vehicle_registration_device (protocol, device_id),
KEY idx_vehicle_registration_plate (protocol, plate)
)
""".formatted(registrationTable);
try (Connection connection = dataSource.getConnection();
Statement statement = connection.createStatement()) {
statement.execute(sql);
statement.execute(registrationSql);
} catch (SQLException e) {
throw new IllegalStateException("vehicle identity mysql schema initialize failed: " + table, e);
""".formatted(tableName));
ensureColumn("phone", "phone VARCHAR(64) NOT NULL DEFAULT ''");
ensureColumn("device_id", "device_id VARCHAR(64) NOT NULL DEFAULT ''");
ensureColumn("plate", "plate VARCHAR(64) NOT NULL DEFAULT ''");
ensureColumn("province", "province VARCHAR(32) NOT NULL DEFAULT ''");
ensureColumn("city", "city VARCHAR(32) NOT NULL DEFAULT ''");
ensureColumn("maker", "maker VARCHAR(64) NOT NULL DEFAULT ''");
ensureColumn("device_type", "device_type VARCHAR(128) NOT NULL DEFAULT ''");
ensureColumn("plate_color", "plate_color VARCHAR(32) NOT NULL DEFAULT ''");
ensureColumn("metadata_json", "metadata_json TEXT");
ensureColumn("created_at", "created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP");
ensureColumn("updated_at", "updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP");
ensureIndex("idx_vehicle_identity_protocol_device", "CREATE INDEX idx_vehicle_identity_protocol_device ON %s (protocol, device_id)");
ensureIndex("idx_vehicle_identity_protocol_plate", "CREATE INDEX idx_vehicle_identity_protocol_plate ON %s (protocol, plate)");
}
private void ensureColumn(String columnName, String definition) {
if (!columnExists(columnName)) {
jdbc.execute("ALTER TABLE " + tableName + " ADD COLUMN " + definition);
}
}
private InMemoryVehicleIdentityService loadIndex() {
InMemoryVehicleIdentityService loaded = new InMemoryVehicleIdentityService();
String sql = "SELECT plate, vin FROM " + table;
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql);
ResultSet resultSet = statement.executeQuery()) {
while (resultSet.next()) {
bindGlobalPlate(loaded, resultSet.getString("vin"), resultSet.getString("plate"));
private boolean columnExists(String columnName) {
return Boolean.TRUE.equals(jdbc.execute((ConnectionCallback<Boolean>) connection -> {
DatabaseMetaData metaData = connection.getMetaData();
for (String table : nameVariants(tableName)) {
for (String column : nameVariants(columnName)) {
try (ResultSet rs = metaData.getColumns(null, null, table, column)) {
if (rs.next()) {
return true;
}
}
}
}
loadRegistrationIndex(loaded, connection);
return loaded;
} catch (SQLException e) {
throw new IllegalStateException("vehicle identity mysql index load failed: " + table, e);
return false;
}));
}
private void ensureIndex(String indexName, String ddlTemplate) {
if (!indexExists(indexName)) {
jdbc.execute(ddlTemplate.formatted(tableName));
}
}
private static void bindGlobalPlate(InMemoryVehicleIdentityService loaded, String vin, String plate) {
for (ProtocolId protocol : new ProtocolId[]{
ProtocolId.GB32960,
ProtocolId.JT808,
ProtocolId.MQTT_YUTONG,
ProtocolId.XINDA_PUSH
}) {
loaded.bind(new VehicleIdentityBinding(protocol, vin, "", "", plate));
}
}
private void loadRegistrationIndex(InMemoryVehicleIdentityService loaded, Connection connection) throws SQLException {
String sql = "SELECT r.protocol, r.phone, r.device_id, r.plate, b.vin FROM " + registrationTable + " r "
+ "JOIN " + table + " b ON b.plate = r.plate "
+ "WHERE b.vin IS NOT NULL AND b.vin <> '' AND LOWER(b.vin) <> 'unknown'";
try (PreparedStatement statement = connection.prepareStatement(sql);
ResultSet resultSet = statement.executeQuery()) {
while (resultSet.next()) {
loaded.bind(new VehicleIdentityBinding(
protocol(resultSet.getString("protocol")),
resultSet.getString("vin"),
resultSet.getString("phone"),
resultSet.getString("device_id"),
resultSet.getString("plate")));
private boolean indexExists(String indexName) {
return Boolean.TRUE.equals(jdbc.execute((ConnectionCallback<Boolean>) connection -> {
DatabaseMetaData metaData = connection.getMetaData();
for (String table : nameVariants(tableName)) {
try (ResultSet rs = metaData.getIndexInfo(null, null, table, false, false)) {
while (rs.next()) {
String existing = rs.getString("INDEX_NAME");
if (existing != null && existing.equalsIgnoreCase(indexName)) {
return true;
}
}
}
}
}
loadLegacyRegistrationVinIndex(loaded, connection);
return false;
}));
}
private void loadLegacyRegistrationVinIndex(InMemoryVehicleIdentityService loaded, Connection connection) throws SQLException {
String sql = "SELECT protocol, phone, device_id, plate, vin FROM " + registrationTable
+ " WHERE vin IS NOT NULL AND vin <> '' AND LOWER(vin) <> 'unknown'";
try (PreparedStatement statement = connection.prepareStatement(sql);
ResultSet resultSet = statement.executeQuery()) {
while (resultSet.next()) {
loaded.bind(new VehicleIdentityBinding(
protocol(resultSet.getString("protocol")),
resultSet.getString("vin"),
resultSet.getString("phone"),
resultSet.getString("device_id"),
resultSet.getString("plate")));
}
}
private static String[] nameVariants(String name) {
return new String[]{
name,
name.toLowerCase(Locale.ROOT),
name.toUpperCase(Locale.ROOT)
};
}
private ScheduledExecutorService startRefresher(Duration refreshInterval) {
if (refreshInterval == null || refreshInterval.isZero() || refreshInterval.isNegative()) {
return null;
private String metadataJson(Map<String, String> metadata) {
if (metadata == null || metadata.isEmpty()) {
return "{}";
}
long delayMillis = Math.max(refreshInterval.toMillis(), 1000L);
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(r -> {
Thread t = new Thread(r, "vehicle-identity-mysql-refresh");
t.setDaemon(true);
return t;
});
executor.scheduleWithFixedDelay(this::safeRefresh, delayMillis, delayMillis, TimeUnit.MILLISECONDS);
return executor;
}
private void safeRefresh() {
try {
refresh();
} catch (RuntimeException e) {
log.warn("vehicle identity mysql refresh failed table={}", table, e);
return mapper.writeValueAsString(metadata);
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("vehicle identity metadata serialize failed", e);
}
}
private void upsert(String plate, String vin) {
String normalizedPlate = normalize(plate);
if (normalizedPlate.isBlank()) {
return;
}
String sql = "INSERT INTO " + table + " (plate, vin) "
+ "VALUES (?, ?) "
+ "ON DUPLICATE KEY UPDATE vin = VALUES(vin)";
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql)) {
statement.setString(1, normalizedPlate);
statement.setString(2, vin.trim());
statement.executeUpdate();
} catch (SQLException e) {
throw new IllegalStateException("vehicle identity mysql bind failed: " + table, e);
}
}
private void upsertRegistration(VehicleRegistrationBinding registration) {
String phone = normalize(registration.phone());
if (phone.isBlank()) {
return;
}
String vin = registration.hasResolvedVin() ? registration.vin().trim() : "unknown";
String sql = "INSERT INTO " + registrationTable
+ " (protocol, phone, vin, device_id, plate, province, city, maker, device_type, plate_color) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
+ "ON DUPLICATE KEY UPDATE "
+ "vin = CASE WHEN VALUES(vin) <> 'unknown' THEN VALUES(vin) ELSE vin END, "
+ "device_id = VALUES(device_id), plate = VALUES(plate), province = VALUES(province), "
+ "city = VALUES(city), maker = VALUES(maker), device_type = VALUES(device_type), "
+ "plate_color = VALUES(plate_color), last_registered_at = CURRENT_TIMESTAMP";
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql)) {
statement.setString(1, protocolName(registration.protocol()));
statement.setString(2, phone);
statement.setString(3, vin);
statement.setString(4, normalize(registration.deviceId()));
statement.setString(5, normalize(registration.plate()));
setInteger(statement, 6, registration.province());
setInteger(statement, 7, registration.city());
statement.setString(8, normalize(registration.maker()));
statement.setString(9, normalize(registration.deviceType()));
setInteger(statement, 10, registration.plateColor());
statement.executeUpdate();
} catch (SQLException e) {
throw new IllegalStateException("vehicle identity mysql registration bind failed: " + registrationTable, e);
}
}
private static void setInteger(PreparedStatement statement, int index, Integer value) throws SQLException {
if (value == null) {
statement.setObject(index, null);
} else {
statement.setInt(index, value);
}
}
private static String sanitizeTable(String table) {
String value = table == null || table.isBlank() ? "vehicle_identity_binding" : table.trim();
if (!value.matches("[A-Za-z0-9_]+")) {
throw new IllegalArgumentException("vehicle identity mysql table must contain only letters, digits, and underscore");
}
return value;
}
private static String protocolName(ProtocolId protocol) {
private static String protocol(ProtocolId protocol) {
return protocol == null ? "UNKNOWN" : protocol.name();
}
private static ProtocolId protocol(String value) {
if (value == null || value.isBlank()) {
return ProtocolId.UNKNOWN;
private static String safeTableName(String value) {
String name = value == null || value.isBlank() ? "vehicle_identity_bindings" : value.trim();
if (!name.matches("[A-Za-z0-9_]+")) {
throw new IllegalArgumentException("invalid vehicle identity mysql table name: " + value);
}
try {
return ProtocolId.valueOf(value.trim().toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException ignored) {
return ProtocolId.UNKNOWN;
}
}
private static String normalize(String value) {
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
return name.toLowerCase(Locale.ROOT);
}
}

View File

@@ -2,18 +2,26 @@ package com.lingniu.ingest.identity;
import com.lingniu.ingest.api.ProtocolId;
import java.util.Map;
public record VehicleIdentityBinding(
ProtocolId protocol,
String vin,
String phone,
String deviceId,
String plate
String plate,
Map<String, String> metadata
) {
public VehicleIdentityBinding(ProtocolId protocol, String vin, String phone, String deviceId, String plate) {
this(protocol, vin, phone, deviceId, plate, Map.of());
}
public VehicleIdentityBinding {
vin = normalize(vin);
phone = normalize(phone);
deviceId = normalize(deviceId);
plate = normalize(plate);
metadata = metadata == null ? Map.of() : Map.copyOf(metadata);
if (vin.isBlank()) {
throw new IllegalArgumentException("vin must not be blank");
}

View File

@@ -2,6 +2,7 @@ package com.lingniu.ingest.identity;
public enum VehicleIdentitySource {
EXPLICIT_VIN,
REGISTERED,
BOUND_PHONE,
BOUND_DEVICE_ID,
BOUND_PLATE,

View File

@@ -11,15 +11,10 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource;
import java.io.PrintWriter;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.logging.Logger;
@AutoConfiguration
@EnableConfigurationProperties(VehicleIdentityProperties.class)
@@ -40,22 +35,15 @@ public class VehicleIdentityAutoConfiguration {
return new FileVehicleIdentityService(Path.of(properties.getFile().getPath()), objectMapper);
}
@Bean
@ConditionalOnMissingBean(DataSource.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql")
public DataSource vehicleIdentityDataSource(VehicleIdentityProperties properties) {
VehicleIdentityProperties.Mysql mysql = properties.getMysql();
return new DriverManagerVehicleIdentityDataSource(
mysql.getDriverClassName(), mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword());
}
@Bean
@ConditionalOnMissingBean({VehicleIdentityResolver.class, VehicleIdentityRegistry.class})
@ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql")
public MySqlVehicleIdentityService mySqlVehicleIdentityService(VehicleIdentityProperties properties,
DataSource dataSource) {
public MySqlVehicleIdentityService mysqlVehicleIdentityService(VehicleIdentityProperties properties,
ObjectMapper objectMapper) {
VehicleIdentityProperties.Mysql mysql = properties.getMysql();
return new MySqlVehicleIdentityService(dataSource, mysql.getTable(), mysql.getRefreshInterval());
DataSource dataSource = new DriverManagerDataSource(
mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword());
return new MySqlVehicleIdentityService(dataSource, mysql, objectMapper);
}
@Bean
@@ -65,61 +53,4 @@ public class VehicleIdentityAutoConfiguration {
public InMemoryVehicleIdentityService vehicleIdentityService() {
return new InMemoryVehicleIdentityService();
}
private static final class DriverManagerVehicleIdentityDataSource implements DataSource {
private final String jdbcUrl;
private final String username;
private final String password;
private PrintWriter logWriter;
private int loginTimeout;
private DriverManagerVehicleIdentityDataSource(String driverClassName,
String jdbcUrl,
String username,
String password) {
if (jdbcUrl == null || jdbcUrl.isBlank()) {
throw new IllegalArgumentException("lingniu.ingest.identity.mysql.jdbc-url is required");
}
if (driverClassName != null && !driverClassName.isBlank()) {
try {
Class.forName(driverClassName.trim());
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("vehicle identity mysql driver class not found: "
+ driverClassName, e);
}
}
this.jdbcUrl = jdbcUrl;
this.username = username == null ? "" : username;
this.password = password == null ? "" : password;
}
@Override
public Connection getConnection() throws SQLException {
DriverManager.setLoginTimeout(loginTimeout);
return DriverManager.getConnection(jdbcUrl, username, password);
}
@Override
public Connection getConnection(String username, String password) throws SQLException {
DriverManager.setLoginTimeout(loginTimeout);
return DriverManager.getConnection(jdbcUrl, username, password);
}
@Override public PrintWriter getLogWriter() { return logWriter; }
@Override public void setLogWriter(PrintWriter out) { this.logWriter = out; }
@Override public void setLoginTimeout(int seconds) { this.loginTimeout = seconds; }
@Override public int getLoginTimeout() { return loginTimeout; }
@Override public Logger getParentLogger() throws SQLFeatureNotSupportedException {
throw new SQLFeatureNotSupportedException();
}
@Override public <T> T unwrap(Class<T> iface) throws SQLException {
if (iface.isInstance(this)) {
return iface.cast(this);
}
throw new SQLException("not a wrapper for " + iface);
}
@Override public boolean isWrapperFor(Class<?> iface) {
return iface.isInstance(this);
}
}
}

View File

@@ -2,8 +2,6 @@ package com.lingniu.ingest.identity.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.time.Duration;
@ConfigurationProperties(prefix = "lingniu.ingest.identity")
public class VehicleIdentityProperties {
@@ -26,24 +24,21 @@ public class VehicleIdentityProperties {
}
public static class Mysql {
private String table = "vehicle_identity_binding";
private String jdbcUrl = "";
private String username = "";
private String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai";
private String username = "root";
private String password = "";
private String driverClassName = "com.mysql.cj.jdbc.Driver";
private Duration refreshInterval = Duration.ofSeconds(60);
private String tableName = "vehicle_identity_bindings";
private boolean initializeSchema = true;
public String getTable() { return table; }
public void setTable(String table) { this.table = table; }
public String getJdbcUrl() { return jdbcUrl; }
public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; }
public String getUsername() { return username; }
public void setUsername(String username) { this.username = username; }
public String getPassword() { return password; }
public void setPassword(String password) { this.password = password; }
public String getDriverClassName() { return driverClassName; }
public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; }
public Duration getRefreshInterval() { return refreshInterval; }
public void setRefreshInterval(Duration refreshInterval) { this.refreshInterval = refreshInterval; }
public String getTableName() { return tableName; }
public void setTableName(String tableName) { this.tableName = tableName; }
public boolean isInitializeSchema() { return initializeSchema; }
public void setInitializeSchema(boolean initializeSchema) { this.initializeSchema = initializeSchema; }
}
}

View File

@@ -1,6 +1,8 @@
package com.lingniu.ingest.identity;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
import org.h2.jdbcx.JdbcDataSource;
import org.junit.jupiter.api.Test;
@@ -9,6 +11,7 @@ import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
@@ -21,62 +24,67 @@ class MySqlVehicleIdentityServiceJdbcTest {
dataSource.setURL("jdbc:h2:mem:" + UUID.randomUUID()
+ ";MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1");
try (MySqlVehicleIdentityService service =
new MySqlVehicleIdentityService(dataSource, "vehicle_identity_binding")) {
service.register(new VehicleRegistrationBinding(
ProtocolId.JT808,
"unknown",
"13079961001",
"dev61001",
"粤B61001",
44,
4401,
"maker-a",
"type-a",
1));
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
mysql.setTableName("vehicle_identity_binding");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
dataSource,
mysql,
new ObjectMapper());
try (Connection connection = dataSource.getConnection();
Statement statement = connection.createStatement()) {
ResultSet registered = statement.executeQuery("""
SELECT vin, device_id, plate, province, city, maker, device_type, plate_color
FROM vehicle_identity_binding_registration
WHERE protocol = 'JT808' AND phone = '13079961001'
""");
assertThat(registered.next()).isTrue();
assertThat(registered.getString("vin")).isEqualTo("unknown");
assertThat(registered.getString("device_id")).isEqualTo("DEV61001");
assertThat(registered.getString("plate")).isEqualTo("粤B61001");
assertThat(registered.getInt("province")).isEqualTo(44);
assertThat(registered.getInt("city")).isEqualTo(4401);
assertThat(registered.getString("maker")).isEqualTo("MAKER-A");
assertThat(registered.getString("device_type")).isEqualTo("TYPE-A");
assertThat(registered.getInt("plate_color")).isEqualTo(1);
service.bind(new VehicleIdentityBinding(
ProtocolId.JT808,
"LNVIN00000061001",
"13079961001",
"dev61001",
"粤B61001",
Map.of(
"province", "44",
"city", "4401",
"maker", "maker-a",
"deviceType", "type-a",
"plateColor", "1")));
List<String> columns = new ArrayList<>();
ResultSet bindingColumns = connection.getMetaData()
.getColumns(null, null, "vehicle_identity_binding", null);
while (bindingColumns.next()) {
columns.add(bindingColumns.getString("COLUMN_NAME"));
}
assertThat(columns).containsExactly("plate", "vin");
try (Connection connection = dataSource.getConnection();
Statement statement = connection.createStatement()) {
ResultSet binding = statement.executeQuery("""
SELECT vin, phone, device_id, plate, province, city, maker, device_type, plate_color
FROM vehicle_identity_binding
WHERE protocol = 'JT808' AND phone = '13079961001'
""");
assertThat(binding.next()).isTrue();
assertThat(binding.getString("vin")).isEqualTo("LNVIN00000061001");
assertThat(binding.getString("phone")).isEqualTo("13079961001");
assertThat(binding.getString("device_id")).isEqualTo("dev61001");
assertThat(binding.getString("plate")).isEqualTo("粤B61001");
assertThat(binding.getString("province")).isEqualTo("44");
assertThat(binding.getString("city")).isEqualTo("4401");
assertThat(binding.getString("maker")).isEqualTo("maker-a");
assertThat(binding.getString("device_type")).isEqualTo("type-a");
assertThat(binding.getString("plate_color")).isEqualTo("1");
statement.executeUpdate("""
INSERT INTO vehicle_identity_binding (plate, vin)
VALUES ('粤B61001', 'LNVIN00000061001')
""");
List<String> columns = new ArrayList<>();
ResultSet bindingColumns = connection.getMetaData()
.getColumns(null, null, "vehicle_identity_binding", null);
while (bindingColumns.next()) {
columns.add(bindingColumns.getString("COLUMN_NAME"));
}
service.refresh();
assertThat(service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079961001", "", "")).vin())
.isEqualTo("LNVIN00000061001");
assertThat(service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "dev61001", "")).vin())
.isEqualTo("LNVIN00000061001");
assertThat(service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "", "粤B61001")).vin())
.isEqualTo("LNVIN00000061001");
assertThat(columns).contains(
"protocol",
"phone",
"device_id",
"plate",
"vin",
"metadata_json");
}
assertThat(service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079961001", "", "")).vin())
.isEqualTo("LNVIN00000061001");
assertThat(service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "dev61001", "")).vin())
.isEqualTo("LNVIN00000061001");
assertThat(service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "", "粤B61001")).vin())
.isEqualTo("LNVIN00000061001");
}
}

View File

@@ -1,478 +1,157 @@
package com.lingniu.ingest.identity;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.identity.config.VehicleIdentityProperties;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource;
import java.io.PrintWriter;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.logging.Logger;
import static org.assertj.core.api.Assertions.assertThat;
class MySqlVehicleIdentityServiceTest {
@Test
void bindsAndResolvesIdentityThroughJdbcStore() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
void upsertsRegistrationBindingAndResolvesByExternalIdentifiers() {
DriverManagerDataSource dataSource = new DriverManagerDataSource(
"jdbc:h2:mem:mysql-identity;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
mysql.setTableName("vehicle_identity_bindings");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
dataSource,
mysql,
new ObjectMapper());
service.bind(new VehicleIdentityBinding(
ProtocolId.JT808, "LNVIN000000000123", "13079960001", "DEV001", "粤B00001"));
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960001", "", ""));
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "DEV001", ""));
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "", "粤B00001"));
assertThat(jdbc.createdTables)
.anySatisfy(sql -> assertThat(sql).contains("CREATE TABLE IF NOT EXISTS vehicle_identity_binding"));
assertThat(jdbc.bindings).containsEntry("粤B00001", "LNVIN000000000123");
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000123");
assertThat(byPhone.resolved()).isTrue();
assertThat(byPhone.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000123");
assertThat(byDevice.source()).isEqualTo(VehicleIdentitySource.BOUND_DEVICE_ID);
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000123");
assertThat(byPlate.source()).isEqualTo(VehicleIdentitySource.BOUND_PLATE);
}
@Test
void ignoresUnknownVinBinding() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
service.bind(new VehicleIdentityBinding(
ProtocolId.JT808, "unknown", "13079960001", "DEV001", "粤B00001"));
assertThat(jdbc.bindings).isEmpty();
}
@Test
void resolvesFromStartupCacheWithoutPerFrameDatabaseLookup() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
jdbc.seedRegistration("JT808", "13079960001", "DEV001", "粤B00001");
jdbc.seed("粤B00001", "LNVIN000000000123");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960001", "", ""));
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "DEV001", ""));
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "", "粤B00001"));
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000123");
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000123");
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000123");
assertThat(jdbc.loadAllSelects).isEqualTo(1);
assertThat(jdbc.lookupSelects).isZero();
}
@Test
void resolvesGlobalPlateBindingForXindaPush() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
jdbc.seed("浙F00780F", "LA9GG68L8PBAF4776");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
ProtocolId.XINDA_PUSH, "", "", "", "浙F00780F"));
assertThat(byPlate.vin()).isEqualTo("LA9GG68L8PBAF4776");
assertThat(byPlate.resolved()).isTrue();
assertThat(byPlate.source()).isEqualTo(VehicleIdentitySource.BOUND_PLATE);
assertThat(jdbc.loadAllSelects).isEqualTo(1);
assertThat(jdbc.lookupSelects).isZero();
}
@Test
void refreshLoadsExternallyInsertedBindingWithoutPerFrameDatabaseLookup() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
jdbc.seedRegistration("JT808", "13079960002", "DEV002", "粤B00002");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
VehicleIdentity before = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960002", "", ""));
jdbc.seed("粤B00002", "LNVIN000000000456");
service.refresh();
VehicleIdentity after = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960002", "", ""));
assertThat(before.resolved()).isFalse();
assertThat(after.vin()).isEqualTo("LNVIN000000000456");
assertThat(after.resolved()).isTrue();
assertThat(after.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
assertThat(jdbc.loadAllSelects).isEqualTo(2);
assertThat(jdbc.lookupSelects).isZero();
}
@Test
void storesRegistrationWithoutVinAndResolvesAfterPlateVinIsWrittenBack() {
RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc();
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding");
service.register(new VehicleRegistrationBinding(
ProtocolId.JT808,
"unknown",
"13079960003",
"DEV003",
"粤B00003",
44,
4401,
"MAKER",
"TYPE-A",
1));
"JT808AABBCCDDEE1",
"13079963289",
"9963289",
"沪A00113F",
Map.of(
"province", "31",
"city", "113",
"maker", "70112",
"deviceType", "SEG-9888G",
"plateColor", "2")));
VehicleIdentity before = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960003", "", ""));
assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "13079963289", "", "")))
.extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_PHONE);
assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "", "9963289", "")))
.extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_DEVICE_ID);
assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "", "", "沪A00113F")))
.extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source)
.containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_PLATE);
jdbc.seed("粤B00003", "LNVIN000000000789");
service.refresh();
VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "13079960003", "", ""));
VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "DEV003", ""));
VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "", "", "粤B00003"));
assertThat(jdbc.createdTables)
.anySatisfy(sql -> assertThat(sql).contains("CREATE TABLE IF NOT EXISTS vehicle_identity_binding_registration"));
assertThat(jdbc.registrations).containsKey("JT808|13079960003");
assertThat(jdbc.bindings).containsEntry("粤B00003", "LNVIN000000000789");
assertThat(before.resolved()).isFalse();
assertThat(byPhone.vin()).isEqualTo("LNVIN000000000789");
assertThat(byPhone.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
assertThat(byDevice.vin()).isEqualTo("LNVIN000000000789");
assertThat(byPlate.vin()).isEqualTo("LNVIN000000000789");
JdbcTemplate jdbc = new JdbcTemplate(dataSource);
Map<String, Object> row = jdbc.queryForMap("""
SELECT vin, phone, device_id, plate, province, city, maker, device_type, plate_color
FROM vehicle_identity_bindings
WHERE protocol = 'JT808' AND phone = '13079963289'
""");
assertThat(row)
.containsEntry("vin", "JT808AABBCCDDEE1")
.containsEntry("phone", "13079963289")
.containsEntry("device_id", "9963289")
.containsEntry("plate", "沪A00113F")
.containsEntry("province", "31")
.containsEntry("city", "113")
.containsEntry("maker", "70112")
.containsEntry("device_type", "SEG-9888G")
.containsEntry("plate_color", "2");
}
private static final class RecordingIdentityJdbc {
private final List<String> createdTables = new ArrayList<>();
private final Map<String, String> bindings = new HashMap<>();
private final Map<String, RegistrationRow> registrations = new HashMap<>();
private int loadAllSelects;
private int lookupSelects;
@Test
void initializesMissingColumnsOnExistingLegacyTable() {
DriverManagerDataSource dataSource = new DriverManagerDataSource(
"jdbc:h2:mem:mysql-identity-legacy;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
JdbcTemplate jdbc = new JdbcTemplate(dataSource);
jdbc.execute("""
CREATE TABLE vehicle_identity_binding (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL,
legacy_key VARCHAR(64) NOT NULL DEFAULT ''
)
""");
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
mysql.setTableName("vehicle_identity_binding");
private void seed(String plate, String vin) {
bindings.put(normalize(plate), vin);
}
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
dataSource,
mysql,
new ObjectMapper());
service.bind(new VehicleIdentityBinding(
ProtocolId.JT808,
"JT808LEGACY00001",
"13079963289",
"9963289",
"沪A00113F",
Map.of("maker", "70112", "deviceType", "SEG-9888G")));
private void seedRegistration(String protocol, String phone, String deviceId, String plate) {
registrations.put(protocol + "|" + normalize(phone),
new RegistrationRow(protocol, normalize(phone), "unknown", normalize(deviceId), normalize(plate)));
}
Map<String, Object> row = jdbc.queryForMap("""
SELECT vin, maker, device_type, metadata_json
FROM vehicle_identity_binding
WHERE protocol = 'JT808' AND phone = '13079963289'
""");
assertThat(row)
.containsEntry("vin", "JT808LEGACY00001")
.containsEntry("maker", "70112")
.containsEntry("device_type", "SEG-9888G");
assertThat((String) row.get("metadata_json")).contains("SEG-9888G");
}
private DataSource dataSource() {
return new DataSource() {
@Override public Connection getConnection() { return connection(); }
@Override public Connection getConnection(String username, String password) { return connection(); }
@Override public PrintWriter getLogWriter() { return null; }
@Override public void setLogWriter(PrintWriter out) {}
@Override public void setLoginTimeout(int seconds) {}
@Override public int getLoginTimeout() { return 0; }
@Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { throw new SQLFeatureNotSupportedException(); }
@Override public <T> T unwrap(Class<T> iface) { throw new UnsupportedOperationException(); }
@Override public boolean isWrapperFor(Class<?> iface) { return false; }
};
}
@Test
void upsertsByCompositeProtocolPhonePlatePrimaryKey() {
DriverManagerDataSource dataSource = new DriverManagerDataSource(
"jdbc:h2:mem:mysql-identity-composite;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", "");
JdbcTemplate jdbc = new JdbcTemplate(dataSource);
jdbc.execute("""
CREATE TABLE vehicle_identity_binding (
protocol VARCHAR(32) NOT NULL DEFAULT 'ALL',
phone VARCHAR(128) NOT NULL DEFAULT '',
plate VARCHAR(128) NOT NULL DEFAULT '',
vin VARCHAR(64) NOT NULL,
province VARCHAR(32) NOT NULL DEFAULT '',
city VARCHAR(32) NOT NULL DEFAULT '',
maker VARCHAR(64) NOT NULL DEFAULT '',
device_type VARCHAR(128) NOT NULL DEFAULT '',
plate_color VARCHAR(32) NOT NULL DEFAULT '',
metadata_json TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
device_id VARCHAR(64) NOT NULL DEFAULT '',
PRIMARY KEY (protocol, phone, plate)
)
""");
VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql();
mysql.setTableName("vehicle_identity_binding");
MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(
dataSource,
mysql,
new ObjectMapper());
service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
"VIN-A", "13079963289", "DEV-A", "沪A00113F"));
private Connection connection() {
InvocationHandler handler = (proxy, method, args) -> switch (method.getName()) {
case "createStatement" -> statement();
case "prepareStatement" -> preparedStatement((String) args[0]);
case "close" -> null;
case "isClosed" -> false;
case "unwrap" -> throw new UnsupportedOperationException();
case "isWrapperFor" -> false;
default -> defaultValue(method.getReturnType());
};
return (Connection) Proxy.newProxyInstance(
Connection.class.getClassLoader(), new Class<?>[]{Connection.class}, handler);
}
service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
"VIN-B", "13079963289", "DEV-B", "沪A00114F"));
service.bind(new VehicleIdentityBinding(ProtocolId.JT808,
"VIN-B2", "13079963289", "DEV-B2", "沪A00114F"));
private Statement statement() {
InvocationHandler handler = (proxy, method, args) -> {
if ("execute".equals(method.getName())) {
createdTables.add((String) args[0]);
return false;
}
if ("close".equals(method.getName())) {
return null;
}
return defaultValue(method.getReturnType());
};
return (Statement) Proxy.newProxyInstance(
Statement.class.getClassLoader(), new Class<?>[]{Statement.class}, handler);
}
private PreparedStatement preparedStatement(String sql) {
List<Object> params = new ArrayList<>();
InvocationHandler handler = (proxy, method, args) -> {
switch (method.getName()) {
case "setString", "setInt", "setObject" -> {
set(params, (Integer) args[0], args[1]);
return null;
}
case "executeUpdate" -> {
if (sql.startsWith("INSERT INTO vehicle_identity_binding_registration ")) {
upsertRegistration(params);
} else {
upsert(params);
}
return 1;
}
case "executeQuery" -> {
if (sql.startsWith("SELECT plate, vin FROM ")) {
loadAllSelects++;
return allBindingsResultSet();
}
if (sql.startsWith("SELECT protocol, identifier_type, identifier_value, vin FROM ")) {
loadAllSelects++;
return legacyAllBindingsResultSet();
}
if (sql.startsWith("SELECT r.protocol, r.phone, r.device_id, r.plate, b.vin FROM ")) {
return registrationResultSet();
}
if (sql.startsWith("SELECT protocol, phone, device_id, plate, vin FROM ")) {
return legacyRegistrationResultSet();
}
lookupSelects++;
return resultSet(resolve(params));
}
case "close" -> {
return null;
}
default -> {
return defaultValue(method.getReturnType());
}
}
};
return (PreparedStatement) Proxy.newProxyInstance(
PreparedStatement.class.getClassLoader(), new Class<?>[]{PreparedStatement.class}, handler);
}
private void upsert(List<Object> params) {
String plate;
String vin;
if (params.size() >= 4) {
plate = normalize((String) params.get(2));
vin = (String) params.get(3);
} else {
plate = normalize((String) params.get(0));
vin = (String) params.get(1);
}
bindings.put(plate, vin);
}
private void upsertRegistration(List<Object> params) {
String protocol = (String) params.get(0);
String phone = normalize((String) params.get(1));
String vin = (String) params.get(2);
String key = protocol + "|" + phone;
RegistrationRow existing = registrations.get(key);
String effectiveVin = existing != null && "unknown".equalsIgnoreCase(vin) ? existing.vin : vin;
registrations.put(key, new RegistrationRow(
protocol,
phone,
effectiveVin,
normalize((String) params.get(3)),
normalize((String) params.get(4))));
}
private String resolve(List<Object> params) {
String protocol = (String) params.get(0);
String identifierType = (String) params.get(1);
String identifierValue = normalize((String) params.get(2));
if (!"PLATE".equals(identifierType)) {
return null;
}
return bindings.get(identifierValue);
}
private ResultSet allBindingsResultSet() {
List<Map.Entry<String, String>> entries = new ArrayList<>(bindings.entrySet());
InvocationHandler handler = new InvocationHandler() {
private int index = -1;
@Override
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
return switch (method.getName()) {
case "next" -> {
index++;
yield index < entries.size();
}
case "getString" -> getColumn(entries.get(index), args[0]);
case "close" -> null;
default -> defaultValue(method.getReturnType());
};
}
};
return (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
}
private ResultSet legacyAllBindingsResultSet() {
List<Map.Entry<String, String>> entries = new ArrayList<>(bindings.entrySet());
InvocationHandler handler = new InvocationHandler() {
private int index = -1;
@Override
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
return switch (method.getName()) {
case "next" -> {
index++;
yield index < entries.size();
}
case "getString" -> legacyColumn(entries.get(index), args[0]);
case "close" -> null;
default -> defaultValue(method.getReturnType());
};
}
};
return (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
}
private static String getColumn(Map.Entry<String, String> entry, Object column) {
String name = column.toString();
return switch (name) {
case "plate" -> entry.getKey();
case "vin" -> entry.getValue();
default -> throw new IllegalArgumentException("unsupported column " + name);
};
}
private static String legacyColumn(Map.Entry<String, String> entry, Object column) {
return switch (column.toString()) {
case "protocol" -> "JT808";
case "identifier_type" -> "PLATE";
case "identifier_value" -> entry.getKey();
case "vin" -> entry.getValue();
default -> throw new IllegalArgumentException("unsupported column " + column);
};
}
private ResultSet registrationResultSet() {
List<RegistrationRow> rows = registrations.values().stream()
.filter(row -> bindings.containsKey(row.plate))
.map(row -> new RegistrationRow(row.protocol, row.phone, bindings.get(row.plate), row.deviceId, row.plate))
.toList();
return registrationRowsResultSet(rows);
}
private ResultSet legacyRegistrationResultSet() {
List<RegistrationRow> rows = registrations.values().stream()
.filter(row -> row.vin != null && !row.vin.isBlank() && !"unknown".equalsIgnoreCase(row.vin))
.toList();
return registrationRowsResultSet(rows);
}
private ResultSet registrationRowsResultSet(List<RegistrationRow> rows) {
InvocationHandler handler = new InvocationHandler() {
private int index = -1;
@Override
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
return switch (method.getName()) {
case "next" -> {
index++;
yield index < rows.size();
}
case "getString" -> registrationColumn(rows.get(index), args[0]);
case "close" -> null;
default -> defaultValue(method.getReturnType());
};
}
};
return (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
}
private static String registrationColumn(RegistrationRow row, Object column) {
return switch (column.toString()) {
case "protocol" -> row.protocol;
case "phone" -> row.phone;
case "device_id" -> row.deviceId;
case "plate" -> row.plate;
case "vin" -> row.vin;
default -> throw new IllegalArgumentException("unsupported column " + column);
};
}
private static ResultSet resultSet(String vin) {
InvocationHandler handler = new InvocationHandler() {
private boolean read;
@Override
public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) {
return switch (method.getName()) {
case "next" -> {
boolean hasNext = !read && vin != null;
read = true;
yield hasNext;
}
case "getString" -> vin;
case "close" -> null;
default -> defaultValue(method.getReturnType());
};
}
};
return (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class}, handler);
}
private static void set(List<Object> params, int oneBasedIndex, Object value) {
while (params.size() < oneBasedIndex) {
params.add(null);
}
params.set(oneBasedIndex - 1, value);
}
private static String normalize(String value) {
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
}
private static Object defaultValue(Class<?> type) {
if (type == Void.TYPE) return null;
if (type == Boolean.TYPE) return false;
if (type == Integer.TYPE) return 0;
if (type == Long.TYPE) return 0L;
if (type == Double.TYPE) return 0D;
if (type == Float.TYPE) return 0F;
if (type == Short.TYPE) return (short) 0;
if (type == Byte.TYPE) return (byte) 0;
return null;
}
private static final class RegistrationRow {
private final String protocol;
private final String phone;
private String vin;
private final String deviceId;
private final String plate;
private RegistrationRow(String protocol, String phone, String vin, String deviceId, String plate) {
this.protocol = protocol;
this.phone = phone;
this.vin = vin;
this.deviceId = deviceId;
this.plate = plate;
}
}
Integer rows = jdbc.queryForObject("""
SELECT COUNT(*)
FROM vehicle_identity_binding
WHERE protocol = 'JT808' AND phone = '13079963289'
""", Integer.class);
assertThat(rows).isEqualTo(2);
assertThat(jdbc.queryForObject("""
SELECT vin
FROM vehicle_identity_binding
WHERE protocol = 'JT808' AND phone = '13079963289' AND plate = '沪A00114F'
""", String.class)).isEqualTo("VIN-B2");
}
}

View File

@@ -7,14 +7,7 @@ import org.junit.jupiter.api.io.TempDir;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import javax.sql.DataSource;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.lang.reflect.Proxy;
import static org.assertj.core.api.Assertions.assertThat;
@@ -52,75 +45,19 @@ class VehicleIdentityAutoConfigurationTest {
@Test
void createsMysqlIdentityServiceWhenConfigured() {
runner
.withBean(DataSource.class, StubDataSource::new)
.withPropertyValues(
"lingniu.ingest.identity.store=mysql",
"lingniu.ingest.identity.mysql.table=vehicle_identity_binding")
"lingniu.ingest.identity.mysql.jdbc-url=jdbc:h2:mem:auto-identity;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1",
"lingniu.ingest.identity.mysql.username=sa",
"lingniu.ingest.identity.mysql.password=",
"lingniu.ingest.identity.mysql.table-name=vehicle_identity_bindings")
.run(context -> {
assertThat(context).hasSingleBean(VehicleIdentityResolver.class);
assertThat(context.getBean(VehicleIdentityResolver.class).getClass().getSimpleName())
.isEqualTo("MySqlVehicleIdentityService");
assertThat(context).hasSingleBean(VehicleIdentityRegistry.class);
assertThat(context.getBean(VehicleIdentityResolver.class))
.isSameAs(context.getBean(VehicleIdentityRegistry.class));
assertThat(context.getBean(VehicleIdentityResolver.class).getClass().getSimpleName())
.isEqualTo("MySqlVehicleIdentityService");
});
}
private static final class StubDataSource implements DataSource {
@Override public Connection getConnection() { return connection(); }
@Override public Connection getConnection(String username, String password) { return connection(); }
@Override public java.io.PrintWriter getLogWriter() { return null; }
@Override public void setLogWriter(java.io.PrintWriter out) {}
@Override public void setLoginTimeout(int seconds) {}
@Override public int getLoginTimeout() { return 0; }
@Override public java.util.logging.Logger getParentLogger() throws java.sql.SQLFeatureNotSupportedException { throw new java.sql.SQLFeatureNotSupportedException(); }
@Override public <T> T unwrap(Class<T> iface) { throw new UnsupportedOperationException(); }
@Override public boolean isWrapperFor(Class<?> iface) { return false; }
private static Connection connection() {
return (Connection) Proxy.newProxyInstance(Connection.class.getClassLoader(), new Class<?>[]{Connection.class},
(proxy, method, args) -> switch (method.getName()) {
case "createStatement" -> statement();
case "prepareStatement" -> preparedStatement();
case "close" -> null;
case "isClosed" -> false;
default -> defaultValue(method.getReturnType());
});
}
private static Statement statement() {
return (Statement) Proxy.newProxyInstance(Statement.class.getClassLoader(), new Class<?>[]{Statement.class},
(proxy, method, args) -> switch (method.getName()) {
case "execute" -> false;
case "close" -> null;
default -> defaultValue(method.getReturnType());
});
}
private static PreparedStatement preparedStatement() {
return (PreparedStatement) Proxy.newProxyInstance(
PreparedStatement.class.getClassLoader(), new Class<?>[]{PreparedStatement.class},
(proxy, method, args) -> switch (method.getName()) {
case "executeQuery" -> emptyResultSet();
case "close" -> null;
default -> defaultValue(method.getReturnType());
});
}
private static ResultSet emptyResultSet() {
return (ResultSet) Proxy.newProxyInstance(ResultSet.class.getClassLoader(), new Class<?>[]{ResultSet.class},
(proxy, method, args) -> switch (method.getName()) {
case "next" -> false;
case "close" -> null;
default -> defaultValue(method.getReturnType());
});
}
private static Object defaultValue(Class<?> type) {
if (type == Void.TYPE) return null;
if (type == Boolean.TYPE) return false;
if (type == Integer.TYPE) return 0;
if (type == Long.TYPE) return 0L;
return null;
}
}
}

View File

@@ -9,7 +9,7 @@
</parent>
<artifactId>inbound-mqtt</artifactId>
<name>inbound-mqtt</name>
<description>MQTT 接入模块(支持多 endpoint宇通等Fusesource MQTT Client + 双向 TLS。</description>
<description>MQTT 接入模块(支持多 endpoint宇通等Eclipse Paho MQTT Client + 双向 TLS。</description>
<dependencies>
<dependency>
@@ -34,8 +34,8 @@
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.fusesource.mqtt-client</groupId>
<artifactId>mqtt-client</artifactId>
<groupId>org.eclipse.paho</groupId>
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>

View File

@@ -11,30 +11,33 @@ import com.lingniu.ingest.identity.VehicleIdentitySource;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import com.lingniu.ingest.inbound.mqtt.model.MqttPayload;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import org.fusesource.mqtt.client.Future;
import org.fusesource.mqtt.client.FutureConnection;
import org.fusesource.mqtt.client.MQTT;
import org.fusesource.mqtt.client.Message;
import org.fusesource.mqtt.client.QoS;
import org.fusesource.mqtt.client.Topic;
import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
import org.eclipse.paho.client.mqttv3.MqttAsyncClient;
import org.eclipse.paho.client.mqttv3.MqttCallbackExtended;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.MqttException;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
/**
* MQTT 多 endpoint 生命周期管理。
*
* <p>为每个 endpoint 创建一个 fusesource MQTT FutureConnection → 连接 → 订阅 → 接收循环
* <p>为每个 endpoint 创建一个 Paho 异步客户端 → 连接 → 订阅 → 注册回调
* 回调里解析 payload → {@link Dispatcher#dispatch(RawFrame)}。
*
* <p>每个 endpoint 在独立虚拟线程中阻塞接收,业务处理由 Dispatcher 侧的 Disruptor 承接。
* <p>MQTT 回调不做业务处理,虚拟线程由 Dispatcher 侧的 Disruptor 承接。
*/
public final class MqttEndpointManager implements AutoCloseable {
@@ -44,9 +47,8 @@ public final class MqttEndpointManager implements AutoCloseable {
private final MqttProfileRegistry profileRegistry;
private final VehicleIdentityResolver identityResolver;
private final Dispatcher dispatcher;
private final List<FutureConnection> connections = new ArrayList<>();
private final List<Thread> workers = new ArrayList<>();
private volatile boolean running;
private final List<MqttAsyncClient> clients = new ArrayList<>();
private boolean started;
public MqttEndpointManager(MqttInboundProperties props,
MqttProfileRegistry profileRegistry,
@@ -64,111 +66,78 @@ public final class MqttEndpointManager implements AutoCloseable {
this.dispatcher = dispatcher;
}
public void start() {
running = true;
log.info("mqtt endpoint manager starting, endpoints={}", props.getEndpoints().size());
public synchronized void start() {
if (started) {
return;
}
started = true;
if (props.getEndpoints().isEmpty()) {
log.warn("mqtt inbound enabled but no endpoints configured");
return;
}
log.info("mqtt inbound starting endpoints={}", props.getEndpoints().size());
for (MqttInboundProperties.Endpoint ep : props.getEndpoints()) {
Thread worker = Thread.ofVirtual()
.name("mqtt-endpoint-" + safeName(ep.getName()))
.start(() -> runEndpoint(ep));
workers.add(worker);
try {
// 多 endpoint 互不影响:单个连接初始化失败会派发 operationalError但不会阻止其他 endpoint。
log.info("mqtt endpoint [{}] initializing uri={} topic={} profile={} qos={} cleanSession={} tlsHostnameVerification={}",
ep.getName(), ep.getUri(), ep.getTopic(), ep.getProfile(), ep.getQos(), ep.isCleanSession(),
ep.getTls() == null || ep.getTls().isHostnameVerificationEnabled());
MqttAsyncClient client = buildClient(ep);
clients.add(client);
connectAndSubscribe(client, ep);
} catch (Exception e) {
log.error("mqtt endpoint [{}] init failed, skipped", ep.getName(), e);
publishOperationalError(ep, "init", e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
}
}
}
private void runEndpoint(MqttInboundProperties.Endpoint ep) {
while (running && !Thread.currentThread().isInterrupted()) {
FutureConnection connection = null;
try {
log.info("mqtt endpoint [{}] initializing uri={} topic={} qos={} tls={}",
ep.getName(), ep.getUri(), ep.getTopic(), ep.getQos(), hasCustomTls(ep.getTls()));
MQTT mqtt = buildClient(ep);
connection = mqtt.futureConnection();
connections.add(connection);
log.info("mqtt endpoint [{}] connect sending", ep.getName());
connection.connect().await();
log.info("mqtt endpoint [{}] connected", ep.getName());
connection.subscribe(new Topic[]{new Topic(ep.getTopic(), mapQos(ep.getQos()))}).await();
log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
receiveLoop(ep, connection);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
} catch (Exception e) {
if (isPeerDisconnect(e)) {
log.warn("mqtt endpoint [{}] peer disconnected, reconnecting: {}", ep.getName(), e.getMessage());
} else {
log.error("mqtt endpoint [{}] connect/receive failed", ep.getName(), e);
publishOperationalError(ep, "connect",
private MqttAsyncClient buildClient(MqttInboundProperties.Endpoint ep) throws MqttException {
String clientId = ep.getClientId() == null || ep.getClientId().isBlank()
? "lingniu-" + UUID.randomUUID() : ep.getClientId();
return new MqttAsyncClient(pahoServerUri(ep.getUri()), clientId, new MemoryPersistence());
}
private void connectAndSubscribe(MqttAsyncClient client, MqttInboundProperties.Endpoint ep) throws MqttException {
client.setCallback(new MqttCallbackExtended() {
@Override
public void connectComplete(boolean reconnect, String serverURI) {
if (!reconnect) {
return;
}
try {
client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep));
log.info("mqtt endpoint [{}] re-subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
} catch (Exception e) {
log.error("mqtt endpoint [{}] re-subscribe failed topic={}", ep.getName(), ep.getTopic(), e);
publishOperationalError(ep, "subscribe",
e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
}
sleepBeforeReconnect(ep);
} finally {
if (connection != null) {
connections.remove(connection);
try {
connection.disconnect().await();
} catch (Exception ignored) {
// best-effort close
}
}
}
}
}
private MQTT buildClient(MqttInboundProperties.Endpoint ep) throws Exception {
MQTT mqtt = new MQTT();
mqtt.setHost(ep.getUri());
if (ep.getClientId() != null && !ep.getClientId().isBlank()) {
mqtt.setClientId(ep.getClientId());
}
mqtt.setCleanSession(ep.isCleanSession());
if (ep.getUsername() != null && !ep.getUsername().isBlank()) {
mqtt.setUserName(ep.getUsername());
mqtt.setPassword(ep.getPassword() == null ? "" : ep.getPassword());
}
mqtt.setConnectAttemptsMax(1);
mqtt.setReconnectAttemptsMax(0);
if (ep.getKeepAliveSeconds() > 0) {
mqtt.setKeepAlive((short) ep.getKeepAliveSeconds());
}
mqtt.setVersion("3.1.1");
if (hasCustomTls(ep.getTls())) {
mqtt.setSslContext(MqttTlsSupport.buildSslContext(ep.getTls()));
}
return mqtt;
}
private void receiveLoop(MqttInboundProperties.Endpoint ep, FutureConnection connection) throws Exception {
while (running && !Thread.currentThread().isInterrupted()) {
Future<Message> futureMessage = connection.receive();
Message message = futureMessage.await();
try {
message.ack();
onMessage(ep, message.getTopic(), message.getPayload());
} catch (Exception e) {
log.warn("mqtt endpoint [{}] message ack/dispatch failed topic={} len={}",
ep.getName(), message.getTopic(), message.getPayload() == null ? 0 : message.getPayload().length, e);
@Override
public void connectionLost(Throwable cause) {
log.warn("mqtt endpoint [{}] connection lost", ep.getName(), cause);
}
}
}
private boolean isPeerDisconnect(Exception e) {
Throwable current = e;
while (current != null) {
if (current instanceof EOFException) {
return true;
@Override
public void messageArrived(String topic, MqttMessage message) {
log.info("mqtt endpoint [{}] received topic={} bytes={}", ep.getName(), topic, message.getPayload().length);
onMessage(ep, topic, message.getPayload());
}
current = current.getCause();
}
return false;
}
private void sleepBeforeReconnect(MqttInboundProperties.Endpoint ep) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@Override
public void deliveryComplete(IMqttDeliveryToken token) {
// inbound only
}
});
MqttConnectOptions options = connectOptions(ep);
client.connect(options).waitForCompletion(timeoutMs(ep));
log.info("mqtt endpoint [{}] connected serverUri={} clientId={}",
ep.getName(), client.getServerURI(), client.getClientId());
client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep));
log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
}
private void onMessage(MqttInboundProperties.Endpoint ep, String topic, byte[] payload) {
@@ -319,11 +288,58 @@ public final class MqttEndpointManager implements AutoCloseable {
return value.replace("\\", "\\\\").replace("\"", "\\\"");
}
private static QoS mapQos(int level) {
private static MqttConnectOptions connectOptions(MqttInboundProperties.Endpoint ep) {
MqttConnectOptions options = new MqttConnectOptions();
options.setAutomaticReconnect(true);
options.setCleanSession(ep.isCleanSession());
if (ep.getKeepAliveSeconds() > 0) {
options.setKeepAliveInterval(ep.getKeepAliveSeconds());
}
if (ep.getConnectionTimeoutSeconds() > 0) {
options.setConnectionTimeout(ep.getConnectionTimeoutSeconds());
}
if (hasText(ep.getUsername())) {
options.setUserName(ep.getUsername());
options.setPassword(ep.getPassword() == null ? new char[0] : ep.getPassword().toCharArray());
}
if (isSslUri(ep.getUri()) && hasCustomTls(ep.getTls())) {
options.setSocketFactory(MqttTlsSupport.socketFactory(ep.getTls()));
if (ep.getTls() != null && !ep.getTls().isHostnameVerificationEnabled()) {
options.setSSLHostnameVerifier((hostname, session) -> true);
}
}
return options;
}
private static String pahoServerUri(String value) {
URI uri = URI.create(value);
String scheme = uri.getScheme();
String pahoScheme = switch (scheme == null ? "" : scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> "ssl";
default -> "tcp";
};
int port = uri.getPort() > 0 ? uri.getPort() : defaultPort(scheme);
return pahoScheme + "://" + uri.getHost() + ":" + port;
}
private static int mapQos(int level) {
return switch (level) {
case 0 -> QoS.AT_MOST_ONCE;
case 2 -> QoS.EXACTLY_ONCE;
default -> QoS.AT_LEAST_ONCE;
case 0 -> 0;
case 2 -> 2;
default -> 1;
};
}
private static long timeoutMs(MqttInboundProperties.Endpoint ep) {
int seconds = ep.getConnectionTimeoutSeconds() > 0 ? ep.getConnectionTimeoutSeconds() : 10;
return TimeUnit.SECONDS.toMillis(seconds);
}
private static int defaultPort(String scheme) {
if (scheme == null) return 1883;
return switch (scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> 8883;
default -> 1883;
};
}
@@ -331,13 +347,6 @@ public final class MqttEndpointManager implements AutoCloseable {
return value != null && value.trim().length() == 17;
}
private static String safeName(String value) {
if (value == null || value.isBlank()) {
return "unnamed";
}
return value.replaceAll("[^a-zA-Z0-9_.-]", "_");
}
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}
private static boolean hasCustomTls(MqttInboundProperties.Tls tls) {
@@ -345,23 +354,30 @@ public final class MqttEndpointManager implements AutoCloseable {
|| (hasText(tls.getClientPem()) && hasText(tls.getClientKey())));
}
private static boolean isSslUri(String value) {
String scheme = URI.create(value).getScheme();
return scheme != null && switch (scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> true;
default -> false;
};
}
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
@Override
public void close() {
running = false;
for (FutureConnection connection : List.copyOf(connections)) {
for (MqttAsyncClient c : clients) {
try {
connection.disconnect().await();
if (c.isConnected()) {
c.disconnectForcibly(1000, 1000, false);
}
c.close(true);
} catch (Exception ignored) {
// best-effort
}
}
for (Thread worker : workers) {
worker.interrupt();
}
log.info("mqtt endpoint manager stopped, connections={}", connections.size());
log.info("mqtt endpoint manager stopped, clients={}", clients.size());
}
}

View File

@@ -2,19 +2,31 @@ package com.lingniu.ingest.inbound.mqtt.client;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import java.io.InputStream;
import javax.net.ssl.X509ExtendedTrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyFactory;
import java.security.KeyStore;
import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.spec.PKCS8EncodedKeySpec;
import java.util.ArrayList;
import java.util.Base64;
@@ -25,93 +37,81 @@ public final class MqttTlsSupport {
private MqttTlsSupport() {}
public static SSLContext buildSslContext(MqttInboundProperties.Tls tls) {
public static SSLSocketFactory socketFactory(MqttInboundProperties.Tls tls) {
try {
TrustManagerFactory tmf = null;
KeyManagerFactory kmf = null;
KeyManager[] keyManagers = null;
TrustManager[] trustManagers = null;
if (tls != null && hasText(tls.getCaPem())) {
tmf = trustManagerFactory(Path.of(tls.getCaPem()));
trustManagers = trustManagerFactory(Path.of(tls.getCaPem())).getTrustManagers();
if (!tls.isHostnameVerificationEnabled()) {
trustManagers = withoutHostnameVerification(trustManagers);
}
}
if (tls != null && hasText(tls.getClientPem()) && hasText(tls.getClientKey())) {
kmf = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey()));
keyManagers = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey())).getKeyManagers();
}
SSLContext context = SSLContext.getInstance("TLSv1.2");
context.init(
kmf == null ? null : kmf.getKeyManagers(),
tmf == null ? null : tmf.getTrustManagers(),
new SecureRandom());
return context;
SSLContext context = SSLContext.getInstance("TLS");
context.init(keyManagers, trustManagers, null);
SSLSocketFactory factory = context.getSocketFactory();
if (tls != null && !tls.isHostnameVerificationEnabled()) {
return new HostnameVerificationDisablingSocketFactory(factory);
}
return factory;
} catch (Exception e) {
throw new IllegalStateException("mqtt tls config build failed", e);
}
}
private static TrustManagerFactory trustManagerFactory(Path caPath) throws Exception {
private static TrustManagerFactory trustManagerFactory(Path caPem) throws Exception {
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(null, null);
int i = 0;
for (Certificate cert : readCertificates(caPath)) {
for (Certificate cert : readCertificates(caPem)) {
trustStore.setCertificateEntry("ca-" + i++, cert);
}
TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore);
return tmf;
}
private static KeyManagerFactory keyManagerFactory(Path clientCertPath, Path clientKeyPath) throws Exception {
List<Certificate> chain = readCertificates(clientCertPath);
PrivateKey key = readPrivateKey(clientKeyPath);
private static KeyManagerFactory keyManagerFactory(Path clientPem, Path clientKey) throws Exception {
List<Certificate> chain = readCertificates(clientPem);
PrivateKey key = readPrivateKey(clientKey);
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
keyStore.load(null, null);
char[] password = new char[0];
keyStore.setCertificateEntry("client", chain.getFirst());
keyStore.setKeyEntry("clientKey", key, password, chain.toArray(Certificate[]::new));
KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX");
keyStore.setKeyEntry("mqtt-client", key, password, chain.toArray(Certificate[]::new));
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
kmf.init(keyStore, password);
return kmf;
}
private static List<Certificate> readCertificates(Path path) throws Exception {
private static List<Certificate> readCertificates(Path pem) throws Exception {
String text = Files.readString(pem, StandardCharsets.UTF_8);
List<Certificate> certificates = new ArrayList<>();
CertificateFactory factory = CertificateFactory.getInstance("X.509");
String text = readUtf8IfPossible(path);
if (text != null && text.contains("-----BEGIN CERTIFICATE-----")) {
List<Certificate> certificates = new ArrayList<>();
for (String block : pemBlocks(text, "CERTIFICATE")) {
certificates.add(factory.generateCertificate(
new java.io.ByteArrayInputStream(Base64.getMimeDecoder().decode(block))));
}
if (!certificates.isEmpty()) {
return certificates;
for (String block : pemBlocks(text, "CERTIFICATE")) {
try (ByteArrayInputStream in = new ByteArrayInputStream(Base64.getMimeDecoder().decode(block))) {
certificates.add(factory.generateCertificate(in));
}
}
try (InputStream in = Files.newInputStream(path)) {
Collection<? extends Certificate> certificates = factory.generateCertificates(in);
if (certificates.isEmpty()) {
throw new IllegalArgumentException("no certificate found in " + path);
}
return List.copyOf(certificates);
if (certificates.isEmpty()) {
throw new IllegalArgumentException("no certificate found in " + pem);
}
return certificates;
}
private static PrivateKey readPrivateKey(Path path) throws Exception {
String text = Files.readString(path, StandardCharsets.UTF_8);
private static PrivateKey readPrivateKey(Path pem) throws Exception {
String text = Files.readString(pem, StandardCharsets.UTF_8);
List<String> blocks = pemBlocks(text, "PRIVATE KEY");
if (blocks.isEmpty()) {
throw new IllegalArgumentException("no PKCS#8 private key found in " + path);
throw new IllegalArgumentException("no PKCS#8 private key found in " + pem);
}
byte[] der = Base64.getMimeDecoder().decode(blocks.getFirst());
PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der);
return KeyFactory.getInstance("RSA").generatePrivate(spec);
}
private static String readUtf8IfPossible(Path path) {
try {
return Files.readString(path, StandardCharsets.UTF_8);
} catch (Exception ignored) {
return null;
}
}
private static List<String> pemBlocks(String text, String type) {
String begin = "-----BEGIN " + type + "-----";
String end = "-----END " + type + "-----";
@@ -132,4 +132,129 @@ public final class MqttTlsSupport {
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
private static TrustManager[] withoutHostnameVerification(TrustManager[] managers) {
TrustManager[] wrapped = new TrustManager[managers.length];
for (int i = 0; i < managers.length; i++) {
TrustManager manager = managers[i];
if (manager instanceof X509TrustManager x509) {
wrapped[i] = new ChainOnlyTrustManager(x509);
} else {
wrapped[i] = manager;
}
}
return wrapped;
}
private static final class ChainOnlyTrustManager extends X509ExtendedTrustManager {
private final X509TrustManager delegate;
private ChainOnlyTrustManager(X509TrustManager delegate) {
this.delegate = delegate;
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket)
throws CertificateException {
if (delegate instanceof X509ExtendedTrustManager extended) {
extended.checkClientTrusted(chain, authType, socket);
} else {
delegate.checkClientTrusted(chain, authType);
}
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket)
throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
throws CertificateException {
if (delegate instanceof X509ExtendedTrustManager extended) {
extended.checkClientTrusted(chain, authType, engine);
} else {
delegate.checkClientTrusted(chain, authType);
}
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
delegate.checkClientTrusted(chain, authType);
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return delegate.getAcceptedIssuers();
}
}
private static final class HostnameVerificationDisablingSocketFactory extends SSLSocketFactory {
private final SSLSocketFactory delegate;
private HostnameVerificationDisablingSocketFactory(SSLSocketFactory delegate) {
this.delegate = delegate;
}
@Override
public String[] getDefaultCipherSuites() {
return delegate.getDefaultCipherSuites();
}
@Override
public String[] getSupportedCipherSuites() {
return delegate.getSupportedCipherSuites();
}
@Override
public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException {
return prepare(delegate.createSocket(socket, host, port, autoClose));
}
@Override
public Socket createSocket() throws IOException {
return prepare(delegate.createSocket());
}
@Override
public Socket createSocket(String host, int port) throws IOException {
return prepare(delegate.createSocket(host, port));
}
@Override
public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException {
return prepare(delegate.createSocket(host, port, localHost, localPort));
}
@Override
public Socket createSocket(InetAddress host, int port) throws IOException {
return prepare(delegate.createSocket(host, port));
}
@Override
public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException {
return prepare(delegate.createSocket(address, port, localAddress, localPort));
}
private static Socket prepare(Socket socket) {
if (socket instanceof SSLSocket sslSocket) {
SSLParameters parameters = sslSocket.getSSLParameters();
parameters.setEndpointIdentificationAlgorithm(null);
sslSocket.setSSLParameters(parameters);
}
return socket;
}
}
}

View File

@@ -10,19 +10,20 @@ import com.lingniu.ingest.inbound.mqtt.parser.MqttPayloadParser;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfile;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import com.lingniu.ingest.inbound.mqtt.profile.YutongMqttProfile;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import java.util.ArrayList;
import java.util.List;
@AutoConfiguration
@EnableConfigurationProperties(MqttInboundProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.mqtt", name = "enabled", havingValue = "true")
@Conditional(MqttInboundEnabledCondition.class)
public class MqttInboundAutoConfiguration {
@Bean
@@ -61,25 +62,144 @@ public class MqttInboundAutoConfiguration {
return new MqttRealtimeHandler(profileRegistry);
}
@Bean(destroyMethod = "close")
@Bean(initMethod = "start", destroyMethod = "close")
@Lazy(false)
@ConditionalOnMissingBean
public MqttEndpointManager mqttEndpointManager(MqttInboundProperties props,
MqttProfileRegistry profileRegistry,
VehicleIdentityResolver identityResolver,
Dispatcher dispatcher) {
Dispatcher dispatcher,
Environment environment) {
applyLegacySingleEndpointFallback(props, environment);
return new MqttEndpointManager(props, profileRegistry, identityResolver, dispatcher);
}
/**
* Spring Boot 启动完成后触发 MQTT 连接,避免与 Netty / Kafka 启动竞争。
*/
@Bean
public ApplicationRunner mqttStartupRunner(MqttEndpointManager manager) {
return new ApplicationRunner() {
@Override
public void run(ApplicationArguments args) {
manager.start();
private static void applyLegacySingleEndpointFallback(MqttInboundProperties props, Environment env) {
if (!props.getEndpoints().isEmpty()) {
return;
}
String uri = firstText(env,
"lingniu.ingest.mqtt.endpoints[0].uri",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_URI",
"MQTT_URI",
"YUTONG_MQTT_URI");
if (uri.isBlank() || uri.contains("${")) {
return;
}
MqttInboundProperties.Endpoint endpoint = new MqttInboundProperties.Endpoint();
endpoint.setName(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].name",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_NAME",
"MQTT_ENDPOINT_NAME",
"YUTONG_MQTT_ENDPOINT_NAME"), "yutong"));
endpoint.setUri(uri);
endpoint.setTopic(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].topic",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TOPIC",
"MQTT_TOPIC",
"YUTONG_MQTT_TOPIC"), "#"));
endpoint.setQos(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].qos",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_QOS",
"MQTT_QOS",
"YUTONG_MQTT_QOS"), 1));
endpoint.setClientId(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].client-id",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLIENT_ID",
"MQTT_CLIENT_ID",
"YUTONG_MQTT_CLIENT_ID"));
endpoint.setUsername(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].username",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_USERNAME",
"MQTT_USERNAME",
"YUTONG_MQTT_USERNAME"));
endpoint.setPassword(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].password",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_PASSWORD",
"MQTT_PASSWORD",
"YUTONG_MQTT_PASSWORD"));
endpoint.setCleanSession(booleanValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].clean-session",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLEAN_SESSION",
"MQTT_CLEAN_SESSION",
"YUTONG_MQTT_CLEAN_SESSION"), false));
endpoint.setKeepAliveSeconds(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].keep-alive-seconds",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_KEEP_ALIVE_SECONDS",
"MQTT_KEEP_ALIVE_SECONDS",
"YUTONG_MQTT_KEEP_ALIVE_SECONDS"), 20));
endpoint.setConnectionTimeoutSeconds(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].connection-timeout-seconds",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CONNECTION_TIMEOUT_SECONDS",
"MQTT_CONNECTION_TIMEOUT_SECONDS",
"YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS"), 10));
endpoint.setProfile(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].profile",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_PROFILE",
"MQTT_PROFILE",
"YUTONG_MQTT_PROFILE"), "yutong"));
MqttInboundProperties.Tls tls = endpoint.getTls();
tls.setCaPem(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.ca-pem",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CA_PEM",
"MQTT_TLS_CA_PEM",
"YUTONG_MQTT_TLS_CA_PEM"));
tls.setClientPem(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.client-pem",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_PEM",
"MQTT_TLS_CLIENT_PEM",
"YUTONG_MQTT_TLS_CLIENT_PEM"));
tls.setClientKey(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.client-key",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_KEY",
"MQTT_TLS_CLIENT_KEY",
"YUTONG_MQTT_TLS_CLIENT_KEY"));
tls.setHostnameVerificationEnabled(booleanValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.hostname-verification-enabled",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_HOSTNAME_VERIFICATION_ENABLED",
"MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED",
"YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED"), true));
props.setEndpoints(new ArrayList<>(List.of(endpoint)));
}
private static String firstText(Environment env, String... keys) {
for (String key : keys) {
String value = env.getProperty(key);
if (value != null && !value.isBlank() && !value.contains("${")) {
return value;
}
};
}
return "";
}
private static String defaultText(String value, String fallback) {
return value == null || value.isBlank() ? fallback : value;
}
private static int intValue(String value, int fallback) {
if (value == null || value.isBlank()) {
return fallback;
}
try {
return Integer.parseInt(value.trim());
} catch (NumberFormatException ex) {
return fallback;
}
}
private static boolean booleanValue(String value, boolean fallback) {
if (value == null || value.isBlank()) {
return fallback;
}
String normalized = value.trim().toLowerCase();
if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized)
|| "on".equals(normalized)) {
return true;
}
if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized)
|| "off".equals(normalized)) {
return false;
}
return fallback;
}
}

View File

@@ -0,0 +1,41 @@
package com.lingniu.ingest.inbound.mqtt.config;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.env.Environment;
import org.springframework.core.type.AnnotatedTypeMetadata;
final class MqttInboundEnabledCondition implements Condition {
@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
Environment env = context.getEnvironment();
Boolean direct = parseBoolean(env.getProperty("lingniu.ingest.mqtt.enabled"));
if (direct != null) {
return direct;
}
return isTrue(env.getProperty("MQTT_ENABLED"))
|| isTrue(env.getProperty("YUTONG_MQTT_ENABLED"));
}
private static boolean isTrue(String value) {
Boolean parsed = parseBoolean(value);
return parsed != null && parsed;
}
private static Boolean parseBoolean(String value) {
if (value == null || value.isBlank() || value.contains("${")) {
return null;
}
String normalized = value.trim().toLowerCase();
if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized)
|| "on".equals(normalized)) {
return true;
}
if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized)
|| "off".equals(normalized)) {
return false;
}
return null;
}
}

View File

@@ -82,6 +82,8 @@ public class MqttInboundProperties {
public String getClientKey() { return clientKey; }
public void setClientKey(String clientKey) { this.clientKey = clientKey; }
public boolean isHostnameVerificationEnabled() { return hostnameVerificationEnabled; }
public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) { this.hostnameVerificationEnabled = hostnameVerificationEnabled; }
public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) {
this.hostnameVerificationEnabled = hostnameVerificationEnabled;
}
}
}

View File

@@ -172,7 +172,7 @@ class MqttEndpointManagerTest {
.containsEntry("endpoint", "bad-endpoint")
.containsEntry("profile", "yutong")
.containsEntry("operationalError", "true")
.containsEntry("phase", "connect")
.containsEntry("phase", "init")
.containsEntry("identityResolved", "false")
.containsEntry("identitySource", "UNKNOWN");
});
@@ -184,7 +184,7 @@ class MqttEndpointManagerTest {
.containsEntry("operationalError", "true")
.containsEntry("endpoint", "bad-endpoint")
.containsEntry("profile", "yutong")
.containsEntry("phase", "connect");
.containsEntry("phase", "init");
});
batchExecutor.close();

View File

@@ -5,11 +5,10 @@ import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import javax.net.ssl.SSLSocketFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.net.ssl.SSLContext;
import static org.assertj.core.api.Assertions.assertThat;
class MqttTlsSupportTest {
@@ -24,9 +23,9 @@ class MqttTlsSupportTest {
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
tls.setCaPem(ca.toString());
SSLContext config = MqttTlsSupport.buildSslContext(tls);
SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
assertThat(factory).isNotNull();
}
@Test
@@ -42,22 +41,9 @@ class MqttTlsSupportTest {
tls.setClientPem(cert.toString());
tls.setClientKey(key.toString());
SSLContext config = MqttTlsSupport.buildSslContext(tls);
SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
}
@Test
void hostnameVerificationFlagDoesNotBlockSslContextCreation() throws Exception {
Path ca = tempDir.resolve("ca.pem");
Files.writeString(ca, CERT);
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
tls.setCaPem(ca.toString());
tls.setHostnameVerificationEnabled(false);
SSLContext config = MqttTlsSupport.buildSslContext(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
assertThat(factory).isNotNull();
}
private static final String CERT = """

View File

@@ -41,6 +41,30 @@
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>

View File

@@ -6,6 +6,7 @@ import com.gps31.push.netty.PushClient;
import com.gps31.push.netty.PushMsg;
import com.gps31.push.netty.client.TcpClient;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.pipeline.RawFrame;
import com.lingniu.ingest.core.dispatcher.Dispatcher;
import com.lingniu.ingest.identity.InMemoryVehicleIdentityService;
@@ -220,6 +221,7 @@ public class XindaPushClient extends PushClient {
meta.put("parseError", "true");
meta.put("parseErrorMessage", parsed.errorMessage());
}
meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(payload.command(), json, parsed));
dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH,
payload.commandCode(),
@@ -247,6 +249,8 @@ public class XindaPushClient extends PushClient {
meta.put("operationalError", "true");
meta.put("phase", phase == null ? "" : phase);
meta.put("reason", reason == null ? "" : reason);
meta.put(RawArchiveKeys.META_PARSED_JSON,
parsedJson(payload.command(), payload.json(), new JsonParseResult(json, false, "")));
dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH,
payload.commandCode(),
@@ -271,6 +275,8 @@ public class XindaPushClient extends PushClient {
meta.put("operationalError", "true");
meta.put("phase", "message");
meta.put("reason", reason == null ? "" : reason);
JsonParseResult parsed = parseForMetadata(json);
meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(cmd, json, parsed));
dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH,
0,
@@ -388,6 +394,19 @@ public class XindaPushClient extends PushClient {
}
}
private static String parsedJson(String command, String rawJson, JsonParseResult parsed) {
JSONObject root = new JSONObject(true);
root.put("command", command == null ? "" : command);
root.put("parseError", parsed != null && parsed.parseError());
if (parsed != null && parsed.parseError()) {
root.put("parseErrorMessage", parsed.errorMessage() == null ? "" : parsed.errorMessage());
root.put("rawJson", rawJson == null ? "" : rawJson);
} else {
root.put("body", parsed == null || parsed.json() == null ? new JSONObject() : parsed.json());
}
return root.toJSONString();
}
private record JsonParseResult(JSONObject json, boolean parseError, String errorMessage) {}
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}

View File

@@ -185,7 +185,6 @@ public final class XindaPushEventMapper implements EventMapper<XindaPushMessage>
Map<String, String> out = new java.util.LinkedHashMap<>();
out.put("source", "xinda-push");
out.put("cmd", message.command());
out.put("rawJson", message.json());
out.put("vin", normalizedVin(identity));
out.put("externalVin", firstNonBlank(json, "vin"));
out.put("phone", phone(json));

View File

@@ -37,7 +37,8 @@ class XindaPushEventMapperTest {
.containsEntry("externalVin", "LNVIN000000000101")
.containsEntry("phone", "13800138000")
.containsEntry("deviceId", "XD-101")
.containsEntry("plateNo", "粤B10101");
.containsEntry("plateNo", "粤B10101")
.doesNotContainKey("rawJson");
}
@Test

View File

@@ -43,7 +43,7 @@ class Gb32960ChannelHandlerAckBoundaryTest {
dispatch.awaitPublishCount(1);
dispatch.completeDurability();
ByteBuf ack = readOutboundAfterRunningTasks(channel);
ByteBuf ack = awaitOutbound(channel);
assertThat(ack).isNotNull();
assertThat(ack.getUnsignedByte(2)).isEqualTo((short) 0x02);
assertThat(ack.getUnsignedByte(3)).isEqualTo((short) 0x01);
@@ -85,8 +85,8 @@ class Gb32960ChannelHandlerAckBoundaryTest {
dispatch.completeDurability(0);
ByteBuf firstAck = readOutboundAfterRunningTasks(channel);
ByteBuf secondAck = readOutboundAfterRunningTasks(channel);
ByteBuf firstAck = awaitOutbound(channel);
ByteBuf secondAck = awaitOutbound(channel);
assertThat(firstAck).isNotNull();
assertThat(secondAck).isNotNull();
assertThat(firstAck.getUnsignedByte(2)).isEqualTo((short) 0x02);
@@ -97,42 +97,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
}
}
@Test
void decodeFailureIsArchivedWithParseErrorMetadata() {
try (DispatchHarness dispatch = new DispatchHarness()) {
EmbeddedChannel channel = new EmbeddedChannel(handlerWith(dispatch.dispatcher()));
byte[] malformed = new byte[]{0x23, 0x23, 0x02};
assertThat(channel.writeInbound(malformed)).isFalse();
dispatch.awaitPublishCount(1);
VehicleEvent event = dispatch.event(0);
assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class);
VehicleEvent.RawArchive raw = (VehicleEvent.RawArchive) event;
assertThat(raw.source()).isEqualTo(com.lingniu.ingest.api.ProtocolId.GB32960);
assertThat(raw.command()).isZero();
assertThat(raw.rawBytes()).containsExactly(malformed);
assertThat(raw.metadata())
.containsEntry("vin", "unknown")
.containsEntry("parseError", "true")
.containsKey("parseErrorMessage");
channel.finishAndReleaseAll();
}
}
private static ByteBuf readOutboundAfterRunningTasks(EmbeddedChannel channel) {
long deadline = System.currentTimeMillis() + 3000;
while (System.currentTimeMillis() < deadline) {
channel.runPendingTasks();
ByteBuf outbound = channel.readOutbound();
if (outbound != null) {
return outbound;
}
Thread.onSpinWait();
}
return null;
}
private static Gb32960ChannelHandler handlerWith(Dispatcher dispatcher) {
return new Gb32960ChannelHandler(
decoder(),
@@ -174,6 +138,19 @@ class Gb32960ChannelHandlerAckBoundaryTest {
return out.toByteArray();
}
private static ByteBuf awaitOutbound(EmbeddedChannel channel) {
long deadline = System.currentTimeMillis() + 3000;
while (System.currentTimeMillis() < deadline) {
channel.runPendingTasks();
ByteBuf outbound = channel.readOutbound();
if (outbound != null) {
return outbound;
}
Thread.onSpinWait();
}
return null;
}
private static final class DispatchHarness implements AutoCloseable {
private final ControlledSink sink = new ControlledSink();
private final DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
@@ -214,12 +191,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
throw new AssertionError("expected " + count + " sink publishes, actual=" + sink.futures.size());
}
VehicleEvent event(int index) {
synchronized (sink.futures) {
return sink.events.get(index);
}
}
@Override
public void close() {
batchExecutor.close();
@@ -229,7 +200,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
private static final class ControlledSink implements EventSink {
private final List<CompletableFuture<Void>> futures = new ArrayList<>();
private final List<VehicleEvent> events = new ArrayList<>();
@Override
public String name() {
@@ -240,7 +210,6 @@ class Gb32960ChannelHandlerAckBoundaryTest {
public CompletableFuture<Void> publish(VehicleEvent event) {
CompletableFuture<Void> future = new CompletableFuture<>();
synchronized (futures) {
events.add(event);
futures.add(future);
}
return future;

View File

@@ -9,13 +9,14 @@ import com.lingniu.ingest.identity.VehicleIdentityLookup;
import com.lingniu.ingest.identity.VehicleIdentityRegistry;
import com.lingniu.ingest.identity.VehicleIdentityResolver;
import com.lingniu.ingest.identity.VehicleIdentitySource;
import com.lingniu.ingest.identity.VehicleRegistrationBinding;
import com.lingniu.ingest.protocol.jt808.codec.Jt808FrameEncoder;
import com.lingniu.ingest.protocol.jt808.codec.Jt808MalformedFrame;
import com.lingniu.ingest.protocol.jt808.codec.Jt808MessageDecoder;
import com.lingniu.ingest.protocol.jt808.downlink.Jt808Commands;
import com.lingniu.ingest.protocol.jt808.model.Jt808Body;
import com.lingniu.ingest.protocol.jt808.model.Jt808Header;
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationAdditionalInfo;
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationFlagInfo;
import com.lingniu.ingest.protocol.jt808.model.Jt808Message;
import com.lingniu.ingest.protocol.jt808.model.Jt808MessageId;
import com.lingniu.ingest.protocol.jt808.session.Jt808ChannelRegistry;
@@ -25,17 +26,18 @@ import com.lingniu.ingest.session.SessionStore;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.util.AttributeKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
/**
* JT808 Netty 入站处理器。职责:
@@ -51,8 +53,6 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
private static final Logger log = LoggerFactory.getLogger(Jt808ChannelHandler.class);
private static final String PROCESSING_ERROR_PREFIX = "processing failed: ";
private static final AttributeKey<CompletableFuture<Void>> DURABLE_ACK_CHAIN_ATTR =
AttributeKey.valueOf("jt808.durableAckChain");
private final Jt808MessageDecoder decoder;
private final Dispatcher dispatcher;
@@ -123,21 +123,29 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
// ====== 会话维护 & 同步应答匹配 ======
String phone = msg.header().phone();
IdentityResolution identity = resolveIdentity(msg);
byte[] durableAck = null;
if (msg.body() instanceof Jt808Body.Register reg) {
identity = registerIdentity(msg, reg, identity);
}
byte[] registerAck = null;
switch (msg.header().messageId()) {
case Jt808MessageId.TERMINAL_REGISTER -> {
DeviceSession session = upsertSession(ctx, msg, identity.identity());
channelRegistry.bind(phone, ctx.channel());
durableAck = registerAckFrame(phone, msg.header().serialNo(), session.token());
var command = Jt808Commands.registerAck(msg.header().serialNo(), 0, session.token());
registerAck = Jt808FrameEncoder.encode(
command.messageId(),
phone,
channelRegistry.nextSerial(phone),
command.body());
}
case Jt808MessageId.TERMINAL_AUTH -> {
upsertSession(ctx, msg, identity.identity());
channelRegistry.bind(phone, ctx.channel());
durableAck = terminalAckFrame(phone, msg.header().serialNo(), msg.header().messageId(), 0);
ackTerminal(ctx, phone, msg.header().serialNo(), msg.header().messageId(), 0);
}
case Jt808MessageId.TERMINAL_HEARTBEAT -> {
channelRegistry.bind(phone, ctx.channel());
durableAck = terminalAckFrame(phone, msg.header().serialNo(), msg.header().messageId(), 0);
ackTerminal(ctx, phone, msg.header().serialNo(), msg.header().messageId(), 0);
}
case Jt808MessageId.TERMINAL_GENERAL_RESPONSE -> {
// body: ackSerial(2) + ackMsgId(2) + result(1)
@@ -153,7 +161,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
}
// ====== 派发到 Dispatcher ======
Map<String, String> meta = new HashMap<>(6);
Map<String, String> meta = new HashMap<>(32);
meta.put("vin", identity.identity().vin());
meta.put("phone", phone);
meta.put("identityResolved", Boolean.toString(identity.identity().resolved()));
@@ -163,8 +171,11 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
meta.put("identityErrorMessage", identity.errorMessage());
}
meta.put("seq", Integer.toString(msg.header().serialNo()));
meta.put("messageId", "0x" + Jt808MessageId.hex(msg.header().messageId()));
meta.put("messageName", Jt808MessageId.nameOf(msg.header().messageId()));
meta.put("reportType", Jt808MessageId.reportTypeOf(msg.header().messageId()));
putBodyMetadata(meta, msg.body());
meta.put("peer", addr(ctx));
addRegistrationMetadata(meta, msg.body());
RawFrame rf = new RawFrame(
ProtocolId.JT808,
msg.header().messageId(),
@@ -173,63 +184,19 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
frame,
meta,
Instant.now());
if (durableAck == null) {
dispatcher.dispatch(rf);
if (registerAck != null) {
byte[] ack = registerAck;
dispatcher.dispatchAndAwait(rf)
.thenRun(() -> ctx.writeAndFlush(Unpooled.wrappedBuffer(ack)))
.exceptionally(ex -> {
log.warn("[jt808] register ack withheld because dispatch durability failed peer={} phone={} seq={} error={}",
addr(ctx), phone, msg.header().serialNo(),
ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
return null;
});
return;
}
enqueueDurableAck(ctx, phone, msg.header().messageId(), dispatcher.dispatchAndAwait(rf), durableAck);
}
private void enqueueDurableAck(ChannelHandlerContext ctx,
String phone,
int messageId,
CompletableFuture<Void> dispatchFuture,
byte[] ackFrame) {
CompletableFuture<Void> previous = ctx.channel().attr(DURABLE_ACK_CHAIN_ATTR).get();
if (previous == null) {
previous = CompletableFuture.completedFuture(null);
}
CompletableFuture<Void> next = previous
.handle((ignored, previousFailure) -> null)
.thenCompose(ignored -> dispatchFuture.handle((ok, failure) -> failure))
.thenCompose(failure -> runOnEventLoop(ctx,
() -> handleDurableAckResult(ctx, phone, messageId, failure, ackFrame)));
ctx.channel().attr(DURABLE_ACK_CHAIN_ATTR).set(next);
}
private CompletableFuture<Void> runOnEventLoop(ChannelHandlerContext ctx, Runnable task) {
CompletableFuture<Void> done = new CompletableFuture<>();
Runnable wrapped = () -> {
try {
task.run();
done.complete(null);
} catch (Throwable t) {
done.completeExceptionally(t);
}
};
if (ctx.executor().inEventLoop()) {
wrapped.run();
} else {
ctx.executor().execute(wrapped);
}
return done;
}
private void handleDurableAckResult(ChannelHandlerContext ctx,
String phone,
int messageId,
Throwable failure,
byte[] ackFrame) {
if (!ctx.channel().isActive()) {
return;
}
if (failure != null) {
log.warn("[jt808] dispatch failed before ack peer={} phone={} msgId=0x{}",
addr(ctx), phone, Integer.toHexString(messageId), failure);
ctx.close();
return;
}
ctx.writeAndFlush(Unpooled.wrappedBuffer(ackFrame));
dispatcher.dispatch(rf);
}
private IdentityResolution resolveIdentity(Jt808Message msg) {
@@ -267,6 +234,28 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
return new VehicleIdentity("unknown", false, identity.source());
}
private IdentityResolution registerIdentity(Jt808Message msg,
Jt808Body.Register reg,
IdentityResolution current) {
VehicleIdentity currentIdentity = current.identity();
boolean alreadyResolved = currentIdentity != null && currentIdentity.resolved();
String vin = alreadyResolved ? currentIdentity.vin() : generatedInternalVin(msg.header().phone(), reg);
identityRegistry.bind(new VehicleIdentityBinding(
ProtocolId.JT808,
vin,
msg.header().phone(),
reg.deviceId(),
reg.plate(),
Map.of(
"province", Integer.toString(reg.province()),
"city", Integer.toString(reg.city()),
"maker", reg.maker(),
"deviceType", reg.deviceType(),
"plateColor", Integer.toString(reg.plateColor()))));
VehicleIdentitySource source = alreadyResolved ? currentIdentity.source() : VehicleIdentitySource.REGISTERED;
return new IdentityResolution(new VehicleIdentity(vin, true, source), current.errorMessage());
}
private void dispatchFrameError(ChannelHandlerContext ctx, Jt808MalformedFrame malformed) {
String peer = malformed.peer() == null || malformed.peer().isBlank() ? addr(ctx) : malformed.peer();
dispatchMalformed(peer, malformed.rawBytes(), malformed.reason(), true);
@@ -283,7 +272,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
new Jt808Header(0, frame == null ? 0 : frame.length, 0, false,
Jt808Header.ProtocolVersion.V2013, "unknown", 0, 0, 0),
new Jt808Body.Malformed(frame == null ? new byte[0] : frame, peer, message));
Map<String, String> meta = new HashMap<>(4);
Map<String, String> meta = new HashMap<>(8);
meta.put("vin", "unknown");
meta.put("peer", peer);
meta.put("identityResolved", "false");
@@ -295,6 +284,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
meta.put("parseError", "true");
meta.put("parseErrorMessage", message);
}
putBodyMetadata(meta, malformed.body());
dispatcher.dispatch(new RawFrame(
ProtocolId.JT808,
0,
@@ -312,15 +302,19 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
msg.header().version(), msg.header().phone(), msg.header().serialNo(), 0, 0),
new Jt808Body.Malformed(frame == null ? new byte[0] : frame, addr(ctx),
PROCESSING_ERROR_PREFIX + message));
Map<String, String> meta = new HashMap<>(8);
Map<String, String> meta = new HashMap<>(16);
meta.put("vin", "unknown");
meta.put("phone", msg.header().phone());
meta.put("peer", addr(ctx));
meta.put("seq", Integer.toString(msg.header().serialNo()));
meta.put("messageId", "0x" + Jt808MessageId.hex(msg.header().messageId()));
meta.put("messageName", Jt808MessageId.nameOf(msg.header().messageId()));
meta.put("reportType", Jt808MessageId.reportTypeOf(msg.header().messageId()));
meta.put("identityResolved", "false");
meta.put("identitySource", VehicleIdentitySource.UNKNOWN.name());
meta.put("processingError", "true");
meta.put("processingErrorMessage", message);
putBodyMetadata(meta, failed.body());
dispatcher.dispatch(new RawFrame(
ProtocolId.JT808,
0,
@@ -343,25 +337,6 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
Instant.now(), Instant.now(),
Map.of("protocolVersion", msg.header().version().name())));
session = session.withVin(identity.vin());
if (msg.body() instanceof Jt808Body.Register reg) {
identityRegistry.register(new VehicleRegistrationBinding(
ProtocolId.JT808,
session.vin(),
phone,
reg.deviceId(),
reg.plate(),
reg.province(),
reg.city(),
reg.maker(),
reg.deviceType(),
reg.plateColor()));
if (identity.resolved() && session.vin() != null
&& !session.vin().isBlank()
&& !"unknown".equalsIgnoreCase(session.vin())) {
identityRegistry.bind(new VehicleIdentityBinding(
ProtocolId.JT808, session.vin(), phone, reg.deviceId(), reg.plate()));
}
}
sessions.put(session);
return session;
}
@@ -380,37 +355,175 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler<Object> {
return body instanceof Jt808Body.Register reg ? reg.plate() : "";
}
private static void addRegistrationMetadata(Map<String, String> meta, Jt808Body body) {
if (!(body instanceof Jt808Body.Register reg)) {
return;
private static void putBodyMetadata(Map<String, String> meta, Jt808Body body) {
switch (body) {
case Jt808Body.Register reg -> {
put(meta, "body.province", reg.province());
put(meta, "body.city", reg.city());
put(meta, "body.maker", reg.maker());
put(meta, "body.deviceType", reg.deviceType());
put(meta, "body.deviceId", reg.deviceId());
put(meta, "body.plateColor", reg.plateColor());
put(meta, "body.plate", reg.plate());
}
case Jt808Body.Auth auth -> {
put(meta, "body.token", auth.token());
put(meta, "body.imei", auth.imei());
put(meta, "body.softwareVersion", auth.softwareVersion());
}
case Jt808Body.Location location -> putLocationMetadata(meta, "body.", location);
case Jt808Body.LocationBatch batch -> {
put(meta, "body.batchType", batch.batchType());
put(meta, "body.locationCount", batch.locations() == null ? 0 : batch.locations().size());
}
case Jt808Body.ParamsReport report -> {
put(meta, "body.responseSerialNo", report.responseSerialNo());
put(meta, "body.parameterCount", report.parameters() == null ? 0 : report.parameters().size());
if (report.parameters() != null) {
report.parameters().forEach((id, bytes) ->
put(meta, "body.param.0x" + String.format("%08X", id), toHex(bytes)));
}
}
case Jt808Body.TerminalAttrs attrs -> {
put(meta, "body.terminalType", attrs.terminalType());
put(meta, "body.maker", attrs.maker());
put(meta, "body.terminalModel", attrs.terminalModel());
put(meta, "body.terminalId", attrs.terminalId());
put(meta, "body.iccid", attrs.iccid());
put(meta, "body.hardwareVersion", attrs.hardwareVersion());
put(meta, "body.firmwareVersion", attrs.firmwareVersion());
put(meta, "body.gnssProperty", attrs.gnssProperty());
put(meta, "body.commProperty", attrs.commProperty());
}
case Jt808Body.MediaEvent event -> {
put(meta, "body.mediaId", event.mediaId());
put(meta, "body.mediaType", event.mediaType());
put(meta, "body.mediaFormat", event.mediaFormat());
put(meta, "body.eventCode", event.eventCode());
put(meta, "body.channelId", event.channelId());
}
case Jt808Body.MediaUpload upload -> {
put(meta, "body.mediaId", upload.mediaId());
put(meta, "body.mediaType", upload.mediaType());
put(meta, "body.mediaFormat", upload.mediaFormat());
put(meta, "body.eventCode", upload.eventCode());
put(meta, "body.channelId", upload.channelId());
put(meta, "body.dataSizeBytes", upload.data() == null ? 0 : upload.data().length);
if (upload.location() != null) {
putLocationMetadata(meta, "body.location.", upload.location());
}
}
case Jt808Body.MediaAttributes attrs -> {
put(meta, "body.inputAudioEncoding", attrs.inputAudioEncoding());
put(meta, "body.inputAudioChannels", attrs.inputAudioChannels());
put(meta, "body.inputAudioSampleRate", attrs.inputAudioSampleRate());
put(meta, "body.inputAudioBitDepth", attrs.inputAudioBitDepth());
put(meta, "body.audioFrameLength", attrs.audioFrameLength());
put(meta, "body.audioOutputSupported", attrs.audioOutputSupported());
put(meta, "body.videoEncoding", attrs.videoEncoding());
put(meta, "body.maxAudioChannels", attrs.maxAudioChannels());
put(meta, "body.maxVideoChannels", attrs.maxVideoChannels());
put(meta, "body.rawSizeBytes", attrs.raw() == null ? 0 : attrs.raw().length);
}
case Jt808Body.FileUploadComplete complete -> {
put(meta, "body.responseSerialNo", complete.responseSerialNo());
put(meta, "body.result", complete.result());
put(meta, "body.rawSizeBytes", complete.raw() == null ? 0 : complete.raw().length);
}
case Jt808Body.FileList files -> {
put(meta, "body.responseSerialNo", files.responseSerialNo());
put(meta, "body.fileCount", files.fileCount());
put(meta, "body.rawSizeBytes", files.raw() == null ? 0 : files.raw().length);
}
case Jt808Body.PassengerVolume volume -> {
put(meta, "body.channelId", volume.channelId());
put(meta, "body.startTime", volume.startTime());
put(meta, "body.endTime", volume.endTime());
put(meta, "body.passengerGetOn", volume.passengerGetOn());
put(meta, "body.passengerGetOff", volume.passengerGetOff());
put(meta, "body.rawSizeBytes", volume.raw() == null ? 0 : volume.raw().length);
}
case Jt808Body.Passthrough passthrough -> {
put(meta, "body.passthroughType", "0x" + String.format("%02X", passthrough.passthroughType()));
put(meta, "body.dataSizeBytes", passthrough.data() == null ? 0 : passthrough.data().length);
put(meta, "body.dataHex", toHex(passthrough.data()));
}
case Jt808Body.Raw raw -> {
put(meta, "body.rawMessageId", "0x" + Jt808MessageId.hex(raw.messageId()));
put(meta, "body.rawSizeBytes", raw.bytes() == null ? 0 : raw.bytes().length);
put(meta, "body.rawHex", toHex(raw.bytes()));
}
case Jt808Body.Malformed malformed -> {
put(meta, "body.peer", malformed.peer());
put(meta, "body.errorMessage", malformed.errorMessage());
put(meta, "body.rawSizeBytes", malformed.bytes() == null ? 0 : malformed.bytes().length);
}
case Jt808Body.Heartbeat ignored -> {
}
case Jt808Body.Unregister ignored -> {
}
}
meta.put("jt808.register.province", Integer.toString(reg.province()));
meta.put("jt808.register.city", Integer.toString(reg.city()));
meta.put("jt808.register.maker", nullToEmpty(reg.maker()));
meta.put("jt808.register.deviceType", nullToEmpty(reg.deviceType()));
meta.put("jt808.register.deviceId", nullToEmpty(reg.deviceId()));
meta.put("jt808.register.plateColor", Integer.toString(reg.plateColor()));
meta.put("jt808.register.plate", nullToEmpty(reg.plate()));
}
private static String nullToEmpty(String value) {
return value == null ? "" : value;
private static void putLocationMetadata(Map<String, String> meta, String prefix, Jt808Body.Location location) {
put(meta, prefix + "alarmFlag", location.alarmFlag());
put(meta, prefix + "statusFlag", location.statusFlag());
meta.putAll(Jt808LocationFlagInfo.decodeStatus(location.statusFlag(), prefix + "status."));
meta.putAll(Jt808LocationFlagInfo.decodeAlarm(location.alarmFlag(), prefix + "alarm."));
put(meta, prefix + "longitude", location.longitude());
put(meta, prefix + "latitude", location.latitude());
put(meta, prefix + "altitudeM", location.altitudeM());
put(meta, prefix + "speedKmh", location.speedKmh());
put(meta, prefix + "directionDeg", location.directionDeg());
put(meta, prefix + "deviceTime", location.deviceTime());
if (location.extensionItems() != null) {
location.extensionItems().forEach((id, bytes) ->
put(meta, prefix + "extra.0x" + String.format("%02X", id), toHex(bytes)));
meta.putAll(Jt808LocationAdditionalInfo.decode(location.extensionItems(), prefix + "extra."));
}
}
private byte[] registerAckFrame(String phone, int ackSerial, String token) {
var cmd = Jt808Commands.registerAck(ackSerial, 0, token);
return Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body());
private static void put(Map<String, String> meta, String key, Object value) {
if (value != null) {
meta.put(key, String.valueOf(value));
}
}
private byte[] terminalAckFrame(String phone, int ackSerial, int ackMsgId, int result) {
private static String toHex(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return "";
}
StringBuilder out = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
out.append(String.format("%02X", b & 0xFF));
}
return out.toString();
}
private void ackTerminal(ChannelHandlerContext ctx, String phone, int ackSerial, int ackMsgId, int result) {
var cmd = Jt808Commands.platformAck(ackSerial, ackMsgId, result);
return Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body());
byte[] frame = Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body());
ctx.writeAndFlush(Unpooled.wrappedBuffer(frame));
}
private static String generateToken(String phone) {
return UUID.nameUUIDFromBytes(("jt808:" + phone).getBytes()).toString().replace("-", "").substring(0, 16);
}
private static String generatedInternalVin(String phone, Jt808Body.Register reg) {
String material = String.join("|", phone, reg.deviceId(), reg.plate(), reg.maker(), reg.deviceType());
try {
byte[] digest = MessageDigest.getInstance("SHA-256").digest(material.getBytes(StandardCharsets.UTF_8));
StringBuilder suffix = new StringBuilder(12);
for (int i = 0; i < digest.length && suffix.length() < 12; i++) {
suffix.append(String.format("%02X", digest[i] & 0xFF));
}
return "JT808" + suffix.substring(0, 12);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 not available", e);
}
}
@Override
public void channelInactive(ChannelHandlerContext ctx) {
channelRegistry.unbind(ctx.channel())

View File

@@ -10,6 +10,8 @@ import com.lingniu.ingest.identity.VehicleIdentityResolver;
import com.lingniu.ingest.identity.VehicleIdentitySource;
import com.lingniu.ingest.protocol.jt808.model.Jt808Body;
import com.lingniu.ingest.protocol.jt808.model.Jt808Header;
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationAdditionalInfo;
import com.lingniu.ingest.protocol.jt808.model.Jt808LocationFlagInfo;
import com.lingniu.ingest.protocol.jt808.model.Jt808Message;
import java.time.Instant;
@@ -276,12 +278,14 @@ public final class Jt808EventMapper implements EventMapper<Jt808Message> {
}
private static Map<String, String> locationMeta(Map<String, String> meta, Jt808Body.Location loc) {
if (loc.extensionItems().isEmpty()) {
return meta;
}
java.util.HashMap<String, String> copy = new java.util.HashMap<>(meta);
loc.extensionItems().forEach((id, bytes) ->
copy.put("jt808.extra.0x" + String.format("%02X", id), hex(bytes)));
copy.putAll(Jt808LocationFlagInfo.decodeStatus(loc.statusFlag(), "jt808.status."));
copy.putAll(Jt808LocationFlagInfo.decodeAlarm(loc.alarmFlag(), "jt808.alarm."));
if (!loc.extensionItems().isEmpty()) {
loc.extensionItems().forEach((id, bytes) ->
copy.put("jt808.extra.0x" + String.format("%02X", id), hex(bytes)));
copy.putAll(Jt808LocationAdditionalInfo.decode(loc.extensionItems()));
}
return Map.copyOf(copy);
}
@@ -304,15 +308,7 @@ public final class Jt808EventMapper implements EventMapper<Jt808Message> {
}
private static Double totalMileageKm(Jt808Body.Location loc) {
byte[] bytes = loc.extensionItems().get(0x01);
if (bytes == null || bytes.length != 4) {
return null;
}
long raw = ((long) bytes[0] & 0xFF) << 24
| ((long) bytes[1] & 0xFF) << 16
| ((long) bytes[2] & 0xFF) << 8
| ((long) bytes[3] & 0xFF);
return raw * 0.1;
return Jt808LocationAdditionalInfo.mileageKm(loc.extensionItems());
}
private static String hex(byte[] bytes) {

View File

@@ -0,0 +1,247 @@
package com.lingniu.ingest.protocol.jt808.model;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public final class Jt808LocationAdditionalInfo {
public static final String DEFAULT_PREFIX = "jt808.extra.";
private static final String[] VEHICLE_SIGNAL_BITS = {
"low_beam",
"high_beam",
"right_turn",
"left_turn",
"brake",
"reverse",
"fog_lamp",
"position_lamp",
"horn",
"air_conditioner",
"neutral",
"retarder",
"abs",
"heater",
"clutch"
};
private static final String[] IO_STATUS_BITS = {
"deep_sleep",
"sleep"
};
private Jt808LocationAdditionalInfo() {
}
public static Map<String, String> decode(Map<Integer, byte[]> extensionItems) {
return decode(extensionItems, DEFAULT_PREFIX);
}
public static Map<String, String> decode(Map<Integer, byte[]> extensionItems, String prefix) {
if (extensionItems == null || extensionItems.isEmpty()) {
return Map.of();
}
String p = prefix == null ? "" : prefix;
Map<String, String> out = new LinkedHashMap<>();
extensionItems.forEach((id, bytes) -> decodeItem(out, p, id == null ? -1 : id, bytes));
return Map.copyOf(out);
}
public static Double mileageKm(Map<Integer, byte[]> extensionItems) {
if (extensionItems == null) {
return null;
}
byte[] bytes = extensionItems.get(0x01);
if (bytes == null || bytes.length != 4) {
return null;
}
return u32(bytes, 0) * 0.1;
}
private static void decodeItem(Map<String, String> out, String prefix, int id, byte[] bytes) {
if (bytes == null) {
return;
}
switch (id) {
case 0x01 -> put(out, prefix + "mileage_km", scaled(bytes, 4, 1));
case 0x02 -> put(out, prefix + "fuel_l", scaled(bytes, 2, 1));
case 0x03 -> put(out, prefix + "recorder_speed_kmh", scaled(bytes, 2, 1));
case 0x04 -> put(out, prefix + "manual_alarm_event_id", unsigned(bytes, 2));
case 0x05 -> decodeTirePressure(out, prefix, bytes);
case 0x06 -> put(out, prefix + "compartment_temp_c", signedHighBitWord(bytes));
case 0x11 -> decodeOverspeed(out, prefix, bytes);
case 0x12 -> decodeAreaRouteAlarm(out, prefix, bytes);
case 0x13 -> decodeRouteTime(out, prefix, bytes);
case 0x25 -> decodeVehicleSignal(out, prefix, bytes);
case 0x2A -> decodeIoStatus(out, prefix, bytes);
case 0x2B -> decodeAnalog(out, prefix, bytes);
case 0x30 -> put(out, prefix + "network_signal_strength", unsigned(bytes, 1));
case 0x31 -> put(out, prefix + "gnss_satellite_count", unsigned(bytes, 1));
default -> {
}
}
}
private static void decodeTirePressure(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 30) {
return;
}
List<String> values = new ArrayList<>();
List<Integer> validIndexes = new ArrayList<>();
List<Integer> invalidIndexes = new ArrayList<>();
for (int i = 0; i < bytes.length; i++) {
int value = u8(bytes, i);
if (value == 0xFF) {
invalidIndexes.add(i);
} else {
validIndexes.add(i);
values.add(Integer.toString(value));
}
}
put(out, prefix + "tire_pressure_pa_values", String.join(",", values));
put(out, prefix + "tire_pressure_valid_indexes", ranges(validIndexes));
put(out, prefix + "tire_pressure_invalid_indexes", ranges(invalidIndexes));
}
private static void decodeOverspeed(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 1 && bytes.length != 5) {
return;
}
put(out, prefix + "overspeed.position_type", Integer.toString(u8(bytes, 0)));
if (bytes.length == 5) {
put(out, prefix + "overspeed.area_route_id", Long.toString(u32(bytes, 1)));
}
}
private static void decodeAreaRouteAlarm(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 6) {
return;
}
put(out, prefix + "area_route_alarm.position_type", Integer.toString(u8(bytes, 0)));
put(out, prefix + "area_route_alarm.area_route_id", Long.toString(u32(bytes, 1)));
put(out, prefix + "area_route_alarm.direction", Integer.toString(u8(bytes, 5)));
}
private static void decodeRouteTime(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 7) {
return;
}
put(out, prefix + "route_time.segment_id", Long.toString(u32(bytes, 0)));
put(out, prefix + "route_time.duration_seconds", Integer.toString(u16(bytes, 4)));
put(out, prefix + "route_time.result", Integer.toString(u8(bytes, 6)));
}
private static void decodeVehicleSignal(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 4) {
return;
}
long raw = u32(bytes, 0);
put(out, prefix + "vehicle_signal_raw", Long.toString(raw));
for (int i = 0; i < VEHICLE_SIGNAL_BITS.length; i++) {
put(out, prefix + "vehicle_signal." + VEHICLE_SIGNAL_BITS[i], Boolean.toString(bit(raw, i)));
}
}
private static void decodeIoStatus(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 2) {
return;
}
int raw = u16(bytes, 0);
put(out, prefix + "io_status_raw", Integer.toString(raw));
for (int i = 0; i < IO_STATUS_BITS.length; i++) {
put(out, prefix + "io_status." + IO_STATUS_BITS[i], Boolean.toString(bit(raw, i)));
}
}
private static void decodeAnalog(Map<String, String> out, String prefix, byte[] bytes) {
if (bytes.length != 4) {
return;
}
long raw = u32(bytes, 0);
put(out, prefix + "analog_raw", Long.toString(raw));
put(out, prefix + "analog_ad0", Integer.toString((int) (raw & 0xFFFF)));
put(out, prefix + "analog_ad1", Integer.toString((int) ((raw >>> 16) & 0xFFFF)));
}
private static String scaled(byte[] bytes, int expectedLength, int scale) {
if (bytes.length != expectedLength) {
return null;
}
long raw = expectedLength == 4 ? u32(bytes, 0) : u16(bytes, 0);
return BigDecimal.valueOf(raw, scale).stripTrailingZeros().toPlainString();
}
private static String unsigned(byte[] bytes, int expectedLength) {
if (bytes.length != expectedLength) {
return null;
}
return switch (expectedLength) {
case 1 -> Integer.toString(u8(bytes, 0));
case 2 -> Integer.toString(u16(bytes, 0));
case 4 -> Long.toString(u32(bytes, 0));
default -> null;
};
}
private static String signedHighBitWord(byte[] bytes) {
if (bytes.length != 2) {
return null;
}
int raw = u16(bytes, 0);
int value = (raw & 0x8000) == 0 ? raw : -(raw & 0x7FFF);
return Integer.toString(value);
}
private static int u8(byte[] bytes, int offset) {
return bytes[offset] & 0xFF;
}
private static int u16(byte[] bytes, int offset) {
return (u8(bytes, offset) << 8) | u8(bytes, offset + 1);
}
private static long u32(byte[] bytes, int offset) {
return ((long) u8(bytes, offset) << 24)
| ((long) u8(bytes, offset + 1) << 16)
| ((long) u8(bytes, offset + 2) << 8)
| u8(bytes, offset + 3);
}
private static boolean bit(long raw, int bit) {
return ((raw >>> bit) & 1L) == 1L;
}
private static void put(Map<String, String> out, String key, String value) {
if (value != null) {
out.put(key, value);
}
}
private static String ranges(List<Integer> indexes) {
if (indexes.isEmpty()) {
return "";
}
List<String> ranges = new ArrayList<>();
int start = indexes.getFirst();
int previous = start;
for (int i = 1; i < indexes.size(); i++) {
int current = indexes.get(i);
if (current == previous + 1) {
previous = current;
continue;
}
ranges.add(formatRange(start, previous));
start = current;
previous = current;
}
ranges.add(formatRange(start, previous));
return String.join(",", ranges);
}
private static String formatRange(int start, int end) {
return start == end ? Integer.toString(start) : start + "-" + end;
}
}

View File

@@ -0,0 +1,99 @@
package com.lingniu.ingest.protocol.jt808.model;
import java.util.LinkedHashMap;
import java.util.Map;
public final class Jt808LocationFlagInfo {
private static final String[] ALARM_BITS = {
"emergency",
"overspeed",
"fatigue_driving",
"warning",
"gnss_module_fault",
"gnss_antenna_disconnected",
"gnss_antenna_short_circuit",
"terminal_main_power_undervoltage",
"terminal_main_power_lost",
"display_fault",
"tts_fault",
"camera_fault",
"transport_ic_card_fault",
"overspeed_warning",
"fatigue_driving_warning",
"prohibited_driving",
null,
null,
"daily_driving_timeout",
"overtime_parking",
"area_in_out",
"route_in_out",
"route_time_abnormal",
"route_deviation",
"vss_fault",
"fuel_abnormal",
"stolen",
"illegal_ignition",
"illegal_displacement",
"collision_rollover",
"rollover_warning"
};
private Jt808LocationFlagInfo() {
}
public static Map<String, String> decodeAlarm(long alarmFlag, String prefix) {
Map<String, String> out = new LinkedHashMap<>();
String p = prefix == null ? "" : prefix;
for (int i = 0; i < ALARM_BITS.length; i++) {
String name = ALARM_BITS[i];
if (name != null) {
out.put(p + name, Boolean.toString(bit(alarmFlag, i)));
}
}
return Map.copyOf(out);
}
public static Map<String, String> decodeStatus(long statusFlag, String prefix) {
Map<String, String> out = new LinkedHashMap<>();
String p = prefix == null ? "" : prefix;
out.put(p + "acc_on", Boolean.toString(bit(statusFlag, 0)));
out.put(p + "positioned", Boolean.toString(bit(statusFlag, 1)));
out.put(p + "south_latitude", Boolean.toString(bit(statusFlag, 2)));
out.put(p + "west_longitude", Boolean.toString(bit(statusFlag, 3)));
out.put(p + "out_of_service", Boolean.toString(bit(statusFlag, 4)));
out.put(p + "encrypted", Boolean.toString(bit(statusFlag, 5)));
out.put(p + "forward_collision_warning", Boolean.toString(bit(statusFlag, 6)));
out.put(p + "lane_departure_warning", Boolean.toString(bit(statusFlag, 7)));
out.put(p + "load_status", loadStatus(statusFlag));
out.put(p + "fuel_circuit_cut", Boolean.toString(bit(statusFlag, 10)));
out.put(p + "electric_circuit_cut", Boolean.toString(bit(statusFlag, 11)));
out.put(p + "door_locked", Boolean.toString(bit(statusFlag, 12)));
out.put(p + "door1_open", Boolean.toString(bit(statusFlag, 13)));
out.put(p + "door2_open", Boolean.toString(bit(statusFlag, 14)));
out.put(p + "door3_open", Boolean.toString(bit(statusFlag, 15)));
out.put(p + "door4_open", Boolean.toString(bit(statusFlag, 16)));
out.put(p + "door5_open", Boolean.toString(bit(statusFlag, 17)));
out.put(p + "gps_used", Boolean.toString(bit(statusFlag, 18)));
out.put(p + "beidou_used", Boolean.toString(bit(statusFlag, 19)));
out.put(p + "glonass_used", Boolean.toString(bit(statusFlag, 20)));
out.put(p + "galileo_used", Boolean.toString(bit(statusFlag, 21)));
out.put(p + "moving", Boolean.toString(bit(statusFlag, 22)));
return Map.copyOf(out);
}
private static String loadStatus(long statusFlag) {
int code = (int) ((statusFlag >>> 8) & 0x03);
return switch (code) {
case 0 -> "EMPTY";
case 1 -> "HALF_LOADED";
case 2 -> "RESERVED";
case 3 -> "FULL";
default -> "UNKNOWN";
};
}
private static boolean bit(long raw, int bit) {
return ((raw >>> bit) & 1L) == 1L;
}
}

View File

@@ -36,4 +36,56 @@ public final class Jt808MessageId {
public static final int PLATFORM_TRACK_CONTROL = 0x8202;
public static final int PLATFORM_ALARM_ACK = 0x8203;
public static final int PLATFORM_DELETE_CIRCLE_AREA = 0x8601;
public static String nameOf(int messageId) {
return switch (messageId) {
case TERMINAL_GENERAL_RESPONSE -> "TERMINAL_GENERAL_RESPONSE";
case TERMINAL_HEARTBEAT -> "TERMINAL_HEARTBEAT";
case TERMINAL_UNREGISTER -> "TERMINAL_UNREGISTER";
case TERMINAL_REGISTER -> "TERMINAL_REGISTER";
case TERMINAL_AUTH -> "TERMINAL_AUTH";
case TERMINAL_PARAMS_REPORT -> "TERMINAL_PARAMS_REPORT";
case TERMINAL_ATTRS_REPORT -> "TERMINAL_ATTRS_REPORT";
case TERMINAL_LOCATION -> "TERMINAL_LOCATION";
case TERMINAL_LOCATION_BATCH -> "TERMINAL_LOCATION_BATCH";
case TERMINAL_EVENT_REPORT -> "TERMINAL_EVENT_REPORT";
case TERMINAL_CAN_DATA_REPORT -> "TERMINAL_CAN_DATA_REPORT";
case TERMINAL_MEDIA_EVENT -> "TERMINAL_MEDIA_EVENT";
case TERMINAL_MEDIA_UPLOAD -> "TERMINAL_MEDIA_UPLOAD";
case TERMINAL_PASSTHROUGH -> "TERMINAL_PASSTHROUGH";
case PLATFORM_GENERAL_RESPONSE -> "PLATFORM_GENERAL_RESPONSE";
case PLATFORM_REGISTER_ACK -> "PLATFORM_REGISTER_ACK";
case PLATFORM_SET_PARAMS -> "PLATFORM_SET_PARAMS";
case PLATFORM_QUERY_PARAMS -> "PLATFORM_QUERY_PARAMS";
case PLATFORM_TERMINAL_CONTROL -> "PLATFORM_TERMINAL_CONTROL";
case PLATFORM_QUERY_ATTRS -> "PLATFORM_QUERY_ATTRS";
case PLATFORM_QUERY_LOCATION -> "PLATFORM_QUERY_LOCATION";
case PLATFORM_TRACK_CONTROL -> "PLATFORM_TRACK_CONTROL";
case PLATFORM_ALARM_ACK -> "PLATFORM_ALARM_ACK";
case PLATFORM_DELETE_CIRCLE_AREA -> "PLATFORM_DELETE_CIRCLE_AREA";
default -> "UNKNOWN_0x" + hex(messageId);
};
}
public static String reportTypeOf(int messageId) {
return switch (messageId) {
case TERMINAL_LOCATION -> "LOCATION";
case TERMINAL_LOCATION_BATCH -> "LOCATION_BATCH";
case TERMINAL_HEARTBEAT -> "HEARTBEAT";
case TERMINAL_AUTH -> "AUTH";
case TERMINAL_REGISTER -> "REGISTER";
case TERMINAL_UNREGISTER -> "UNREGISTER";
case TERMINAL_GENERAL_RESPONSE -> "TERMINAL_RESPONSE";
case TERMINAL_PARAMS_REPORT -> "PARAMS_REPORT";
case TERMINAL_ATTRS_REPORT -> "ATTRS_REPORT";
case TERMINAL_MEDIA_EVENT -> "MEDIA_EVENT";
case TERMINAL_MEDIA_UPLOAD -> "MEDIA_UPLOAD";
case TERMINAL_PASSTHROUGH -> "PASSTHROUGH";
default -> "RAW";
};
}
public static String hex(int messageId) {
return String.format("%04X", messageId & 0xFFFF);
}
}

View File

@@ -41,6 +41,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.within;
class Jt808ChannelHandlerTest {
@@ -86,7 +87,7 @@ class Jt808ChannelHandlerTest {
}
@Test
void unresolvedRegisterDoesNotBindUnknownVinToExternalIdentifiers() {
void unresolvedRegisterBindsGeneratedInternalVinToExternalIdentifiers() {
InMemorySessionStore sessions = new InMemorySessionStore();
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(new ImmediateKafkaSink()));
@@ -115,9 +116,9 @@ class Jt808ChannelHandlerTest {
assertThat(identity.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "123456789012", "", "")))
.satisfies(resolved -> {
assertThat(resolved.vin()).isEqualTo("123456789012");
assertThat(resolved.resolved()).isFalse();
assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.FALLBACK_PHONE);
assertThat(resolved.vin()).startsWith("JT808");
assertThat(resolved.resolved()).isTrue();
assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
});
batchExecutor.close();
@@ -269,14 +270,143 @@ class Jt808ChannelHandlerTest {
assertThat(archive.metadata())
.containsEntry("vin", "LNVIN000000000808")
.containsEntry("phone", "123456789012")
.containsEntry("messageId", "0x0200")
.containsEntry("messageName", "TERMINAL_LOCATION")
.containsEntry("reportType", "LOCATION")
.containsEntry("body.alarmFlag", "0")
.containsEntry("body.statusFlag", "196608")
.containsEntry("body.status.acc_on", "false")
.containsEntry("body.status.load_status", "EMPTY")
.containsEntry("body.status.door4_open", "true")
.containsEntry("body.status.door5_open", "true")
.containsEntry("body.alarm.overspeed", "false")
.containsEntry("body.longitude", "116.397128")
.containsEntry("body.latitude", "39.916527")
.containsEntry("body.altitudeM", "50")
.containsEntry("body.directionDeg", "90")
.containsEntry("body.deviceTime", "2024-01-01T19:04:05Z")
.containsEntry("body.extra.0x01", "00003039")
.containsEntry("body.extra.mileage_km", "1234.5")
.containsEntry("body.extra.0x30", "58")
.containsEntry("body.extra.network_signal_strength", "88")
.containsEntry("identityResolved", "true")
.containsEntry("identitySource", "BOUND_PHONE");
assertThat(Double.parseDouble(archive.metadata().get("body.speedKmh"))).isCloseTo(52.3, within(0.001));
});
batchExecutor.close();
eventBus.close();
}
@Test
void registerArchiveIncludesParsedBodyFieldsForRawQuery() throws Exception {
RecordingSink sink = new RecordingSink(1);
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish);
Dispatcher dispatcher = new Dispatcher(
new HandlerRegistry(),
new InterceptorChain(List.of()),
new HandlerInvoker(),
eventBus,
batchExecutor);
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
identity.bind(new VehicleIdentityBinding(
ProtocolId.JT808, "LNVIN000000000808", "123456789012", "DEV808", "B80808"));
Jt808ChannelHandler handler = new Jt808ChannelHandler(
new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser()))),
dispatcher,
new InMemorySessionStore(),
identity,
new Jt808ChannelRegistry(),
new Jt808PendingRequests());
EmbeddedChannel channel = new EmbeddedChannel(handler);
byte[] frame = buildFrame(
Jt808MessageId.TERMINAL_REGISTER,
"123456789012",
3,
buildRegisterBody("DEV808", "B80808"));
channel.writeInbound(frame);
assertThat(sink.await()).isTrue();
assertThat(sink.events).singleElement().satisfies(event -> {
assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class);
VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event;
assertThat(archive.metadata())
.containsEntry("messageId", "0x0100")
.containsEntry("messageName", "TERMINAL_REGISTER")
.containsEntry("reportType", "REGISTER")
.containsEntry("body.province", "44")
.containsEntry("body.city", "4401")
.containsEntry("body.maker", "MAKER")
.containsEntry("body.deviceType", "TYPE-A")
.containsEntry("body.deviceId", "DEV808")
.containsEntry("body.plateColor", "1")
.containsEntry("body.plate", "B80808");
});
batchExecutor.close();
eventBus.close();
}
@Test
void unboundRegisterCreatesVinBindingForRegisterAndFollowingRawFrames() throws Exception {
RecordingSink sink = new RecordingSink(2);
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish);
Dispatcher dispatcher = new Dispatcher(
new HandlerRegistry(),
new InterceptorChain(List.of()),
new HandlerInvoker(),
eventBus,
batchExecutor);
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
Jt808ChannelHandler handler = new Jt808ChannelHandler(
new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser(), new LocationBodyParser()))),
dispatcher,
new InMemorySessionStore(),
identity,
new Jt808ChannelRegistry(),
new Jt808PendingRequests());
EmbeddedChannel channel = new EmbeddedChannel(handler);
channel.writeInbound(buildFrame(
Jt808MessageId.TERMINAL_REGISTER,
"13079963289",
3,
buildRegisterBody("9963289", "HUA00113F")));
channel.writeInbound(buildFrame(
Jt808MessageId.TERMINAL_LOCATION,
"13079963289",
4,
buildLocationBody()));
assertThat(sink.await()).isTrue();
VehicleEvent.RawArchive register = (VehicleEvent.RawArchive) sink.events.get(0);
VehicleEvent.RawArchive location = (VehicleEvent.RawArchive) sink.events.get(1);
assertThat(register.metadata())
.containsEntry("reportType", "REGISTER")
.containsEntry("body.deviceId", "9963289")
.containsEntry("body.plate", "HUA00113F")
.containsEntry("identityResolved", "true")
.containsEntry("identitySource", "REGISTERED");
assertThat(register.vin())
.startsWith("JT808")
.hasSize(17);
assertThat(location.metadata())
.containsEntry("reportType", "LOCATION")
.containsEntry("vin", register.vin())
.containsEntry("identityResolved", "true")
.containsEntry("identitySource", "BOUND_PHONE");
assertThat(location.vin()).isEqualTo(register.vin());
assertThat(identity.resolve(new com.lingniu.ingest.identity.VehicleIdentityLookup(
ProtocolId.JT808, "", "13079963289", "", "")).vin())
.isEqualTo(register.vin());
batchExecutor.close();
eventBus.close();
}
@Test
void unboundLocationArchiveUsesUnknownVinInsteadOfPhoneFallback() throws Exception {
RecordingSink sink = new RecordingSink(1);
@@ -309,6 +439,9 @@ class Jt808ChannelHandlerTest {
assertThat(archive.metadata())
.containsEntry("vin", "unknown")
.containsEntry("phone", "123456789012")
.containsEntry("messageId", "0x0200")
.containsEntry("messageName", "TERMINAL_LOCATION")
.containsEntry("reportType", "LOCATION")
.containsEntry("identityResolved", "false")
.containsEntry("identitySource", "FALLBACK_PHONE");
});
@@ -351,13 +484,13 @@ class Jt808ChannelHandlerTest {
VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event;
assertThat(archive.command()).isEqualTo(Jt808MessageId.TERMINAL_REGISTER);
assertThat(archive.metadata())
.containsEntry("jt808.register.province", "44")
.containsEntry("jt808.register.city", "4401")
.containsEntry("jt808.register.maker", "MAKER")
.containsEntry("jt808.register.deviceType", "TYPE-A")
.containsEntry("jt808.register.deviceId", "DEV808")
.containsEntry("jt808.register.plateColor", "1")
.containsEntry("jt808.register.plate", "B80808");
.containsEntry("body.province", "44")
.containsEntry("body.city", "4401")
.containsEntry("body.maker", "MAKER")
.containsEntry("body.deviceType", "TYPE-A")
.containsEntry("body.deviceId", "DEV808")
.containsEntry("body.plateColor", "1")
.containsEntry("body.plate", "B80808");
});
batchExecutor.close();
@@ -581,6 +714,12 @@ class Jt808ChannelHandlerTest {
writeU16(os, 90);
byte[] bcd = BcdCodec.encode("240102030405");
os.write(bcd, 0, 6);
os.write(0x01);
os.write(4);
writeU32(os, 12345);
os.write(0x30);
os.write(1);
os.write(0x58);
return os.toByteArray();
}

View File

@@ -38,6 +38,10 @@ class Jt808EventMapperTest {
assertThat(events.get(0).metadata())
.containsEntry("vin", "unknown")
.containsEntry("phone", "123456789012")
.containsEntry("jt808.status.acc_on", "false")
.containsEntry("jt808.status.positioned", "false")
.containsEntry("jt808.status.load_status", "EMPTY")
.containsEntry("jt808.alarm.overspeed", "false")
.containsEntry("identityResolved", "false")
.containsEntry("identitySource", "FALLBACK_PHONE");
}
@@ -58,7 +62,66 @@ class Jt808EventMapperTest {
.extracting(VehicleEvent::metadata)
.satisfies(meta -> assertThat(meta)
.containsEntry("jt808.extra.0x01", "00003039")
.containsEntry("jt808.extra.0x30", "58"));
.containsEntry("jt808.extra.0x30", "58")
.containsEntry("jt808.extra.mileage_km", "1234.5")
.containsEntry("jt808.extra.network_signal_strength", "88"));
}
@Test
void locationAdditionalItemsAreDecodedAsTable27Fields() {
var header = new Jt808Header(
Jt808MessageId.TERMINAL_LOCATION, 64, 0, false,
Jt808Header.ProtocolVersion.V2013, "123456789012", 1, 0, 0);
var body = new Jt808Body.Location(
0, 0, 116.397128, 39.916527, 50, 52.3, 90,
Instant.parse("2024-01-02T03:04:05Z"),
java.util.Map.ofEntries(
java.util.Map.entry(0x01, new byte[]{0x00, 0x00, 0x30, 0x39}),
java.util.Map.entry(0x02, new byte[]{0x00, 0x64}),
java.util.Map.entry(0x03, new byte[]{0x01, (byte) 0xF4}),
java.util.Map.entry(0x04, new byte[]{0x00, 0x02}),
java.util.Map.entry(0x05, tirePressureBytes()),
java.util.Map.entry(0x06, new byte[]{(byte) 0x80, 0x05}),
java.util.Map.entry(0x11, new byte[]{0x01, 0x00, 0x00, 0x00, 0x2A}),
java.util.Map.entry(0x12, new byte[]{0x04, 0x00, 0x00, 0x00, 0x2B, 0x01}),
java.util.Map.entry(0x13, new byte[]{0x00, 0x00, 0x00, 0x2C, 0x00, 0x78, 0x01}),
java.util.Map.entry(0x25, new byte[]{0x00, 0x00, 0x00, 0x05}),
java.util.Map.entry(0x2A, new byte[]{0x00, 0x03}),
java.util.Map.entry(0x2B, new byte[]{0x12, 0x34, 0x56, 0x78}),
java.util.Map.entry(0x30, new byte[]{0x1F}),
java.util.Map.entry(0x31, new byte[]{0x0C})));
VehicleEvent.Location event = (VehicleEvent.Location) mapper.toEvents(new Jt808Message(header, body)).getFirst();
assertThat(event.payload().totalMileageKm()).isEqualTo(1234.5);
assertThat(event.metadata())
.containsEntry("jt808.extra.mileage_km", "1234.5")
.containsEntry("jt808.extra.fuel_l", "10")
.containsEntry("jt808.extra.recorder_speed_kmh", "50")
.containsEntry("jt808.extra.manual_alarm_event_id", "2")
.containsEntry("jt808.extra.tire_pressure_pa_values", "32,33")
.containsEntry("jt808.extra.tire_pressure_invalid_indexes", "2-29")
.containsEntry("jt808.extra.compartment_temp_c", "-5")
.containsEntry("jt808.extra.overspeed.position_type", "1")
.containsEntry("jt808.extra.overspeed.area_route_id", "42")
.containsEntry("jt808.extra.area_route_alarm.position_type", "4")
.containsEntry("jt808.extra.area_route_alarm.area_route_id", "43")
.containsEntry("jt808.extra.area_route_alarm.direction", "1")
.containsEntry("jt808.extra.route_time.segment_id", "44")
.containsEntry("jt808.extra.route_time.duration_seconds", "120")
.containsEntry("jt808.extra.route_time.result", "1")
.containsEntry("jt808.extra.vehicle_signal_raw", "5")
.containsEntry("jt808.extra.vehicle_signal.low_beam", "true")
.containsEntry("jt808.extra.vehicle_signal.high_beam", "false")
.containsEntry("jt808.extra.vehicle_signal.right_turn", "true")
.containsEntry("jt808.extra.io_status_raw", "3")
.containsEntry("jt808.extra.io_status.deep_sleep", "true")
.containsEntry("jt808.extra.io_status.sleep", "true")
.containsEntry("jt808.extra.analog_raw", "305419896")
.containsEntry("jt808.extra.analog_ad0", "22136")
.containsEntry("jt808.extra.analog_ad1", "4660")
.containsEntry("jt808.extra.network_signal_strength", "31")
.containsEntry("jt808.extra.gnss_satellite_count", "12");
}
@Test
@@ -249,4 +312,12 @@ class Jt808EventMapperTest {
.containsEntry("parseError", "true");
});
}
private static byte[] tirePressureBytes() {
byte[] bytes = new byte[30];
java.util.Arrays.fill(bytes, (byte) 0xFF);
bytes[0] = 32;
bytes[1] = 33;
return bytes;
}
}

View File

@@ -36,6 +36,10 @@
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>

View File

@@ -12,6 +12,10 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
@@ -27,6 +31,8 @@ import java.util.Map;
@RequestMapping("/api/event-history")
public class EventHistoryController {
private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai");
private final EventFileStore store;
public EventHistoryController(EventFileStore store) {
@@ -40,8 +46,13 @@ public class EventHistoryController {
@RequestParam String dateTo,
@RequestParam(defaultValue = "ASC") EventFileQuery.Order order,
@RequestParam(defaultValue = "100") int limit,
@RequestParam(required = false) String vin) throws IOException {
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin).stream()
@RequestParam(required = false) String vin,
@RequestParam(required = false) String eventType,
@RequestParam(required = false) String cursorEventTime,
@RequestParam(required = false) String cursorIngestTime,
@RequestParam(required = false) String cursorEventId) throws IOException {
return queryRecords(protocol, dateFrom, dateTo, order, limit, vin, eventType,
cursorEventTime, cursorIngestTime, cursorEventId).stream()
.map(RecordResponse::from)
.toList();
}
@@ -51,7 +62,11 @@ public class EventHistoryController {
String dateTo,
EventFileQuery.Order order,
int limit,
String vin) throws IOException {
String vin,
String eventType,
String cursorEventTime,
String cursorIngestTime,
String cursorEventId) throws IOException {
QueryTimeRange range = QueryTimeRange.parse(dateFrom, dateTo);
return store.query(new EventFileQuery(
protocol,
@@ -62,7 +77,22 @@ public class EventHistoryController {
order,
limit,
vin,
null));
eventType,
instantParam(cursorEventTime),
instantParam(cursorIngestTime),
cursorEventId));
}
private static Instant instantParam(String raw) {
if (raw == null || raw.isBlank()) {
return null;
}
String value = raw.trim();
try {
return OffsetDateTime.parse(value).toInstant();
} catch (RuntimeException ignored) {
return LocalDateTime.parse(value).atZone(DEFAULT_ZONE).toInstant();
}
}
public record RecordResponse(

View File

@@ -1,8 +1,8 @@
package com.lingniu.ingest.eventhistory;
import com.google.protobuf.InvalidProtocolBufferException;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
@@ -20,7 +20,7 @@ import java.util.List;
* <p>当前 32960 本机运行配置没有启用 Kafka consumer这个类保留给“其他服务把 envelope
* 写回历史库”的解耦部署方式。失败时返回结构化结果,由 Kafka worker 决定是否进 DLQ。
*/
public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor {
private final EventFileStore store;
private final TelemetryEnvelopeRecordMapper mapper;
@@ -33,7 +33,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter) {
this(mapper, tdengineWriter, true);
this(mapper, tdengineWriter, false);
}
public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper,
@@ -45,7 +45,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor {
public EventHistoryEnvelopeIngestor(EventFileStore store,
TelemetryEnvelopeRecordMapper mapper,
TdengineHistoryWriter tdengineWriter) {
this(store, mapper, tdengineWriter, true, true);
this(store, mapper, tdengineWriter, true, false);
}
public EventHistoryEnvelopeIngestor(EventFileStore store,

View File

@@ -1,7 +1,5 @@
package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
@@ -32,8 +30,6 @@ import java.util.List;
@Tag(name = "jt-808-location-controller", description = "JT808 位置历史分页查询接口。")
public final class Jt808LocationHistoryController {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final TdengineHistoryReader reader;
public Jt808LocationHistoryController(TdengineHistoryReader reader) {
@@ -127,8 +123,7 @@ public final class Jt808LocationHistoryController {
long alarmFlag,
long statusFlag,
Double totalMileageKm,
String rawUri,
String metadataJson) {
String rawUri) {
private static LocationResponse from(TdengineLocationRow row) {
return new LocationResponse(
@@ -147,23 +142,7 @@ public final class Jt808LocationHistoryController {
row.alarmFlag(),
row.statusFlag(),
row.totalMileageKm(),
rawUri(row),
row.metadataJson());
}
private static String rawUri(TdengineLocationRow row) {
if (row.rawUri() != null && !row.rawUri().isBlank()) {
return row.rawUri();
}
String metadataJson = row.metadataJson();
if (metadataJson == null || metadataJson.isBlank()) {
return row.rawUri();
}
try {
return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
} catch (JsonProcessingException ignored) {
return row.rawUri();
}
row.rawUri());
}
}
}

View File

@@ -1,7 +1,5 @@
package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery;
import com.lingniu.ingest.tdenginehistory.TdengineLocationRow;
@@ -32,8 +30,6 @@ import java.util.Locale;
@Tag(name = "location-history-controller", description = "通用位置历史分页查询接口。")
public final class LocationHistoryController {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final TdengineHistoryReader reader;
public LocationHistoryController(TdengineHistoryReader reader) {
@@ -161,7 +157,6 @@ public final class LocationHistoryController {
long statusFlag,
Double totalMileageKm,
String rawUri,
String metadataJson,
String protocol) {
private static LocationResponse from(TdengineLocationRow row) {
@@ -181,24 +176,8 @@ public final class LocationHistoryController {
row.alarmFlag(),
row.statusFlag(),
row.totalMileageKm(),
rawUri(row),
row.metadataJson(),
row.rawUri(),
row.protocol());
}
private static String rawUri(TdengineLocationRow row) {
if (row.rawUri() != null && !row.rawUri().isBlank()) {
return row.rawUri();
}
String metadataJson = row.metadataJson();
if (metadataJson == null || metadataJson.isBlank()) {
return row.rawUri();
}
try {
return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri());
} catch (JsonProcessingException ignored) {
return row.rawUri();
}
}
}
}

View File

@@ -184,6 +184,7 @@ public final class RawFrameHistoryController {
String parseError,
String peer,
String metadataJson,
String parsedJson,
Map<String, Object> metadata,
Map<String, Object> parsedFields,
String protocol,
@@ -209,8 +210,9 @@ public final class RawFrameHistoryController {
row.parseError(),
row.peer(),
row.metadataJson(),
row.parsedJson(),
metadata,
parsedFields(reader, row, metadata, includeParsedFields),
parsedFields(reader, row, metadata, parsed(row.parsedJson()), includeParsedFields),
row.protocol(),
row.vehicleKey(),
row.vin(),
@@ -231,18 +233,23 @@ public final class RawFrameHistoryController {
private static Map<String, Object> parsedFields(TdengineHistoryReader reader,
TdengineRawFrameRow row,
Map<String, Object> metadata,
Map<String, Object> parsed,
boolean includeParsedFields) throws IOException {
Map<String, Object> out = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : metadata.entrySet()) {
String key = entry.getKey();
if (key == null || key.isBlank()) {
continue;
}
if (key.startsWith("raw.") || key.startsWith("jt808.")) {
out.put(key, entry.getValue());
} else {
out.put("raw." + key, entry.getValue());
if (parsed.isEmpty()) {
for (Map.Entry<String, Object> entry : metadata.entrySet()) {
String key = entry.getKey();
if (key == null || key.isBlank()) {
continue;
}
if (key.startsWith("raw.") || key.startsWith("jt808.")) {
out.put(key, entry.getValue());
} else {
out.put("raw." + key, entry.getValue());
}
}
} else {
out.putAll(parsed);
}
if (!includeParsedFields) {
return Map.copyOf(out);
@@ -262,6 +269,17 @@ public final class RawFrameHistoryController {
return Map.copyOf(out);
}
private static Map<String, Object> parsed(String parsedJson) {
if (parsedJson == null || parsedJson.isBlank()) {
return Map.of();
}
try {
return OBJECT_MAPPER.readValue(parsedJson, MAP_TYPE);
} catch (Exception ignored) {
return Map.of("_raw", parsedJson);
}
}
private static void addLocation(Map<String, Object> out, TdengineLocationRow row) {
out.put("location.longitude", row.longitude());
out.put("location.latitude", row.latitude());

View File

@@ -150,6 +150,7 @@ public final class TelemetryEnvelopeRecordMapper {
payload.put("infoType", metadata.get("infoType"));
}
payload.put("rawSizeBytes", rawArchive.getSizeBytes());
putParsedJson(payload, rawArchive.getParsedJson());
payload.put("metadata", metadata);
try {
return OBJECT_MAPPER.writeValueAsString(payload);
@@ -157,4 +158,16 @@ public final class TelemetryEnvelopeRecordMapper {
throw new IllegalArgumentException("failed to serialize raw_archive", e);
}
}
private static void putParsedJson(Map<String, Object> payload, String parsedJson) {
if (parsedJson == null || parsedJson.isBlank()) {
return;
}
try {
payload.put("parsed", OBJECT_MAPPER.readTree(parsedJson));
} catch (JsonProcessingException ex) {
payload.put("parsed", parsedJson);
payload.put("parsedJsonError", ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage());
}
}
}

View File

@@ -26,7 +26,11 @@ class EventHistoryControllerTest {
"2026-06-22",
EventFileQuery.Order.DESC,
50,
"VIN001");
"VIN001",
"RAW_ARCHIVE",
"2026-06-22T08:00:00Z",
"2026-06-22T08:00:01Z",
"event-0");
assertThat(records).extracting(EventHistoryController.RecordResponse::eventId).containsExactly("event-1");
assertThat(records.getFirst().eventTime()).isEqualTo("2026-06-22T08:00:00Z");
@@ -37,6 +41,10 @@ class EventHistoryControllerTest {
assertThat(store.lastQuery.order()).isEqualTo(EventFileQuery.Order.DESC);
assertThat(store.lastQuery.limit()).isEqualTo(50);
assertThat(store.lastQuery.vin()).isEqualTo("VIN001");
assertThat(store.lastQuery.eventType()).isEqualTo("RAW_ARCHIVE");
assertThat(store.lastQuery.cursorEventTime()).isEqualTo(Instant.parse("2026-06-22T08:00:00Z"));
assertThat(store.lastQuery.cursorIngestTime()).isEqualTo(Instant.parse("2026-06-22T08:00:01Z"));
assertThat(store.lastQuery.cursorEventId()).isEqualTo("event-0");
}
private static EventFileRecord record(String eventId) {

View File

@@ -126,7 +126,7 @@ class EventHistoryEnvelopeIngestorTest {
}
@Test
void tryIngestWritesRawLocationAndTelemetryFactsToTdengineWhenWriterExists() throws Exception {
void tryIngestWritesRawAndLocationFactsToTdengineWhenWriterExists() throws Exception {
CapturingStore store = new CapturingStore();
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
@@ -180,10 +180,7 @@ class EventHistoryEnvelopeIngestorTest {
.extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1");
assertThat(tdengineWriter.locations.getFirst().longitude()).isEqualTo(113.12);
assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
assertThat(tdengineWriter.telemetryFields.getFirst().valueDouble()).isEqualTo(42.5);
assertThat(tdengineWriter.telemetryFields).isEmpty();
}
@Test
@@ -200,16 +197,14 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1");
assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
assertThat(tdengineWriter.telemetryFields).isEmpty();
}
@Test
void tryIngestCanSkipTelemetryFieldFactsForHighThroughputRuntime() {
void tryIngestCanWriteTelemetryFieldFactsWhenExplicitlyEnabled() {
CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter();
EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor(
new TelemetryEnvelopeRecordMapper(), tdengineWriter, false);
new TelemetryEnvelopeRecordMapper(), tdengineWriter, true);
EnvelopeIngestResult result = ingestor.tryIngest(
jt808LocationEnvelope("jt808-location-1", "frame-jt808-1", "013800000001").toByteArray());
@@ -219,7 +214,9 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1");
assertThat(tdengineWriter.telemetryFields).isEmpty();
assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh");
}
@Test
@@ -249,9 +246,7 @@ class EventHistoryEnvelopeIngestorTest {
.containsExactly("frame-jt808-1", "frame-jt808-2");
assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId)
.containsExactly("jt808-location-1", "jt808-location-2");
assertThat(tdengineWriter.telemetryFields)
.extracting(TdengineTelemetryFieldRow::fieldKey)
.containsExactly("location.speedKmh", "location.speedKmh");
assertThat(tdengineWriter.telemetryFields).isEmpty();
}
private static VehicleEnvelope envelope(String eventId) {

View File

@@ -314,6 +314,7 @@ class Gb32960DecodedFrameServiceTest {
"",
"127.0.0.1:32960",
"{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960",
vin,
vin,
@@ -360,6 +361,7 @@ class Gb32960DecodedFrameServiceTest {
"",
"8.134.95.166:37302",
"{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960",
"unknown:GB32960:platform-login",
"",
@@ -409,6 +411,7 @@ class Gb32960DecodedFrameServiceTest {
"",
"127.0.0.1:32960",
"{\"platformAccount\":\"Hyundai\"}",
"",
"GB32960",
vin,
vin,

View File

@@ -51,13 +51,12 @@ class Jt808LocationHistoryControllerTest {
}
@Test
void fallsBackToRawArchiveUriInMetadataWhenLocationRawUriIsBlank() throws Exception {
void returnsLocationRawUriFromCoreColumn() throws Exception {
CapturingReader reader = new CapturingReader(new TdenginePage<>(
List.of(row(
"fact-blank-raw",
"2026-06-29T05:00:01Z",
"",
"{\"rawArchiveUri\":\"archive://jt808/metadata-frame.bin\"}")),
"archive://jt808/core-frame.bin")),
Optional.empty()));
Jt808LocationHistoryController controller = new Jt808LocationHistoryController(reader);
@@ -71,7 +70,7 @@ class Jt808LocationHistoryControllerTest {
null);
assertThat(response.items().getFirst().rawUri())
.isEqualTo("archive://jt808/metadata-frame.bin");
.isEqualTo("archive://jt808/core-frame.bin");
}
@Test
@@ -92,10 +91,10 @@ class Jt808LocationHistoryControllerTest {
}
private static TdengineLocationRow row(String factId, String ts) {
return row(factId, ts, "archive://jt808/frame-1.bin", "{\"messageId\":\"512\"}");
return row(factId, ts, "archive://jt808/frame-1.bin");
}
private static TdengineLocationRow row(String factId, String ts, String rawUri, String metadataJson) {
private static TdengineLocationRow row(String factId, String ts, String rawUri) {
Instant instant = Instant.parse(ts);
return new TdengineLocationRow(
instant,
@@ -111,7 +110,6 @@ class Jt808LocationHistoryControllerTest {
3L,
null,
rawUri,
metadataJson,
"JT808",
"jt808:g7gps",
"",

View File

@@ -96,6 +96,7 @@ class Jt808RawFrameHistoryControllerTest {
"",
"222.66.200.68:41000",
"{\"jt808.register.deviceId\":\"9963320\",\"jt808.register.plate\":\"沪A61559F\"}",
"",
"JT808",
"jt808:13079963320",
"unknown",

View File

@@ -82,7 +82,6 @@ class LocationHistoryControllerTest {
1,
null,
"",
"{\"rawArchiveUri\":\"archive://raw.bin\"}",
"MQTT_YUTONG",
"LMRKH9AC2R1004087",
"LMRKH9AC2R1004087",

View File

@@ -91,7 +91,6 @@ class RawFrameHistoryControllerTest {
524290,
14504.446,
raw.rawUri(),
"{\"rawArchiveUri\":\"" + raw.rawUri() + "\"}",
"JT808",
"VIN-JT808-1",
"VIN-JT808-1",
@@ -183,6 +182,7 @@ class RawFrameHistoryControllerTest {
"xinda",
"{\"rawArchiveUri\":\"archive://2026/06/29/XINDA_PUSH/unknown/" + frameId
+ ".bin\",\"plateNo\":\"粤BD12345\",\"rawJson\":\"{\\\"carName\\\":\\\"粤BD12345\\\"}\"}",
"",
"XINDA_PUSH",
"LB9A32A24P0LS1270",
"LB9A32A24P0LS1270",

View File

@@ -2,9 +2,11 @@ package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@@ -12,6 +14,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
class TelemetryEnvelopeRecordMapperTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void mapsFullFieldEnvelopeToEventFileRecord() {
VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
@@ -91,4 +95,33 @@ class TelemetryEnvelopeRecordMapperTest {
.containsEntry("tenant", "ln")
.containsEntry("originalSource", "FUTURE_OEM");
}
@Test
void rawArchiveRecordStoresParsedJsonInPayloadOnly() throws Exception {
VehicleEnvelope envelope = VehicleEnvelope.newBuilder()
.setSchemaVersion("1.0")
.setEventId("raw-808-1")
.setTraceId("trace-raw-1")
.setVin("VIN808")
.setSource("JT808")
.setEventTimeMs(1_782_112_400_000L)
.setIngestTimeMs(1_782_112_401_000L)
.putMetadata("phone", "13307795425")
.setRawArchive(RawArchiveRef.newBuilder()
.setUri("archive://2026/06/30/JT808/VIN808/raw-808-1.bin")
.setSizeBytes(64)
.setParsedJson("{\"messageId\":\"0x0200\",\"body\":{\"speedKmh\":60.5}}")
.build())
.build();
EventFileRecord record = new TelemetryEnvelopeRecordMapper().toRecord(envelope);
assertThat(record.eventType()).isEqualTo("RAW_ARCHIVE");
assertThat(record.metadata())
.containsEntry("phone", "13307795425")
.doesNotContainKey("rawArchiveParsedJson");
var payload = objectMapper.readTree(record.payloadJson());
assertThat(payload.path("parsed").path("messageId").asText()).isEqualTo("0x0200");
assertThat(payload.path("parsed").path("body").path("speedKmh").asDouble()).isEqualTo(60.5);
}
}

View File

@@ -35,11 +35,14 @@ class EventHistoryAutoConfigurationTest {
@Test
void createsHistoryBeansWhenEnabled() {
contextRunner
.withPropertyValues("lingniu.ingest.event-history.enabled=true")
.withPropertyValues(
"lingniu.ingest.event-history.enabled=true",
"lingniu.ingest.sink.mq.consumer.enabled=true")
.run(context -> {
assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class);
assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class);
assertThat(context).hasSingleBean(EnvelopeConsumerProcessor.class);
assertThat(context.getBeansOfType(EnvelopeConsumerProcessor.class))
.containsOnlyKeys("eventHistoryEnvelopeConsumerProcessor");
assertThat(context).hasSingleBean(EventHistoryController.class);
assertThat(context).doesNotHaveBean(Gb32960DecodedFrameService.class);
assertThat(context).doesNotHaveBean(Gb32960FrameController.class);

View File

@@ -23,6 +23,9 @@ import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor;
import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -36,7 +39,11 @@ import java.nio.file.Path;
import java.time.Duration;
import java.time.ZoneId;
@AutoConfiguration
@AutoConfiguration(after = {
DataSourceAutoConfiguration.class,
JdbcTemplateAutoConfiguration.class,
RedisAutoConfiguration.class
})
@EnableConfigurationProperties(VehicleStatProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.vehicle-stat", name = "enabled", havingValue = "true")
public class VehicleStatAutoConfiguration {

View File

@@ -4,13 +4,16 @@ import org.springframework.jdbc.core.JdbcTemplate;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.Locale;
import java.util.Objects;
public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageRepository {
private final JdbcTemplate jdbc;
public JdbcJt808DailyMileageRepository(JdbcTemplate jdbc) {
this.jdbc = jdbc;
this.jdbc = Objects.requireNonNull(jdbc, "jdbc must not be null");
initializeSchema();
}
@Override
@@ -18,18 +21,23 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
jdbc.update("""
insert into vehicle_daily_mileage_jt808 (
stat_date, vehicle_key, vin, phone, first_event_time, last_event_time,
gps_mileage_km, speed_integral_km, odometer_mileage_km,
daily_mileage_km, mileage_source, gps_mileage_km, speed_integral_km,
odometer_mileage_km, first_total_mileage_km, last_total_mileage_km,
accepted_points, bad_jump_segments, long_gap_segments, out_of_order_points,
odometer_anomalies, data_quality
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on duplicate key update
vin = values(vin),
phone = values(phone),
first_event_time = values(first_event_time),
last_event_time = values(last_event_time),
daily_mileage_km = values(daily_mileage_km),
mileage_source = values(mileage_source),
gps_mileage_km = values(gps_mileage_km),
speed_integral_km = values(speed_integral_km),
odometer_mileage_km = values(odometer_mileage_km),
first_total_mileage_km = values(first_total_mileage_km),
last_total_mileage_km = values(last_total_mileage_km),
accepted_points = values(accepted_points),
bad_jump_segments = values(bad_jump_segments),
long_gap_segments = values(long_gap_segments),
@@ -44,9 +52,13 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
result.phone(),
timestamp(result.firstEventTime()),
timestamp(result.lastEventTime()),
result.dailyMileageKm(),
result.mileageSource(),
result.gpsMileageKm(),
result.speedIntegralKm(),
result.odometerMileageKm(),
result.firstTotalMileageKm(),
result.lastTotalMileageKm(),
result.acceptedPoints(),
result.badJumpSegments(),
result.longGapSegments(),
@@ -55,6 +67,59 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR
result.dataQuality());
}
private void initializeSchema() {
jdbc.execute("""
create table if not exists vehicle_daily_mileage_jt808 (
stat_date date not null,
vehicle_key varchar(64) not null,
vin varchar(32) null,
phone varchar(32) null,
first_event_time timestamp null,
last_event_time timestamp null,
daily_mileage_km decimal(12,3) not null default 0,
mileage_source varchar(32) not null default 'INSUFFICIENT',
gps_mileage_km decimal(12,3) not null,
speed_integral_km decimal(12,3) not null,
odometer_mileage_km decimal(12,3) null,
first_total_mileage_km decimal(12,3) null,
last_total_mileage_km decimal(12,3) null,
accepted_points int not null,
bad_jump_segments int not null,
long_gap_segments int not null,
out_of_order_points int not null,
odometer_anomalies int not null,
data_quality varchar(16) not null,
updated_at timestamp not null default current_timestamp on update current_timestamp,
primary key (stat_date, vehicle_key),
key idx_vehicle_daily_mileage_jt808_vehicle_date (vehicle_key, stat_date),
key idx_vehicle_daily_mileage_jt808_vin_date (vin, stat_date),
key idx_vehicle_daily_mileage_jt808_phone_date (phone, stat_date)
)
""");
ensureColumn("daily_mileage_km", "daily_mileage_km decimal(12,3) not null default 0");
ensureColumn("mileage_source", "mileage_source varchar(32) not null default 'INSUFFICIENT'");
ensureColumn("first_total_mileage_km", "first_total_mileage_km decimal(12,3) null");
ensureColumn("last_total_mileage_km", "last_total_mileage_km decimal(12,3) null");
}
private void ensureColumn(String columnName, String definition) {
if (!columnExists(columnName)) {
jdbc.execute("alter table vehicle_daily_mileage_jt808 add column " + definition);
}
}
private boolean columnExists(String columnName) {
Integer count = jdbc.queryForObject("""
select count(*)
from information_schema.columns
where lower(table_name) = ?
and lower(column_name) = ?
""", Integer.class,
"vehicle_daily_mileage_jt808",
columnName.toLowerCase(Locale.ROOT));
return count != null && count > 0;
}
private static Timestamp timestamp(Instant instant) {
return instant == null ? null : Timestamp.from(instant);
}

View File

@@ -10,9 +10,13 @@ public record Jt808DailyMileageResult(
String phone,
Instant firstEventTime,
Instant lastEventTime,
double dailyMileageKm,
String mileageSource,
double gpsMileageKm,
double speedIntegralKm,
Double odometerMileageKm,
Double firstTotalMileageKm,
Double lastTotalMileageKm,
int acceptedPoints,
int badJumpSegments,
int longGapSegments,

View File

@@ -12,6 +12,7 @@ public final class Jt808DailyMileageState {
private Instant firstEventTime;
private Instant lastEventTime;
private Jt808LocationPoint lastPoint;
private double firstTotalMileageKm = Double.NaN;
private double lastTotalMileageKm = Double.NaN;
private double gpsMileageKm;
private double speedIntegralKm;
@@ -47,6 +48,7 @@ public final class Jt808DailyMileageState {
Instant firstEventTime,
Instant lastEventTime,
Jt808LocationPoint lastPoint,
double firstTotalMileageKm,
double lastTotalMileageKm,
double gpsMileageKm,
double speedIntegralKm,
@@ -60,6 +62,7 @@ public final class Jt808DailyMileageState {
state.firstEventTime = firstEventTime;
state.lastEventTime = lastEventTime;
state.lastPoint = lastPoint;
state.firstTotalMileageKm = firstTotalMileageKm;
state.lastTotalMileageKm = lastTotalMileageKm;
state.gpsMileageKm = gpsMileageKm;
state.speedIntegralKm = speedIntegralKm;
@@ -82,7 +85,7 @@ public final class Jt808DailyMileageState {
}
if (lastPoint == null) {
firstEventTime = point.eventTime();
updateOdometer(point);
updateOdometer(point, calculator);
accept(point);
return;
}
@@ -100,7 +103,7 @@ public final class Jt808DailyMileageState {
gpsMileageKm += segment.gpsKm();
speedIntegralKm += segment.speedIntegralKm();
}
updateOdometer(point);
updateOdometer(point, calculator);
accept(point);
}
@@ -112,9 +115,13 @@ public final class Jt808DailyMileageState {
phone,
firstEventTime,
lastEventTime,
dailyMileageKm(),
mileageSource(),
gpsMileageKm,
speedIntegralKm,
odometerMileageKm > 0.0 ? odometerMileageKm : null,
nullable(firstTotalMileageKm),
nullable(lastTotalMileageKm),
acceptedPoints,
badJumpSegments,
longGapSegments,
@@ -129,11 +136,14 @@ public final class Jt808DailyMileageState {
acceptedPoints++;
}
private void updateOdometer(Jt808LocationPoint point) {
private void updateOdometer(Jt808LocationPoint point, Jt808GpsMileageCalculator calculator) {
if (point.totalMileageKm() == null || !Double.isFinite(point.totalMileageKm())) {
return;
}
double current = point.totalMileageKm();
if (!Double.isFinite(firstTotalMileageKm)) {
firstTotalMileageKm = current;
}
if (!Double.isFinite(lastTotalMileageKm)) {
lastTotalMileageKm = current;
return;
@@ -143,15 +153,56 @@ public final class Jt808DailyMileageState {
lastTotalMileageKm = current;
return;
}
odometerMileageKm += current - lastTotalMileageKm;
double delta = current - lastTotalMileageKm;
if (lastPoint != null && !calculator.plausibleOdometerDelta(lastPoint, point, delta)) {
odometerAnomalies++;
lastTotalMileageKm = current;
return;
}
odometerMileageKm += delta;
lastTotalMileageKm = current;
}
private double dailyMileageKm() {
if (hasUsableTotalMileage()) {
return odometerMileageKm;
}
if (gpsMileageKm > 0.0) {
return gpsMileageKm;
}
return 0.0;
}
private String mileageSource() {
if (hasUsableTotalMileage()) {
return odometerAnomalies > 0 ? "JT808_TOTAL_MILEAGE_PARTIAL" : "JT808_TOTAL_MILEAGE";
}
if (gpsMileageKm > 0.0) {
return "GPS_DISTANCE";
}
return "INSUFFICIENT";
}
private boolean hasUsableTotalMileage() {
if (odometerMileageKm > 0.0) {
return true;
}
return acceptedPoints >= 2
&& odometerAnomalies == 0
&& Double.isFinite(firstTotalMileageKm)
&& Double.isFinite(lastTotalMileageKm)
&& lastTotalMileageKm >= firstTotalMileageKm;
}
private static Double nullable(double value) {
return Double.isFinite(value) ? value : null;
}
private String dataQuality() {
if (acceptedPoints < 2) {
return "PARTIAL";
}
if (badJumpSegments > 0 || longGapSegments > 0 || vehicleKey.startsWith("jt808:")) {
if (badJumpSegments > 0 || longGapSegments > 0 || odometerAnomalies > 0 || vehicleKey.startsWith("jt808:")) {
return "PARTIAL";
}
return "GOOD";
@@ -201,6 +252,10 @@ public final class Jt808DailyMileageState {
return lastTotalMileageKm;
}
public double firstTotalMileageKm() {
return firstTotalMileageKm;
}
public int acceptedPoints() {
return acceptedPoints;
}

View File

@@ -5,6 +5,7 @@ import java.time.Duration;
public final class Jt808GpsMileageCalculator {
private static final double EARTH_RADIUS_KM = 6371.0088;
private static final double ODOMETER_DELTA_TOLERANCE_KM = 0.1;
private final Duration maxSegmentGap;
private final double maxImpliedSpeedKmh;
@@ -61,6 +62,20 @@ public final class Jt808GpsMileageCalculator {
return new SegmentResult(gpsKm, speedIntegralKm, true, false, false, false);
}
public boolean plausibleOdometerDelta(Jt808LocationPoint previous,
Jt808LocationPoint current,
double deltaKm) {
if (previous == null || current == null || !Double.isFinite(deltaKm) || deltaKm < 0.0) {
return false;
}
Duration elapsed = Duration.between(previous.eventTime(), current.eventTime());
if (elapsed.isZero() || elapsed.isNegative()) {
return false;
}
double hours = elapsed.toMillis() / 3_600_000.0;
return deltaKm <= maxImpliedSpeedKmh * hours + ODOMETER_DELTA_TOLERANCE_KM;
}
private static double haversineKm(double lon1, double lat1, double lon2, double lat2) {
double latRad1 = Math.toRadians(lat1);
double latRad2 = Math.toRadians(lat2);

View File

@@ -13,7 +13,8 @@ public final class Jt808LocationPointExtractor {
public Optional<Jt808LocationPoint> extract(VehicleEnvelope envelope) {
if (envelope == null
|| !"JT808".equalsIgnoreCase(envelope.getSource())
|| !envelope.hasTelemetrySnapshot()) {
|| !envelope.hasTelemetrySnapshot()
|| !"LOCATION".equalsIgnoreCase(envelope.getTelemetrySnapshot().getEventType())) {
return Optional.empty();
}
Map<String, String> fields = fields(envelope);

View File

@@ -64,7 +64,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
String firstEventTime,
String lastEventTime,
PointSnapshot lastPoint,
double lastTotalMileageKm,
Double firstTotalMileageKm,
Double lastTotalMileageKm,
double gpsMileageKm,
double speedIntegralKm,
double odometerMileageKm,
@@ -83,7 +84,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
format(state.firstEventTime()),
format(state.lastEventTime()),
PointSnapshot.from(state.lastPoint()),
state.lastTotalMileageKm(),
finiteOrNull(state.firstTotalMileageKm()),
finiteOrNull(state.lastTotalMileageKm()),
state.gpsMileageKm(),
state.speedIntegralKm(),
state.odometerMileageKm(),
@@ -103,7 +105,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
parseInstant(firstEventTime),
parseInstant(lastEventTime),
lastPoint == null ? null : lastPoint.toPoint(),
lastTotalMileageKm,
nanIfNull(firstTotalMileageKm),
nanIfNull(lastTotalMileageKm),
gpsMileageKm,
speedIntegralKm,
odometerMileageKm,
@@ -164,4 +167,12 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore
private static Instant parseInstant(String value) {
return value == null || value.isBlank() ? null : Instant.parse(value);
}
private static Double finiteOrNull(double value) {
return Double.isFinite(value) ? value : null;
}
private static double nanIfNull(Double value) {
return value == null ? Double.NaN : value;
}
}

View File

@@ -21,10 +21,13 @@ import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource;
import java.time.LocalDate;
import java.util.List;
import java.util.Optional;
@@ -117,6 +120,29 @@ class VehicleStatAutoConfigurationTest {
});
}
@Test
void createsJt808MileageBeansAfterJdbcAutoConfiguration() {
new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(
JdbcTemplateAutoConfiguration.class,
VehicleStatAutoConfiguration.class))
.withBean(EnvelopeDeadLetterSink.class, () -> record -> {})
.withBean(DataSource.class, () -> new DriverManagerDataSource(
"jdbc:h2:mem:vehicle_stat_auto;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1",
"sa",
""))
.withPropertyValues(
"lingniu.ingest.vehicle-stat.enabled=true",
"lingniu.ingest.vehicle-stat.jt808.enabled=true",
"lingniu.ingest.vehicle-stat.jt808.state-store=memory")
.run(context -> {
assertThat(context).hasSingleBean(JdbcTemplate.class);
assertThat(context).hasSingleBean(Jt808DailyMileageRepository.class);
assertThat(context).hasSingleBean(Jt808MileageStreamProcessor.class);
assertThat(context).hasSingleBean(VehicleStatEnvelopeIngestor.class);
});
}
private static final class InMemoryVehicleStatRepository implements VehicleStatRepository {
@Override public void appendMileagePoint(String vin, MileagePoint point) {}
@Override public List<MileagePoint> mileagePoints(String vin, LocalDate statDate) { return List.of(); }

View File

@@ -24,30 +24,20 @@ class JdbcJt808DailyMileageRepositoryTest {
"");
jdbc = new JdbcTemplate(dataSource);
jdbc.execute("drop table if exists vehicle_daily_mileage_jt808");
jdbc.execute("""
create table vehicle_daily_mileage_jt808 (
stat_date date not null,
vehicle_key varchar(64) not null,
vin varchar(32) null,
phone varchar(32) null,
first_event_time timestamp null,
last_event_time timestamp null,
gps_mileage_km decimal(12,3) not null,
speed_integral_km decimal(12,3) not null,
odometer_mileage_km decimal(12,3) null,
accepted_points int not null,
bad_jump_segments int not null,
long_gap_segments int not null,
out_of_order_points int not null,
odometer_anomalies int not null,
data_quality varchar(16) not null,
updated_at timestamp not null default current_timestamp,
primary key (stat_date, vehicle_key)
)
""");
repository = new JdbcJt808DailyMileageRepository(jdbc);
}
@Test
void initializesTableWhenMissing() {
Integer count = jdbc.queryForObject("""
select count(*)
from information_schema.tables
where table_name = 'vehicle_daily_mileage_jt808'
""", Integer.class);
assertThat(count).isEqualTo(1);
}
@Test
void upsertsDailyMileageResult() {
Jt808DailyMileageResult first = new Jt808DailyMileageResult(
@@ -58,8 +48,12 @@ class JdbcJt808DailyMileageRepositoryTest {
Instant.parse("2026-06-30T00:00:00Z"),
Instant.parse("2026-06-30T00:01:00Z"),
1.23456,
"GPS_DISTANCE",
1.23456,
1.2,
null,
null,
null,
2,
0,
0,
@@ -73,9 +67,13 @@ class JdbcJt808DailyMileageRepositoryTest {
"13900000001",
Instant.parse("2026-06-30T00:00:00Z"),
Instant.parse("2026-06-30T00:02:00Z"),
2.0,
"JT808_TOTAL_MILEAGE",
2.34567,
2.3,
2.0,
100.0,
102.0,
3,
1,
0,
@@ -87,14 +85,19 @@ class JdbcJt808DailyMileageRepositoryTest {
repository.upsert(updated);
Map<String, Object> row = jdbc.queryForMap("""
select vehicle_key, gps_mileage_km, odometer_mileage_km, accepted_points,
bad_jump_segments, data_quality
select vehicle_key, daily_mileage_km, mileage_source, gps_mileage_km,
odometer_mileage_km, first_total_mileage_km, last_total_mileage_km,
accepted_points, bad_jump_segments, data_quality
from vehicle_daily_mileage_jt808
where stat_date = '2026-06-30' and vehicle_key = 'VIN001'
""");
assertThat(row.get("vehicle_key")).isEqualTo("VIN001");
assertThat(row.get("daily_mileage_km").toString()).isEqualTo("2.000");
assertThat(row.get("mileage_source")).isEqualTo("JT808_TOTAL_MILEAGE");
assertThat(row.get("gps_mileage_km").toString()).isEqualTo("2.346");
assertThat(row.get("odometer_mileage_km").toString()).isEqualTo("2.000");
assertThat(row.get("first_total_mileage_km").toString()).isEqualTo("100.000");
assertThat(row.get("last_total_mileage_km").toString()).isEqualTo("102.000");
assertThat(row.get("accepted_points")).isEqualTo(3);
assertThat(row.get("bad_jump_segments")).isEqualTo(1);
assertThat(row.get("data_quality")).isEqualTo("PARTIAL");

View File

@@ -25,6 +25,10 @@ class Jt808DailyMileageStateTest {
assertThat(state.gpsMileageKm()).isZero();
assertThat(state.speedIntegralKm()).isZero();
assertThat(state.odometerMileageKm()).isZero();
assertThat(state.firstTotalMileageKm()).isEqualTo(100.0);
assertThat(state.lastTotalMileageKm()).isEqualTo(100.0);
assertThat(state.toResult().dailyMileageKm()).isZero();
assertThat(state.toResult().mileageSource()).isEqualTo("INSUFFICIENT");
assertThat(state.firstEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z"));
assertThat(state.lastEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z"));
}
@@ -41,6 +45,12 @@ class Jt808DailyMileageStateTest {
assertThat(state.gpsMileageKm()).isBetween(0.99, 1.01);
assertThat(state.speedIntegralKm()).isBetween(0.99, 1.01);
assertThat(state.odometerMileageKm()).isCloseTo(1.2, offset(0.000001));
assertThat(state.firstTotalMileageKm()).isEqualTo(100.0);
assertThat(state.lastTotalMileageKm()).isEqualTo(101.2);
assertThat(state.toResult().dailyMileageKm()).isCloseTo(1.2, offset(0.000001));
assertThat(state.toResult().mileageSource()).isEqualTo("JT808_TOTAL_MILEAGE");
assertThat(state.toResult().firstTotalMileageKm()).isEqualTo(100.0);
assertThat(state.toResult().lastTotalMileageKm()).isEqualTo(101.2);
assertThat(state.badJumpSegments()).isZero();
assertThat(state.longGapSegments()).isZero();
}
@@ -68,6 +78,36 @@ class Jt808DailyMileageStateTest {
assertThat(state.odometerMileageKm()).isZero();
assertThat(state.odometerAnomalies()).isEqualTo(1);
assertThat(state.toResult().dailyMileageKm()).isGreaterThan(0.9);
assertThat(state.toResult().mileageSource()).isEqualTo("GPS_DISTANCE");
}
@Test
void odometerJumpAboveConfiguredSpeedIsIgnoredAndCounted() {
Jt808DailyMileageState state = Jt808DailyMileageState.empty("VIN001", "VIN001", "13900000001",
LocalDate.of(2026, 6, 30));
state.apply(point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0, 100.0), calculator);
state.apply(point("2026-06-30T00:01:00Z", 120.000100, 30.000000, 36.0, 200.0), calculator);
assertThat(state.odometerMileageKm()).isZero();
assertThat(state.odometerAnomalies()).isEqualTo(1);
assertThat(state.toResult().dataQuality()).isEqualTo("PARTIAL");
}
@Test
void gpsDistanceIsFallbackWhenTotalMileageIsMissing() {
Jt808DailyMileageState state = Jt808DailyMileageState.empty("VIN001", "VIN001", "13900000001",
LocalDate.of(2026, 6, 30));
state.apply(point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0, null), calculator);
state.apply(point("2026-06-30T00:01:40Z", 120.010370, 30.000000, 36.0, null), calculator);
assertThat(state.toResult().dailyMileageKm()).isBetween(0.99, 1.01);
assertThat(state.toResult().mileageSource()).isEqualTo("GPS_DISTANCE");
assertThat(state.toResult().odometerMileageKm()).isNull();
assertThat(state.toResult().firstTotalMileageKm()).isNull();
assertThat(state.toResult().lastTotalMileageKm()).isNull();
}
private static Jt808LocationPoint point(String time,

View File

@@ -71,6 +71,18 @@ class Jt808GpsMileageCalculatorTest {
assertThat(result.gpsKm()).isZero();
}
@Test
void validatesPlausibleOdometerDeltaByConfiguredSpeed() {
assertThat(calculator.plausibleOdometerDelta(
point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0),
point("2026-06-30T00:01:00Z", 120.001000, 30.000000, 36.0),
2.0)).isTrue();
assertThat(calculator.plausibleOdometerDelta(
point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0),
point("2026-06-30T00:01:00Z", 120.001000, 30.000000, 36.0),
20.0)).isFalse();
}
private static Jt808LocationPoint point(String time, double longitude, double latitude, Double speedKmh) {
return new Jt808LocationPoint(
"VIN001",

View File

@@ -57,6 +57,21 @@ class Jt808LocationPointExtractorTest {
field("longitude", "118.901034")))).isEmpty();
}
@Test
void ignoresJt808NonLocationSnapshot() {
VehicleEnvelope envelope = envelope("JT808", "VIN001", "13900000001",
field("longitude", "118.901034"),
field("latitude", "31.946325"))
.toBuilder()
.setTelemetrySnapshot(TelemetrySnapshot.newBuilder()
.setEventType("REGISTER")
.addFields(field("longitude", "118.901034"))
.addFields(field("latitude", "31.946325")))
.build();
assertThat(extractor.extract(envelope)).isEmpty();
}
private static VehicleEnvelope envelope(String source, String vin, String phone, TelemetryField... fields) {
TelemetrySnapshot.Builder snapshot = TelemetrySnapshot.newBuilder().setEventType("LOCATION");
for (TelemetryField field : fields) {

View File

@@ -30,6 +30,8 @@ class Jt808MileageStreamProcessorTest {
assertThat(state.gpsMileageKm()).isGreaterThan(0.9);
assertThat(repository.results).hasSize(2);
assertThat(repository.results.getLast().vehicleKey()).isEqualTo("VIN001");
assertThat(repository.results.getLast().dailyMileageKm()).isEqualTo(1.0);
assertThat(repository.results.getLast().mileageSource()).isEqualTo("JT808_TOTAL_MILEAGE");
assertThat(repository.results.getLast().odometerMileageKm()).isEqualTo(1.0);
}

View File

@@ -36,11 +36,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
private static final TypeReference<Map<String, String>> STRING_MAP =
new TypeReference<>() {};
private static final String HEADER_VERSION = "event-records-v1";
private static final int VEHICLE_BUCKETS = 64;
private final Path root;
private final Path indexPath;
private final ZoneId partitionZone;
private final ObjectMapper objectMapper;
private final boolean writeParquetEnabled;
private volatile boolean indexInitialized;
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) {
@@ -48,6 +50,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
}
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) {
this(root, partitionZone, objectMapper, true);
}
public DuckDbParquetEventFileStore(Path root,
ZoneId partitionZone,
ObjectMapper objectMapper,
boolean writeParquetEnabled) {
if (root == null) {
throw new IllegalArgumentException("root must not be null");
}
@@ -55,6 +64,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
this.indexPath = this.root.resolve("events.duckdb");
this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone;
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
this.writeParquetEnabled = writeParquetEnabled;
}
@Override
@@ -67,16 +77,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
Partition partition = partition(record);
byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
}
for (Map.Entry<Partition, List<EventFileRecord>> entry : byPartition.entrySet()) {
writePartition(entry.getKey(), entry.getValue());
appendIndex(entry.getKey(), entry.getValue());
if (writeParquetEnabled) {
writePartitions(byPartition);
}
appendIndex(byPartition);
}
@Override
public List<EventFileRecord> query(EventFileQuery query) throws IOException {
// 有 VIN 时直接扫该车分区 parquet避免先查总索引再和车辆文件 join
if (query.vin() != null) {
// 有 VIN 且启用 Parquet 副本时,可直接扫分区文件;索引-only 模式走 DuckDB 主索引
if (query.vin() != null && writeParquetEnabled) {
return queryParquet(query);
}
// 无 VIN 的管理类查询走 DuckDB 索引,代价更高但不影响高频单车查询路径。
@@ -130,6 +140,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli());
}
if (query.cursorEventTime() != null) {
predicates.add(cursorPredicateSql(query));
}
// 这里的 SQL 只拼接经过 escape 的内部值;外部 rawArchiveUri 查询使用 PreparedStatement。
String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n";
String sql = """
@@ -182,6 +195,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
where.append(" AND event_time_ms <= ?\n");
}
if (query.cursorEventTime() != null) {
where.append(cursorPredicatePrepared(query.order()));
}
String sql = """
SELECT event_id, protocol, event_type, vin,
event_time_ms, ingest_time_ms,
@@ -209,6 +225,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
ps.setLong(index++, query.eventTimeTo().toEpochMilli());
}
if (query.cursorEventTime() != null) {
long eventTimeMs = query.cursorEventTime().toEpochMilli();
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, ingestTimeMs);
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, ingestTimeMs);
ps.setString(index++, query.cursorEventId());
}
ps.setInt(index, query.limit());
try (ResultSet rs = ps.executeQuery()) {
List<EventFileRecord> result = new ArrayList<>();
@@ -222,16 +248,43 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
}
}
private void writePartition(Partition partition, List<EventFileRecord> records) throws IOException {
Path dir = partition.dir(root);
Files.createDirectories(dir);
Path partFile = dir.resolve("events.parquet");
Path tempFile = dir.resolve("events-" + UUID.randomUUID() + ".tmp.parquet");
private static String cursorPredicateSql(EventFileQuery query) {
long eventTimeMs = query.cursorEventTime().toEpochMilli();
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
String eventId = escapeSql(query.cursorEventId());
String cmp = query.order() == EventFileQuery.Order.DESC ? "<" : ">";
return """
(
event_time_ms %s %d
OR (event_time_ms = %d AND ingest_time_ms %s %d)
OR (event_time_ms = %d AND ingest_time_ms = %d AND event_id %s '%s')
)
""".formatted(cmp, eventTimeMs, eventTimeMs, cmp, ingestTimeMs,
eventTimeMs, ingestTimeMs, cmp, eventId);
}
private static String cursorPredicatePrepared(EventFileQuery.Order order) {
String cmp = order == EventFileQuery.Order.DESC ? "<" : ">";
return """
AND (
event_time_ms %s ?
OR (event_time_ms = ? AND ingest_time_ms %s ?)
OR (event_time_ms = ? AND ingest_time_ms = ? AND event_id %s ?)
)
""".formatted(cmp, cmp, cmp);
}
private void writePartitions(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
if (recordsByPartition.isEmpty()) {
return;
}
List<Path> tempFiles = new ArrayList<>();
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
Statement statement = connection.createStatement()) {
statement.execute("""
CREATE TEMPORARY TABLE event_records (
partition_id INTEGER,
event_id VARCHAR,
protocol VARCHAR,
event_type VARCHAR,
@@ -246,54 +299,62 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
)
""");
try (PreparedStatement ps = connection.prepareStatement("""
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EventFileRecord record : records) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setString(6, record.eventTime().toString());
ps.setLong(7, record.ingestTime().toEpochMilli());
ps.setString(8, record.ingestTime().toString());
ps.setString(9, record.rawArchiveUri());
ps.setString(10, objectMapper.writeValueAsString(record.metadata()));
ps.setString(11, record.payloadJson());
ps.addBatch();
int partitionId = 0;
for (List<EventFileRecord> records : recordsByPartition.values()) {
for (EventFileRecord record : records) {
ps.setInt(1, partitionId);
ps.setString(2, record.eventId());
ps.setString(3, record.protocol().name());
ps.setString(4, record.eventType());
ps.setString(5, record.vin());
ps.setLong(6, record.eventTime().toEpochMilli());
ps.setString(7, record.eventTime().toString());
ps.setLong(8, record.ingestTime().toEpochMilli());
ps.setString(9, record.ingestTime().toString());
ps.setString(10, record.rawArchiveUri());
ps.setString(11, objectMapper.writeValueAsString(record.metadata()));
ps.setString(12, record.payloadJson());
ps.addBatch();
}
partitionId++;
}
ps.executeBatch();
}
if (Files.exists(partFile)) {
// DuckDB 不能原地追加 parquet先 UNION 旧文件和本批数据,再原子替换分区文件。
int partitionId = 0;
for (Partition partition : recordsByPartition.keySet()) {
Path dir = partition.dir(root);
Files.createDirectories(dir);
String fragmentId = UUID.randomUUID().toString();
Path partFile = dir.resolve("events-" + fragmentId + ".parquet");
Path tempFile = dir.resolve("events-" + fragmentId + ".tmp.parquet");
tempFiles.add(tempFile);
statement.execute("""
CREATE TEMPORARY TABLE combined_records AS
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM read_parquet('%s')
UNION ALL
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM event_records
""".formatted(escapeSql(partFile)));
statement.execute("COPY combined_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
} else {
statement.execute("COPY event_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
COPY (
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM event_records
WHERE partition_id = %d
) TO '%s' (FORMAT parquet)
""".formatted(partitionId, escapeSql(tempFile)));
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
partitionId++;
}
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (SQLException e) {
throw new IOException("write parquet event store failed: " + partFile, e);
throw new IOException("write parquet event store failed", e);
} finally {
Files.deleteIfExists(tempFile);
for (Path tempFile : tempFiles) {
Files.deleteIfExists(tempFile);
}
}
}
private void appendIndex(Partition partition, List<EventFileRecord> records) throws IOException {
private void appendIndex(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
ensureIndexInitialized();
try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) {
insertIndex(connection, partition, records);
insertIndex(connection, recordsByPartition);
} catch (SQLException e) {
throw new IOException("write duckdb event index failed", e);
}
@@ -364,23 +425,26 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
""".formatted(fileList));
}
private void insertIndex(Connection connection, Partition partition, List<EventFileRecord> records)
private void insertIndex(Connection connection, Map<Partition, List<EventFileRecord>> recordsByPartition)
throws SQLException, IOException {
try (PreparedStatement ps = connection.prepareStatement("""
INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EventFileRecord record : records) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setLong(6, record.ingestTime().toEpochMilli());
ps.setString(7, partition.date().toString());
ps.setString(8, record.rawArchiveUri());
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
ps.setString(10, record.payloadJson());
ps.addBatch();
for (Map.Entry<Partition, List<EventFileRecord>> entry : recordsByPartition.entrySet()) {
Partition partition = entry.getKey();
for (EventFileRecord record : entry.getValue()) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setLong(6, record.ingestTime().toEpochMilli());
ps.setString(7, partition.date().toString());
ps.setString(8, record.rawArchiveUri());
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
ps.setString(10, record.payloadJson());
ps.addBatch();
}
}
ps.executeBatch();
}
@@ -390,15 +454,22 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
List<Path> files = new ArrayList<>();
LocalDate date = query.dateFrom();
while (!date.isAfter(query.dateTo())) {
Path dir = query.vin() == null
? partitionDir(query.protocol(), date)
: partitionDir(query.protocol(), date).resolve("vehicle=" + storageName(query.vin()));
// 目录结构把 VIN 放在 date 下,单车多日查询只需要遍历目标日期内的目标车辆目录。
if (Files.isDirectory(dir)) {
try (Stream<Path> stream = Files.walk(dir)) {
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
.sorted()
.forEach(files::add);
Path dateDir = partitionDir(query.protocol(), date);
List<Path> candidateDirs = query.vin() == null
? List.of(dateDir)
: List.of(
// 新布局:高写入路径按协议/日期写一个批次文件VIN 由 Parquet/索引字段过滤。
dateDir.resolve("header=" + HEADER_VERSION),
// 兼容已落盘的旧布局。
dateDir.resolve("bucket=" + bucketName(query.vin())),
dateDir.resolve("vehicle=" + storageName(query.vin())));
for (Path dir : candidateDirs) {
if (Files.isDirectory(dir)) {
try (Stream<Path> stream = Files.walk(dir)) {
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
.sorted()
.forEach(files::add);
}
}
}
date = date.plusDays(1);
@@ -434,8 +505,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
private Partition partition(EventFileRecord record) {
return new Partition(
record.protocol(),
LocalDate.ofInstant(record.eventTime(), partitionZone),
storageName(record.vin()));
LocalDate.ofInstant(record.eventTime(), partitionZone));
}
private Path partitionDir(ProtocolId protocol, LocalDate date) {
@@ -488,11 +558,18 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
return vin.replaceAll("[^A-Za-z0-9._-]", "_");
}
private record Partition(ProtocolId protocol, LocalDate date, String vehicle) {
private static String bucketName(String vin) {
String storageName = storageName(vin);
if ("_unknown".equals(storageName)) {
return "_unknown";
}
return "%02d".formatted(Math.floorMod(storageName.hashCode(), VEHICLE_BUCKETS));
}
private record Partition(ProtocolId protocol, LocalDate date) {
private Path dir(Path root) {
return root.resolve("protocol=" + protocol.name())
.resolve("date=" + date)
.resolve("vehicle=" + vehicle)
.resolve("header=" + HEADER_VERSION);
}
}

View File

@@ -20,7 +20,10 @@ public record EventFileQuery(
Order order,
int limit,
String vin,
String eventType
String eventType,
Instant cursorEventTime,
Instant cursorIngestTime,
String cursorEventId
) {
public enum Order {
ASC,
@@ -45,6 +48,24 @@ public record EventFileQuery(
limit = limit <= 0 ? 100 : limit;
vin = vin == null || vin.isBlank() ? null : vin.trim();
eventType = eventType == null || eventType.isBlank() ? null : eventType.trim();
cursorEventId = cursorEventId == null || cursorEventId.isBlank() ? null : cursorEventId.trim();
if ((cursorEventTime == null || cursorIngestTime == null || cursorEventId == null)
&& !(cursorEventTime == null && cursorIngestTime == null && cursorEventId == null)) {
throw new IllegalArgumentException("cursor eventTime, ingestTime and eventId must be provided together");
}
}
public EventFileQuery(ProtocolId protocol,
LocalDate dateFrom,
LocalDate dateTo,
Instant eventTimeFrom,
Instant eventTimeTo,
Order order,
int limit,
String vin,
String eventType) {
this(protocol, dateFrom, dateTo, eventTimeFrom, eventTimeTo, order, limit,
vin, eventType, null, null, null);
}
public EventFileQuery(ProtocolId protocol,

View File

@@ -226,9 +226,9 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
return payload;
}
private static Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
String rawArchiveKey,
String rawArchiveUri) {
private Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
String rawArchiveKey,
String rawArchiveUri) throws IOException {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventId", raw.eventId());
payload.put("vin", raw.vin());
@@ -241,10 +241,18 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
payload.put("command", hex(raw.command()));
payload.put("infoType", hex(raw.infoType()));
payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length);
putParsedJson(payload, raw.parsedJson());
payload.put("metadata", raw.metadata());
return payload;
}
private void putParsedJson(Map<String, Object> payload, String parsedJson) throws IOException {
if (parsedJson == null || parsedJson.isBlank()) {
return;
}
payload.put("parsed", objectMapper.readTree(parsedJson));
}
private static String hex(int value) {
return "0x" + String.format("%04X", value);
}

View File

@@ -44,7 +44,8 @@ public class EventFileStoreAutoConfiguration {
return new DuckDbParquetEventFileStore(
Path.of(properties.getPath()),
ZoneId.of(properties.getZoneId()),
mapper);
mapper,
properties.isWriteParquetEnabled());
}
@Bean

View File

@@ -11,10 +11,10 @@ public class EventFileStoreProperties {
private boolean enabled = false;
/**
* Parquet 文件库根路径。
* 历史库根路径。
*
* <p>实际结构为 {@code protocol=GB32960/date=yyyy-MM-dd/vehicle=VIN/header=event-records-v1/events.parquet}
* 另有 {@code events.duckdb} sidecar index 放在根路径下
* <p>{@code events.duckdb} 是可查询主索引;开启 Parquet 副本时,还会在该根目录下按
* {@code protocol/date/header} 保存追加文件
*/
private String path = "./event-store";
@@ -39,6 +39,14 @@ public class EventFileStoreProperties {
*/
private long flushIntervalMillis = 1000;
/**
* 是否同步写 Parquet 副本。
*
* <p>高写入生产链路可以关闭,只保留 DuckDB JSON 主索引和 raw archive 引用;需要离线副本或
* 索引重建能力时再开启。
*/
private boolean writeParquetEnabled = true;
public boolean isEnabled() {
return enabled;
}
@@ -78,4 +86,12 @@ public class EventFileStoreProperties {
public void setFlushIntervalMillis(long flushIntervalMillis) {
this.flushIntervalMillis = flushIntervalMillis;
}
public boolean isWriteParquetEnabled() {
return writeParquetEnabled;
}
public void setWriteParquetEnabled(boolean writeParquetEnabled) {
this.writeParquetEnabled = writeParquetEnabled;
}
}

View File

@@ -1,6 +1,7 @@
package com.lingniu.ingest.eventfilestore;
import com.lingniu.ingest.api.ProtocolId;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -11,6 +12,7 @@ import java.time.LocalDate;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;
import static org.assertj.core.api.Assertions.assertThat;
@@ -28,7 +30,7 @@ class DuckDbParquetEventFileStoreTest {
store.appendAll(List.of(first, second));
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(2);
assertThat(parquetFiles(dateDir)).hasSize(1);
EventFileQuery ascending = new EventFileQuery(
ProtocolId.GB32960,
@@ -135,16 +137,13 @@ class DuckDbParquetEventFileStoreTest {
}
@Test
void appendsSameVehicleDayHeaderToOneParquetFile() throws Exception {
void appendsSameVehicleDayHeaderAsAppendOnlyParquetFragments() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME"));
store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
.resolve("date=2026-06-22")
.resolve("vehicle=VIN-SAME")
.resolve("header=event-records-v1");
assertThat(parquetFiles(vehicleDayDir)).containsExactly(vehicleDayDir.resolve("events.parquet"));
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(2);
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
@@ -159,19 +158,54 @@ class DuckDbParquetEventFileStoreTest {
.containsExactly("first", "second");
}
@Test
void writesManyVehiclesIntoBoundedDailyLayoutInsteadOfVehicleDirectories() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
List<EventFileRecord> records = IntStream.range(0, 80)
.mapToObj(index -> rawRecord(
"raw-" + index,
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
"VIN-" + index))
.toList();
store.appendAll(records);
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
try (var files = Files.list(dateDir)) {
List<String> partitionDirs = files
.filter(Files::isDirectory)
.map(path -> path.getFileName().toString())
.sorted()
.toList();
assertThat(partitionDirs).noneMatch(name -> name.startsWith("vehicle="));
assertThat(partitionDirs).hasSizeLessThan(records.size());
}
}
@Test
void writesOneDailyParquetFragmentForOneBatchAcrossManyVehicles() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
List<EventFileRecord> records = IntStream.range(0, 80)
.mapToObj(index -> rawRecord(
"daily-raw-" + index,
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
"VIN-DAILY-" + index))
.toList();
store.appendAll(records);
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(1);
}
@Test
void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
.resolve("date=2026-06-22")
.resolve("vehicle=VIN-INDEXED")
.resolve("header=event-records-v1");
try (var files = Files.list(vehicleDayDir)) {
for (Path file : files.filter(p -> p.getFileName().toString().endsWith(".parquet")).toList()) {
Files.delete(file);
}
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
for (Path file : parquetFiles(dateDir)) {
Files.delete(file);
}
EventFileQuery query = new EventFileQuery(
@@ -219,6 +253,31 @@ class DuckDbParquetEventFileStoreTest {
assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri());
}
@Test
void canQueryVinRecordsFromIndexWhenParquetWritingIsDisabled() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(
tempDir,
ZoneId.of("Asia/Shanghai"),
new ObjectMapper(),
false);
EventFileRecord record = rawRecord("index-only", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEX-ONLY");
store.append(record);
assertThat(parquetFiles(tempDir)).isEmpty();
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
10,
"VIN-INDEX-ONLY",
"RAW_ARCHIVE");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("index-only");
}
private static List<Path> parquetFiles(Path root) throws Exception {
try (var files = Files.walk(root)) {
return files.filter(p -> p.getFileName().toString().endsWith(".parquet"))

View File

@@ -20,12 +20,15 @@ class EventFileStoreAutoConfigurationTest {
contextRunner
.withPropertyValues(
"lingniu.ingest.event-file-store.enabled=true",
"lingniu.ingest.event-file-store.path=target/test-event-store")
"lingniu.ingest.event-file-store.path=target/test-event-store",
"lingniu.ingest.event-file-store.write-parquet-enabled=false")
.run(context -> {
assertThat(context).hasSingleBean(EventFileStore.class);
assertThat(context).hasSingleBean(EventFileStoreSink.class);
assertThat(context.getBean(EventFileStoreProperties.class).getPath())
.isEqualTo(Path.of("target/test-event-store").toString());
assertThat(context.getBean(EventFileStoreProperties.class).isWriteParquetEnabled())
.isFalse();
});
}

View File

@@ -92,6 +92,7 @@ public final class EnvelopeMapper {
.setUri(uri)
.setChecksum(checksum)
.setSizeBytes(size)
.setParsedJson(nullToEmpty(ra.parsedJson()))
.build());
b.setRawFrameFact(com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload.newBuilder()
.setFrameId(frameId)

View File

@@ -10,17 +10,19 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(KafkaEnvelopeConsumerRunner.class);
private final List<KafkaEnvelopeConsumerWorker> workers;
private final Supplier<List<KafkaEnvelopeConsumerWorker>> workersSupplier;
private final Duration pollTimeout;
private final Duration loopBackoff;
private final boolean autoStartup;
private final AtomicBoolean running = new AtomicBoolean(false);
private final AtomicBoolean closed = new AtomicBoolean(false);
private volatile List<KafkaEnvelopeConsumerWorker> workers;
private ExecutorService executor;
public KafkaEnvelopeConsumerRunner(List<KafkaEnvelopeConsumerWorker> workers,
@@ -30,14 +32,28 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (workers == null || workers.isEmpty()) {
throw new IllegalArgumentException("workers must not be empty");
}
this.workersSupplier = () -> List.copyOf(workers);
this.workers = List.copyOf(workers);
this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout;
this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff;
this.autoStartup = autoStartup;
}
public KafkaEnvelopeConsumerRunner(Supplier<List<KafkaEnvelopeConsumerWorker>> workersSupplier,
Duration pollTimeout,
Duration loopBackoff,
boolean autoStartup) {
if (workersSupplier == null) {
throw new IllegalArgumentException("workersSupplier must not be null");
}
this.workersSupplier = workersSupplier;
this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout;
this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff;
this.autoStartup = autoStartup;
}
public List<KafkaEnvelopeConsumerWorker> workers() {
return workers;
return workersOrCreate();
}
@Override
@@ -45,17 +61,35 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (!running.compareAndSet(false, true)) {
return;
}
List<KafkaEnvelopeConsumerWorker> activeWorkers = workersOrCreate();
// 每个 worker 一条后台线程,避免某个处理器阻塞时拖慢其他消费组。
executor = Executors.newFixedThreadPool(workers.size(), r -> {
executor = Executors.newFixedThreadPool(activeWorkers.size(), r -> {
Thread thread = new Thread(r, "kafka-envelope-consumer");
thread.setDaemon(true);
return thread;
});
for (KafkaEnvelopeConsumerWorker worker : workers) {
for (KafkaEnvelopeConsumerWorker worker : activeWorkers) {
executor.submit(() -> pollLoop(worker));
}
}
private List<KafkaEnvelopeConsumerWorker> workersOrCreate() {
List<KafkaEnvelopeConsumerWorker> current = workers;
if (current != null) {
return current;
}
synchronized (this) {
if (workers == null) {
List<KafkaEnvelopeConsumerWorker> created = workersSupplier.get();
if (created == null || created.isEmpty()) {
throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
}
workers = List.copyOf(created);
}
return workers;
}
}
private void pollLoop(KafkaEnvelopeConsumerWorker worker) {
while (running.get()) {
try {
@@ -121,7 +155,11 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (!closed.compareAndSet(false, true)) {
return;
}
for (KafkaEnvelopeConsumerWorker worker : workers) {
List<KafkaEnvelopeConsumerWorker> current = workers;
if (current == null) {
return;
}
for (KafkaEnvelopeConsumerWorker worker : current) {
worker.close();
}
}

View File

@@ -36,26 +36,26 @@ public final class KafkaEnvelopeConsumerWorker implements AutoCloseable {
public int pollOnce(Duration timeout) {
ConsumerRecords<String, byte[]> records = consumer.poll(timeout == null ? Duration.ZERO : timeout);
Map<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> batches = new LinkedHashMap<>();
Map<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> byProcessor = new LinkedHashMap<>();
int processed = 0;
for (ConsumerRecord<String, byte[]> record : records) {
EnvelopeConsumerProcessor processor = processorsByTopic.get(record.topic());
if (processor == null) {
// worker 可能订阅多个 topic没有显式绑定处理器的 topic 不参与提交语义。
continue;
}
batches.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord(
byProcessor.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord(
record.topic(),
record.partition(),
record.offset(),
record.key(),
record.value()));
processed++;
}
int processed = 0;
for (Map.Entry<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> entry : batches.entrySet()) {
for (Map.Entry<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> entry : byProcessor.entrySet()) {
// EnvelopeConsumerProcessor 内部会把解析或业务错误转成 DLQ 记录,
// 这里保持 Kafka worker 的职责单一:轮询、批量分发、成功后提交 offset。
entry.getKey().processAll(entry.getValue());
processed += entry.getValue().size();
// 这里保持 Kafka worker 的职责单一:轮询、分发、成功后提交 offset。
entry.getKey().processBatch(entry.getValue());
}
if (processed > 0) {
// commitSync 放在批次末尾,保证同一个 poll 批次内的消息按 Kafka offset 一起确认。

View File

@@ -1,6 +1,5 @@
package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
import org.apache.kafka.clients.producer.KafkaProducer;
@@ -8,15 +7,12 @@ import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.ByteArraySerializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
@@ -100,21 +96,4 @@ public class SinkMqAutoConfiguration {
return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq());
}
@Bean
@ConditionalOnMissingBean
@ConditionalOnBean(EnvelopeConsumerProcessor.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(Map<String, EnvelopeConsumerProcessor> processors,
SinkMqProperties props) {
// Consumer runner 根据 processor bean 名和配置 binding 生成 worker没有 binding 时直接失败,避免静默不消费。
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
if (workers.isEmpty()) {
throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
}
return new KafkaEnvelopeConsumerRunner(
workers,
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
props.getConsumer().isAutoStartup());
}
}

View File

@@ -0,0 +1,42 @@
package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import java.time.Duration;
import java.util.List;
import java.util.Map;
@AutoConfiguration(after = SinkMqAutoConfiguration.class)
@AutoConfigureAfter(name = {
"com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration",
"com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration",
"com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration"
})
@EnableConfigurationProperties(SinkMqProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
public class SinkMqConsumerAutoConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory,
SinkMqProperties props) {
return new KafkaEnvelopeConsumerRunner(
() -> createWorkers(beanFactory, props),
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
props.getConsumer().isAutoStartup());
}
private List<KafkaEnvelopeConsumerWorker> createWorkers(ListableBeanFactory beanFactory, SinkMqProperties props) {
Map<String, EnvelopeConsumerProcessor> processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class);
return new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
}
}

View File

@@ -45,6 +45,7 @@ message RawArchiveRef {
string uri = 1;
string checksum = 2;
int64 size_bytes = 3;
string parsed_json = 4;
}
message TelemetrySnapshot {

View File

@@ -1 +1,2 @@
com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration
com.lingniu.ingest.sink.mq.SinkMqConsumerAutoConfiguration

View File

@@ -1,6 +1,7 @@
package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
@@ -14,6 +15,7 @@ import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
@@ -44,4 +46,39 @@ class KafkaEnvelopeConsumerWorkerTest {
assertThat(record.payload()).containsExactly(0x01, 0x02);
});
}
@Test
void pollsKafkaRecordsAndDispatchesBatchWhenProcessorSupportsIt() {
MockConsumer<String, byte[]> consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST);
TopicPartition partition = new TopicPartition("vehicle.raw", 0);
consumer.assign(List.of(partition));
consumer.updateBeginningOffsets(Map.of(partition, 0L));
consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 12L, "VIN001", new byte[]{0x01}));
consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 13L, "VIN002", new byte[]{0x02}));
AtomicInteger batchCalls = new AtomicInteger();
EnvelopeConsumerProcessor processor = new EnvelopeConsumerProcessor(
"event-history",
new EnvelopeBatchIngestor() {
@Override
public EnvelopeIngestResult tryIngest(byte[] kafkaValue) {
throw new AssertionError("single-record ingest should not be used for a batch-capable processor");
}
@Override
public List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues) {
batchCalls.incrementAndGet();
assertThat(kafkaValues).hasSize(2);
return List.of(
EnvelopeIngestResult.stored("event-1", "VIN001"),
EnvelopeIngestResult.invalid("bad envelope"));
}
},
record -> {});
int processed = new KafkaEnvelopeConsumerWorker(
consumer, Map.of("vehicle.raw", processor)).pollOnce(Duration.ZERO);
assertThat(processed).isEqualTo(2);
assertThat(batchCalls).hasValue(1);
}
}

View File

@@ -10,7 +10,7 @@ import static org.assertj.core.api.Assertions.assertThat;
class SinkMqConsumerAutoConfigurationTest {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(SinkMqAutoConfiguration.class)
.withUserConfiguration(SinkMqAutoConfiguration.class, SinkMqConsumerAutoConfiguration.class)
.withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
() -> new EnvelopeConsumerProcessor(
"vehicle-state",

View File

@@ -43,6 +43,7 @@ public final class TdengineEnvelopeRows {
raw.getParseError(),
raw.getPeer(),
json(metadata),
parsedJson(envelope, metadata),
protocol(envelope),
vehicleKey(envelope, raw.getVehicleKey(), raw.getPhone()),
firstNonBlank(raw.getVin(), envelope.getVin()),
@@ -73,7 +74,6 @@ public final class TdengineEnvelopeRows {
location.getStatusFlag(),
totalMileage(envelope),
rawUri,
json(envelope.getMetadataMap()),
protocol(envelope),
vehicleKey(envelope, envelope.getMetadataOrDefault("vehicle_key", ""),
envelope.getMetadataOrDefault("phone", "")),
@@ -188,6 +188,13 @@ public final class TdengineEnvelopeRows {
return Double.parseDouble(value);
}
private static String parsedJson(VehicleEnvelope envelope, Map<String, String> metadata) {
if (envelope.hasRawArchive() && !envelope.getRawArchive().getParsedJson().isBlank()) {
return envelope.getRawArchive().getParsedJson();
}
return firstNonBlank(metadata.get("parsedJson"), metadata.get("parsed_json"));
}
private static Double valueDouble(TelemetryField field) {
String valueType = field.getValueType();
if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) {

View File

@@ -6,11 +6,11 @@ import java.util.List;
public final class TdengineHistoryQueries {
private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json, "
+ "protocol, vehicle_key, vin, phone";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone";
+ "protocol, vehicle_key, vin, phone";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, "
+ "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone";

View File

@@ -64,7 +64,8 @@ public final class TdengineHistorySchema {
parse_status NCHAR(16),
parse_error NCHAR(512),
peer NCHAR(128),
metadata_json NCHAR(4096)
metadata_json NCHAR(4096),
parsed_json NCHAR(16374)
) TAGS (
protocol NCHAR(16),
vehicle_key NCHAR(128),
@@ -88,8 +89,7 @@ public final class TdengineHistorySchema {
alarm_flag BIGINT,
status_flag BIGINT,
total_mileage_km DOUBLE,
raw_uri NCHAR(512),
metadata_json NCHAR(4096)
raw_uri NCHAR(512)
) TAGS (
protocol NCHAR(16),
vehicle_key NCHAR(128),

View File

@@ -6,12 +6,12 @@ import java.util.Arrays;
public final class TdengineHistoryStatements {
private static final String RAW_FRAME_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json";
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json";
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, metadata_json";
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri";
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, "
+ "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json";
@@ -35,7 +35,7 @@ public final class TdengineHistoryStatements {
values(
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
row.parseError(), row.peer(), row.metadataJson()
row.parseError(), row.peer(), row.metadataJson(), row.parsedJson()
)
);
}
@@ -49,7 +49,7 @@ public final class TdengineHistoryStatements {
values(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
row.totalMileageKm(), row.rawUri(), row.metadataJson()
row.totalMileageKm(), row.rawUri()
)
);
}

View File

@@ -157,6 +157,7 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
rs.getString("parse_error"),
rs.getString("peer"),
rs.getString("metadata_json"),
rs.getString("parsed_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),
@@ -179,7 +180,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
rs.getLong("status_flag"),
nullableDouble(rs, "total_mileage_km"),
rs.getString("raw_uri"),
rs.getString("metadata_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),

View File

@@ -16,7 +16,6 @@ public record TdengineLocationRow(
long statusFlag,
Double totalMileageKm,
String rawUri,
String metadataJson,
String protocol,
String vehicleKey,
String vin,

View File

@@ -16,6 +16,7 @@ public record TdengineRawFrameRow(
String parseError,
String peer,
String metadataJson,
String parsedJson,
String protocol,
String vehicleKey,
String vin,

View File

@@ -27,6 +27,7 @@ class TdengineHistoryStatementsTest {
"",
"10.0.0.1:808",
"{\"auth\":\"passed\"}",
"{\"body\":{\"speedKmh\":42.0}}",
"JT808",
"jt808:g7gps",
"VIN123",
@@ -39,13 +40,13 @@ class TdengineHistoryStatementsTest {
.contains(" USING raw_frames TAGS ('JT808', 'jt808:g7gps', 'VIN123', '013800000000')");
assertThat(batch.insertSql())
.startsWith("INSERT INTO raw_jt808_")
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json)")
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json)")
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
assertThat(batch.values())
.containsExactly(
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
row.parseError(), row.peer(), row.metadataJson());
row.parseError(), row.peer(), row.metadataJson(), row.parsedJson());
}
@Test
@@ -64,7 +65,6 @@ class TdengineHistoryStatementsTest {
3,
null,
"archive://gb32960/frame-1.bin",
"{}",
"GB32960",
"vin:O'HARE",
"O'HARE",
@@ -79,7 +79,7 @@ class TdengineHistoryStatementsTest {
.containsExactly(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
row.totalMileageKm(), row.rawUri(), row.metadataJson());
row.totalMileageKm(), row.rawUri());
}
@Test

View File

@@ -74,7 +74,6 @@ class TdengineJdbcHistoryWriterTest {
3,
null,
"archive://gb32960/frame-1.bin",
"{}",
"GB32960",
"vin:VIN123",
"VIN123",
@@ -217,6 +216,7 @@ class TdengineJdbcHistoryWriterTest {
parseError,
"10.0.0.1:808",
"{}",
"{\"frame\":\"" + frameId + "\"}",
"JT808",
"jt808:g7gps",
"VIN123",

View File

@@ -65,6 +65,7 @@
<resilience4j.version>2.2.0</resilience4j.version>
<micrometer.version>1.13.6</micrometer.version>
<otel.version>1.42.1</otel.version>
<paho-mqtt.version>1.2.5</paho-mqtt.version>
<fusesource-mqtt.version>1.12</fusesource-mqtt.version>
<duckdb.version>1.1.3</duckdb.version>
<taos-jdbcdriver.version>3.8.4</taos-jdbcdriver.version>
@@ -306,6 +307,11 @@
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.paho</groupId>
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
<version>${paho-mqtt.version}</version>
</dependency>
<dependency>
<groupId>org.fusesource.mqtt-client</groupId>
<artifactId>mqtt-client</artifactId>