diff --git a/Dockerfile b/Dockerfile index 79786696..626ef2d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,6 @@ ENV JAVA_OPTS="" WORKDIR /app COPY modules/apps/${APP_NAME}/target/${APP_NAME}.jar /app/app.jar -EXPOSE 20100 20200 20300 20400 32960 10808 +EXPOSE 20100 20200 20300 20400 20500 20600 32960 10808 ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app/app.jar"] diff --git a/deploy/portainer/docker-compose.yml b/deploy/portainer/docker-compose.yml index 9180db08..9bece17a 100644 --- a/deploy/portainer/docker-compose.yml +++ b/deploy/portainer/docker-compose.yml @@ -4,7 +4,7 @@ version: "3.8" # main-0.1.0-SNAPSHOT-22468e7 x-common-env: &common-env TZ: Asia/Shanghai - JAVA_OPTS: ${JAVA_OPTS:--Xms128m -Xmx768m -Duser.timezone=Asia/Shanghai} + JAVA_OPTS: ${JAVA_OPTS:--XX:+UseZGC -XX:MaxRAMPercentage=75} NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true} NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848} NACOS_NAMESPACE: ${NACOS_NAMESPACE:-} @@ -13,32 +13,33 @@ x-common-env: &common-env NACOS_REFRESH_ENABLED: ${NACOS_REFRESH_ENABLED:-true} NACOS_USERNAME: ${NACOS_USERNAME:-} NACOS_PASSWORD: ${NACOS_PASSWORD:-} - KAFKA_BROKERS: ${KAFKA_BROKERS:-114.55.58.251:9092} + KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092} KAFKA_ENABLED: ${KAFKA_ENABLED:-true} KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true} + KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:-900000} KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1} KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1} KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1} KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1} KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1} KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1} - KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1} - KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1} - KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1} - KAFKA_TOPIC_XINDA_PUSH_EVENT: ${KAFKA_TOPIC_XINDA_PUSH_EVENT:-vehicle.event.xinda-push.v1} - KAFKA_TOPIC_XINDA_PUSH_RAW: ${KAFKA_TOPIC_XINDA_PUSH_RAW:-vehicle.raw.xinda-push.v1} - KAFKA_TOPIC_XINDA_PUSH_DLQ: ${KAFKA_TOPIC_XINDA_PUSH_DLQ:-vehicle.dlq.xinda-push.v1} + KAFKA_TOPIC_MQTT_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1} + KAFKA_TOPIC_MQTT_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1} + KAFKA_TOPIC_MQTT_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1} + KAFKA_TOPIC_XINDA_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1} + KAFKA_TOPIC_XINDA_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1} + KAFKA_TOPIC_XINDA_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1} + MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-} + MYSQL_USERNAME: ${MYSQL_USERNAME:-} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:-} services: gb32960-ingest-app: image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} container_name: gb32960-ingest-app restart: unless-stopped - mem_limit: 1536m - memswap_limit: 1536m environment: <<: *common-env - JAVA_OPTS: ${JAVA_OPTS_GB32960:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai} HTTP_PORT: 20100 GB32960_PORT: 32960 KAFKA_CONSUMER_ENABLED: "false" @@ -48,13 +49,11 @@ services: GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-} GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-} GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false} - SINK_ARCHIVE_PATH: /archive/ VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl ports: - "${GB32960_HTTP_PORT:-20100}:20100" - "${GB32960_TCP_PORT:-32960}:32960" volumes: - - vehicle-history-archive:/archive - gb32960-ingest-data:/data networks: - vehicle-ingest @@ -63,109 +62,82 @@ services: image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} container_name: jt808-ingest-app restart: unless-stopped - mem_limit: 1536m - memswap_limit: 1536m environment: <<: *common-env - JAVA_OPTS: ${JAVA_OPTS_JT808:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai} - HTTP_PORT: 20482 + HTTP_PORT: 20400 JT808_PORT: 808 KAFKA_CONSUMER_ENABLED: "false" KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer} - SINK_ARCHIVE_PATH: /archive/ - VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file} - VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl - VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding} - VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-} - VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-} - VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-} - VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s} + VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql} + VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-} + VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-} + VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-} ports: - - "${JT808_HTTP_PORT:-20482}:20482" + - "${JT808_HTTP_PORT:-20400}:20400" - "${JT808_TCP_PORT:-808}:808" volumes: - - vehicle-history-archive:/archive - jt808-ingest-data:/data networks: - vehicle-ingest yutong-mqtt-app: - image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} + image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} container_name: yutong-mqtt-app restart: unless-stopped - mem_limit: 1536m - memswap_limit: 1536m environment: <<: *common-env - JAVA_OPTS: ${JAVA_OPTS_YUTONG_MQTT:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai} HTTP_PORT: 20500 + MQTT_ENABLED: "true" + XINDA_PUSH_ENABLED: "false" KAFKA_CONSUMER_ENABLED: "false" - KAFKA_NODE_ID: ${KAFKA_NODE_ID_YUTONG_MQTT:-yutong-mqtt-portainer} - YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false} - YUTONG_MQTT_ENDPOINT_NAME: ${YUTONG_MQTT_ENDPOINT_NAME:-yutong} - YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-} - YUTONG_MQTT_TOPIC: ${YUTONG_MQTT_TOPIC:-#} - YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-1} - YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-yutong-mqtt} - YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-} - YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-} - YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false} - YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20} - YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10} - YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-} - YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-} - YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-} - SINK_ARCHIVE_PATH: /archive/ - VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file} - VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl - VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding} - VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-} - VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-} - VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-} - VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s} + KAFKA_NODE_ID: ${KAFKA_NODE_ID_MQTT:-yutong-mqtt-portainer} + KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1} + KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1} + KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1} + VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql} + VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-} + VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-} + VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-} + MQTT_ENDPOINT_NAME: ${MQTT_ENDPOINT_NAME:-yutong} + MQTT_URI: ${MQTT_URI:-tcp://127.0.0.1:1883} + MQTT_TOPIC: "${MQTT_TOPIC:-#}" + MQTT_QOS: ${MQTT_QOS:-1} + MQTT_CLIENT_ID: ${MQTT_CLIENT_ID:-lingniu-yutong-mqtt} + MQTT_USERNAME: ${MQTT_USERNAME:-} + MQTT_PASSWORD: ${MQTT_PASSWORD:-} + MQTT_CLEAN_SESSION: ${MQTT_CLEAN_SESSION:-true} + MQTT_PROFILE: ${MQTT_PROFILE:-yutong} ports: - "${YUTONG_MQTT_HTTP_PORT:-20500}:20500" - volumes: - - vehicle-history-archive:/archive - - yutong-mqtt-data:/data networks: - vehicle-ingest xinda-push-app: - image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/xinda-push-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} + image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} container_name: xinda-push-app restart: unless-stopped - mem_limit: 1536m - memswap_limit: 1536m environment: <<: *common-env - JAVA_OPTS: ${JAVA_OPTS_XINDA_PUSH:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai} HTTP_PORT: 20600 + MQTT_ENABLED: "false" + XINDA_PUSH_ENABLED: "true" KAFKA_CONSUMER_ENABLED: "false" - KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA_PUSH:-xinda-push-portainer} - XINDA_PUSH_ENABLED: ${XINDA_PUSH_ENABLED:-false} + KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA:-xinda-push-portainer} + KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1} + KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1} + KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1} + VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql} + VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-} + VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-} + VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-} XINDA_PUSH_HOST: ${XINDA_PUSH_HOST:-} XINDA_PUSH_PORT: ${XINDA_PUSH_PORT:-10100} XINDA_PUSH_USERNAME: ${XINDA_PUSH_USERNAME:-} XINDA_PUSH_PASSWORD: ${XINDA_PUSH_PASSWORD:-} XINDA_PUSH_CLIENT_DESC: ${XINDA_PUSH_CLIENT_DESC:-lingniu-ingest} - XINDA_PUSH_MSG_ID_LOCATION: ${XINDA_PUSH_MSG_ID_LOCATION:-0200} - XINDA_PUSH_MSG_ID_ALARM: ${XINDA_PUSH_MSG_ID_ALARM:-0300} - XINDA_PUSH_MSG_ID_PASSTHROUGH: ${XINDA_PUSH_MSG_ID_PASSTHROUGH:-0401} - XINDA_PUSH_RECONNECT_INTERVAL_SEC: ${XINDA_PUSH_RECONNECT_INTERVAL_SEC:-5} - SINK_ARCHIVE_PATH: /archive/ - VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file} - VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl - VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding} - VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-} - VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-} - VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-} - VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s} + XINDA_PUSH_SUBSCRIBE_MSG_IDS: "${XINDA_PUSH_SUBSCRIBE_MSG_IDS:-0200,0300,0401}" ports: - "${XINDA_PUSH_HTTP_PORT:-20600}:20600" - volumes: - - vehicle-history-archive:/archive - - xinda-push-data:/data networks: - vehicle-ingest @@ -173,8 +145,6 @@ services: image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} container_name: vehicle-history-app restart: unless-stopped - mem_limit: 2560m - memswap_limit: 2560m depends_on: - gb32960-ingest-app - jt808-ingest-app @@ -182,29 +152,28 @@ services: - xinda-push-app environment: <<: *common-env - JAVA_OPTS: ${JAVA_OPTS_HISTORY:--Xms512m -Xmx1536m -Duser.timezone=Asia/Shanghai} HTTP_PORT: 20200 KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history} + KAFKA_CONSUMER_MAX_POLL_RECORDS: ${KAFKA_CONSUMER_MAX_POLL_RECORDS_HISTORY:-2000} KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history} - KAFKA_CONSUMER_CONCURRENCY: ${KAFKA_CONSUMER_CONCURRENCY:-3} + KAFKA_GROUP_HISTORY_GB32960: ${KAFKA_GROUP_HISTORY_GB32960:-vehicle-history} + KAFKA_GROUP_HISTORY_JT808: ${KAFKA_GROUP_HISTORY_JT808:-vehicle-history} + KAFKA_GROUP_HISTORY_MQTT_YUTONG: ${KAFKA_GROUP_HISTORY_MQTT_YUTONG:-vehicle-history} + KAFKA_GROUP_HISTORY_XINDA: ${KAFKA_GROUP_HISTORY_XINDA:-vehicle-history} SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true} SINK_ARCHIVE_PATH: /archive/ - # 高吞吐生产热路径默认只写 TDengine。需要 GB32960 snapshot/fields 旧查询时再显式开启。 - EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-false} + EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true} EVENT_FILE_STORE_PATH: /event-store/ EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai} - TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true} - TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts} - TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://tdengine:6041/vehicle_ts} - TDENGINE_DRIVER_CLASS_NAME: ${TDENGINE_DRIVER_CLASS_NAME:-com.taosdata.jdbc.ws.WebSocketDriver} + EVENT_FILE_STORE_BATCH_SIZE: ${EVENT_FILE_STORE_BATCH_SIZE:-500} + EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS: ${EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS:-1000} + EVENT_FILE_STORE_WRITE_PARQUET_ENABLED: ${EVENT_FILE_STORE_WRITE_PARQUET_ENABLED:-false} + TDENGINE_ENABLED: ${TDENGINE_ENABLED:-true} + TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts} + TDENGINE_DATABASE: ${TDENGINE_DATABASE:-vehicle_ts} + TDENGINE_JT808_LOCATION_STABLE: ${TDENGINE_JT808_LOCATION_STABLE:-jt808_locations} TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root} TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata} - TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32} - TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-0} - TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-5000} - TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:--1} - # 默认关闭逐字段宽表，避免 telemetry_fields 写放大；需要字段趋势查询时显式打开。 - TDENGINE_TELEMETRY_FIELDS_ENABLED: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:-false} ports: - "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200" volumes: @@ -217,16 +186,11 @@ services: image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION} container_name: vehicle-analytics-app restart: unless-stopped - mem_limit: 1536m - memswap_limit: 1536m depends_on: - gb32960-ingest-app - jt808-ingest-app - - yutong-mqtt-app - - xinda-push-app environment: <<: *common-env - JAVA_OPTS: ${JAVA_OPTS_ANALYTICS:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai} HTTP_PORT: 20300 KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics} KAFKA_GROUP_STATE: ${KAFKA_GROUP_STATE:-vehicle-state} @@ -244,6 +208,7 @@ services: REDIS_HOST: ${REDIS_HOST:-} REDIS_PORT: ${REDIS_PORT:-6379} REDIS_DATABASE: ${REDIS_DATABASE:-50} + REDIS_USERNAME: ${REDIS_USERNAME:-} REDIS_PASSWORD: ${REDIS_PASSWORD:-} ports: - "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300" @@ -259,8 +224,6 @@ networks: volumes: gb32960-ingest-data: jt808-ingest-data: - yutong-mqtt-data: - xinda-push-data: vehicle-history-archive: vehicle-history-event-store: vehicle-analytics-stat: diff --git a/deploy/tools/EnsureKafkaTopics.java b/deploy/tools/EnsureKafkaTopics.java new file mode 100644 index 00000000..85d4012a --- /dev/null +++ b/deploy/tools/EnsureKafkaTopics.java @@ -0,0 +1,92 @@ +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewPartitions; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; + +public final class EnsureKafkaTopics { + private static final short REPLICATION_FACTOR = 1; + + private record TopicSpec(String name, int partitions) {} + + private EnsureKafkaTopics() {} + + public static void main(String[] args) throws Exception { + String brokers = args.length > 0 ? args[0] : "172.17.111.56:9092"; + List specs = + List.of( + topic("vehicle.event.gb32960.v1", 12), + topic("vehicle.raw.gb32960.v1", 12), + topic("vehicle.dlq.gb32960.v1", 3), + topic("vehicle.event.jt808.v1", 12), + topic("vehicle.raw.jt808.v1", 12), + topic("vehicle.dlq.jt808.v1", 3), + topic("vehicle.event.mqtt-yutong.v1", 12), + topic("vehicle.raw.mqtt-yutong.v1", 12), + topic("vehicle.dlq.mqtt-yutong.v1", 3), + topic("vehicle.event.xinda.v1", 12), + topic("vehicle.raw.xinda.v1", 12), + topic("vehicle.dlq.xinda.v1", 3), + topic("vehicle.event.xinda-push.v1", 12), + topic("vehicle.raw.xinda-push.v1", 12), + topic("vehicle.dlq.xinda-push.v1", 3)); + + Properties props = new Properties(); + props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); + props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "15000"); + props.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "30000"); + + try (AdminClient admin = AdminClient.create(props)) { + Set existing = admin.listTopics().names().get(); + List missing = + specs.stream() + .filter(spec -> !existing.contains(spec.name())) + .map(spec -> new NewTopic(spec.name(), spec.partitions(), REPLICATION_FACTOR)) + .toList(); + if (!missing.isEmpty()) { + try { + admin.createTopics(missing).all().get(); + } catch (ExecutionException e) { + System.out.println("createTopics result=" + e.getCause()); + } + } + + Map byName = + specs.stream().collect(Collectors.toMap(TopicSpec::name, spec -> spec)); + Map descriptions = + admin.describeTopics(byName.keySet()).allTopicNames().get(); + + Map increases = + descriptions.entrySet().stream() + .filter(entry -> entry.getValue().partitions().size() < byName.get(entry.getKey()).partitions()) + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> NewPartitions.increaseTo(byName.get(entry.getKey()).partitions()))); + if (!increases.isEmpty()) { + admin.createPartitions(increases).all().get(); + Thread.sleep(Duration.ofSeconds(2).toMillis()); + descriptions = admin.describeTopics(byName.keySet()).allTopicNames().get(); + } + + List lines = new ArrayList<>(); + descriptions.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> lines.add(entry.getKey() + " partitions=" + entry.getValue().partitions().size())); + System.out.println("bootstrap=" + brokers); + lines.forEach(System.out::println); + } + } + + private static TopicSpec topic(String name, int partitions) { + return new TopicSpec(name, partitions); + } +} diff --git a/modules/apps/jt808-ingest-app/src/main/resources/application.yml b/modules/apps/jt808-ingest-app/src/main/resources/application.yml index 88606230..3b849c94 100644 --- a/modules/apps/jt808-ingest-app/src/main/resources/application.yml +++ b/modules/apps/jt808-ingest-app/src/main/resources/application.yml @@ -52,12 +52,11 @@ lingniu: file: path: ${VEHICLE_IDENTITY_FILE:./data/vehicle-identity.jsonl} mysql: - table: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding} - jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:} - username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:} + jdbc-url: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai} + username: ${VEHICLE_IDENTITY_MYSQL_USERNAME:root} password: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:} - driver-class-name: ${VEHICLE_IDENTITY_MYSQL_DRIVER_CLASS_NAME:com.mysql.cj.jdbc.Driver} - refresh-interval: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s} + table-name: ${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings} + initialize-schema: ${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true} sink: mq: enabled: ${KAFKA_ENABLED:true} @@ -79,10 +78,6 @@ lingniu: dlq: ${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1} consumer: enabled: false - archive: - enabled: ${SINK_ARCHIVE_ENABLED:true} - type: local - path: ${SINK_ARCHIVE_PATH:./archive/} event-file-store: enabled: false event-history: diff --git a/modules/apps/jt808-ingest-app/src/test/java/com/lingniu/ingest/jt808app/Jt808IngestAppDefaultsTest.java b/modules/apps/jt808-ingest-app/src/test/java/com/lingniu/ingest/jt808app/Jt808IngestAppDefaultsTest.java index 458b4258..5e292ef3 100644 --- a/modules/apps/jt808-ingest-app/src/test/java/com/lingniu/ingest/jt808app/Jt808IngestAppDefaultsTest.java +++ b/modules/apps/jt808-ingest-app/src/test/java/com/lingniu/ingest/jt808app/Jt808IngestAppDefaultsTest.java @@ -29,12 +29,14 @@ class Jt808IngestAppDefaultsTest { .containsEntry("lingniu.ingest.sink.mq.topics.raw-archive", "${KAFKA_TOPIC_JT808_RAW:vehicle.raw.jt808.v1}") .containsEntry("lingniu.ingest.sink.mq.topics.dlq", "${KAFKA_TOPIC_JT808_DLQ:vehicle.dlq.jt808.v1}") .containsEntry("lingniu.ingest.identity.store", "${VEHICLE_IDENTITY_STORE:file}") - .containsEntry("lingniu.ingest.identity.mysql.table", "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_binding}") - .containsEntry("lingniu.ingest.identity.mysql.jdbc-url", "${VEHICLE_IDENTITY_MYSQL_JDBC_URL:}") - .containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:}") - .containsEntry("lingniu.ingest.identity.mysql.refresh-interval", "${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:60s}") - .containsEntry("lingniu.ingest.sink.archive.enabled", "${SINK_ARCHIVE_ENABLED:true}") - .containsEntry("lingniu.ingest.sink.archive.path", "${SINK_ARCHIVE_PATH:./archive/}") + .containsEntry("lingniu.ingest.identity.mysql.table-name", + "${VEHICLE_IDENTITY_MYSQL_TABLE:vehicle_identity_bindings}") + .containsEntry("lingniu.ingest.identity.mysql.jdbc-url", + "${VEHICLE_IDENTITY_MYSQL_JDBC_URL:jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai}") + .containsEntry("lingniu.ingest.identity.mysql.username", "${VEHICLE_IDENTITY_MYSQL_USERNAME:root}") + .containsEntry("lingniu.ingest.identity.mysql.password", "${VEHICLE_IDENTITY_MYSQL_PASSWORD:}") + .containsEntry("lingniu.ingest.identity.mysql.initialize-schema", + "${VEHICLE_IDENTITY_MYSQL_INITIALIZE_SCHEMA:true}") .containsEntry("lingniu.ingest.event-file-store.enabled", false) .containsEntry("lingniu.ingest.event-history.enabled", false) .containsEntry("lingniu.ingest.vehicle-state.enabled", false) diff --git a/modules/apps/vehicle-analytics-app/pom.xml b/modules/apps/vehicle-analytics-app/pom.xml index c5414f46..4f6e9f71 100644 --- a/modules/apps/vehicle-analytics-app/pom.xml +++ b/modules/apps/vehicle-analytics-app/pom.xml @@ -33,6 +33,10 @@ com.lingniu.ingest vehicle-stat-service + + org.springframework.boot + spring-boot-starter-jdbc + org.springframework.boot spring-boot-starter-web diff --git a/modules/apps/vehicle-analytics-app/src/main/resources/application.yml b/modules/apps/vehicle-analytics-app/src/main/resources/application.yml index ac3bab4d..6144fa6f 100644 --- a/modules/apps/vehicle-analytics-app/src/main/resources/application.yml +++ b/modules/apps/vehicle-analytics-app/src/main/resources/application.yml @@ -26,6 +26,7 @@ spring: redis: host: ${REDIS_HOST:} port: ${REDIS_PORT:6379} + username: ${REDIS_USERNAME:} password: ${REDIS_PASSWORD:} database: ${REDIS_DATABASE:50} @@ -55,6 +56,7 @@ lingniu: client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-analytics} auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest} max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:500} + max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:900000} bindings: vehicleStateEnvelopeConsumerProcessor: enabled: ${VEHICLE_STATE_ENABLED:false} diff --git a/modules/apps/vehicle-history-app/src/main/resources/application.yml b/modules/apps/vehicle-history-app/src/main/resources/application.yml index 08d52973..d4f24f9b 100644 --- a/modules/apps/vehicle-history-app/src/main/resources/application.yml +++ b/modules/apps/vehicle-history-app/src/main/resources/application.yml @@ -46,7 +46,7 @@ lingniu: enabled: ${KAFKA_CONSUMER_ENABLED:true} client-id-prefix: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX:vehicle-history} auto-offset-reset: ${KAFKA_CONSUMER_AUTO_OFFSET_RESET:earliest} - max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:100} + max-poll-records: ${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000} max-poll-interval-millis: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS:1800000} concurrency: ${KAFKA_CONSUMER_CONCURRENCY:3} bindings: diff --git a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java index 493ec4cf..3eb5dd99 100644 --- a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java +++ b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/VehicleHistoryAppDefaultsTest.java @@ -43,7 +43,7 @@ class VehicleHistoryAppDefaultsTest { .containsEntry("lingniu.ingest.vehicle-state.enabled", false) .containsEntry("lingniu.ingest.vehicle-stat.enabled", false) .containsEntry("lingniu.ingest.sink.mq.consumer.enabled", "${KAFKA_CONSUMER_ENABLED:true}") - .containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:100}") + .containsEntry("lingniu.ingest.sink.mq.consumer.max-poll-records", "${KAFKA_CONSUMER_MAX_POLL_RECORDS:2000}") .containsEntry("lingniu.ingest.sink.mq.consumer.concurrency", "${KAFKA_CONSUMER_CONCURRENCY:3}") .containsEntry( "lingniu.ingest.sink.mq.consumer.max-poll-interval-millis", diff --git a/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/consumer/EnvelopeBatchIngestor.java b/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/consumer/EnvelopeBatchIngestor.java new file mode 100644 index 00000000..03aaafe7 --- /dev/null +++ b/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/consumer/EnvelopeBatchIngestor.java @@ -0,0 +1,12 @@ +package com.lingniu.ingest.api.consumer; + +import java.util.List; + +/** + * Optional extension for consumers that can persist a Kafka poll batch in one + * storage operation. + */ +public interface EnvelopeBatchIngestor extends EnvelopeIngestor { + + List tryIngestAll(List kafkaValues); +} diff --git a/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/consumer/EnvelopeConsumerProcessor.java b/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/consumer/EnvelopeConsumerProcessor.java index 7359179f..f5b51abb 100644 --- a/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/consumer/EnvelopeConsumerProcessor.java +++ b/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/consumer/EnvelopeConsumerProcessor.java @@ -70,6 +70,32 @@ public final class EnvelopeConsumerProcessor { } } + public List processBatch(List records) { + if (records == null || records.isEmpty()) { + return List.of(); + } + if (!(ingestor instanceof EnvelopeBatchIngestor batchIngestor)) { + return records.stream().map(this::process).toList(); + } + List payloads = records.stream() + .map(EnvelopeConsumerRecord::payload) + .toList(); + List results = batchIngestor.tryIngestAll(payloads); + if (results.size() != records.size()) { + throw new IllegalStateException("batch ingestor returned " + results.size() + + " results for " + records.size() + " records"); + } + List copy = new ArrayList<>(results.size()); + for (int i = 0; i < records.size(); i++) { + EnvelopeIngestResult result = results.get(i); + if (DEAD_LETTER_STATUSES.contains(result.status())) { + deadLetterSink.publish(toDeadLetter(records.get(i), result)); + } + copy.add(result); + } + return List.copyOf(copy); + } + private EnvelopeDeadLetterRecord toDeadLetter(EnvelopeConsumerRecord record, EnvelopeIngestResult result) { return new EnvelopeDeadLetterRecord( service, diff --git a/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/event/RawArchiveKeys.java b/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/event/RawArchiveKeys.java index bcc36a0e..f2f14a7b 100644 --- a/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/event/RawArchiveKeys.java +++ b/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/event/RawArchiveKeys.java @@ -19,6 +19,7 @@ public final class RawArchiveKeys { public static final String META_EVENT_ID = "rawArchiveEventId"; public static final String META_KEY = "rawArchiveKey"; public static final String META_URI = "rawArchiveUri"; + public static final String META_PARSED_JSON = "rawArchiveParsedJson"; private static final DateTimeFormatter DATE_KEY = DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneId.of("Asia/Shanghai")); diff --git a/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/event/VehicleEvent.java b/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/event/VehicleEvent.java index c3c425ef..26bdaf88 100644 --- a/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/event/VehicleEvent.java +++ b/modules/core/ingest-api/src/main/java/com/lingniu/ingest/api/event/VehicleEvent.java @@ -150,6 +150,7 @@ public sealed interface VehicleEvent * @param command 协议主命令码（如 32960 0x02/0x03 等），冗余在事件里便于按命令分片归档 * @param infoType 协议子类型（可为 0），同上 * @param rawBytes 原始入站字节，不可为 null + * @param parsedJson 已解析结构化 JSON。只用于 RAW 明细查询，不应复制到位置/字段等派生表。 */ record RawArchive( String eventId, @@ -161,6 +162,25 @@ public sealed interface VehicleEvent Map metadata, int command, int infoType, - byte[] rawBytes - ) implements VehicleEvent {} + byte[] rawBytes, + String parsedJson + ) implements VehicleEvent { + public RawArchive(String eventId, + String vin, + ProtocolId source, + Instant eventTime, + Instant ingestTime, + String traceId, + Map metadata, + int command, + int infoType, + byte[] rawBytes) { + this(eventId, vin, source, eventTime, ingestTime, traceId, metadata, + command, infoType, rawBytes, ""); + } + + public RawArchive { + parsedJson = parsedJson == null ? "" : parsedJson; + } + } } diff --git a/modules/core/ingest-core/pom.xml b/modules/core/ingest-core/pom.xml index a436d69a..fb45680f 100644 --- a/modules/core/ingest-core/pom.xml +++ b/modules/core/ingest-core/pom.xml @@ -24,6 +24,14 @@ org.springframework.boot spring-boot-starter + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + org.springframework.boot spring-boot-autoconfigure diff --git a/modules/core/ingest-core/src/main/java/com/lingniu/ingest/core/dispatcher/Dispatcher.java b/modules/core/ingest-core/src/main/java/com/lingniu/ingest/core/dispatcher/Dispatcher.java index 083ee314..bbf5316c 100644 --- a/modules/core/ingest-core/src/main/java/com/lingniu/ingest/core/dispatcher/Dispatcher.java +++ b/modules/core/ingest-core/src/main/java/com/lingniu/ingest/core/dispatcher/Dispatcher.java @@ -1,5 +1,8 @@ package com.lingniu.ingest.core.dispatcher; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.lingniu.ingest.api.event.RawArchiveKeys; import com.lingniu.ingest.api.event.VehicleEvent; import com.lingniu.ingest.api.pipeline.IngestContext; @@ -36,6 +39,9 @@ public final class Dispatcher { private static final Logger log = LoggerFactory.getLogger(Dispatcher.class); private static final AtomicLong RAW_ARCHIVE_SEQUENCE = new AtomicLong(); private static final String REQUIRED_DURABLE_SINK = "kafka"; + private static final ObjectMapper RAW_ARCHIVE_JSON = JsonMapper.builder() + .findAndAddModules() + .build(); private final HandlerRegistry registry; private final InterceptorChain interceptors; @@ -167,7 +173,10 @@ public final class Dispatcher { byte[] bytes = frame.rawBytes(); if (bytes == null || bytes.length == 0) return RawArchiveLookup.empty(); - Map meta = frame.sourceMeta() == null ? Map.of() : frame.sourceMeta(); + Map meta = frame.sourceMeta() == null + ? Map.of() + : new LinkedHashMap<>(frame.sourceMeta()); + String parsedJson = parsedJson(frame.payload(), meta.remove(RawArchiveKeys.META_PARSED_JSON)); String vin = meta.getOrDefault("vin", ""); Instant ingestTime = frame.receivedAt() != null ? frame.receivedAt() : Instant.now(); String eventId = nextRawArchiveEventId(ingestTime); @@ -184,11 +193,34 @@ public final class Dispatcher { archiveMeta, frame.command(), frame.infoType(), - bytes); + bytes, + parsedJson); out.add(raw); return new RawArchiveLookup(eventId, key, RawArchiveKeys.logicalUri(key)); } + private static String parsedJson(Object payload, String explicitParsedJson) { + if (explicitParsedJson != null && !explicitParsedJson.isBlank()) { + return explicitParsedJson; + } + if (payload == null) { + return "{}"; + } + try { + return RAW_ARCHIVE_JSON.writeValueAsString(payload); + } catch (JsonProcessingException ex) { + return "{\"serializationError\":\"" + escapeJson(ex.getMessage()) + "\",\"payloadType\":\"" + + escapeJson(payload.getClass().getName()) + "\"}"; + } + } + + private static String escapeJson(String value) { + if (value == null) { + return ""; + } + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + private static String nextRawArchiveEventId(Instant ingestTime) { // 文件名需要可排序且单进程内唯一：毫秒时间放大到微秒量级，再用 AtomicLong 递增兜底。 long base = (ingestTime == null ? Instant.now() : ingestTime).toEpochMilli() * 1000L; diff --git a/modules/core/ingest-core/src/test/java/com/lingniu/ingest/core/dispatcher/DispatcherRawArchiveMetadataTest.java b/modules/core/ingest-core/src/test/java/com/lingniu/ingest/core/dispatcher/DispatcherRawArchiveMetadataTest.java index 58d90ac3..34529313 100644 --- a/modules/core/ingest-core/src/test/java/com/lingniu/ingest/core/dispatcher/DispatcherRawArchiveMetadataTest.java +++ b/modules/core/ingest-core/src/test/java/com/lingniu/ingest/core/dispatcher/DispatcherRawArchiveMetadataTest.java @@ -42,7 +42,7 @@ class DispatcherRawArchiveMetadataTest { ProtocolId.JT808, 0x0200, 0, - new Object(), + new SamplePayload("payload-0200", 7), new byte[]{0x7e, 0x01, 0x7e}, Map.of("vin", "VIN001", "peer", "127.0.0.1:10001"), Instant.parse("2026-06-22T08:00:00Z"))); @@ -52,6 +52,8 @@ class DispatcherRawArchiveMetadataTest { String rawArchiveKey = raw.metadata().get("rawArchiveKey"); assertThat(rawArchiveKey).isNotBlank(); + assertThat(raw.parsedJson()).contains("\"name\":\"payload-0200\""); + assertThat(raw.metadata()).doesNotContainKey("rawArchiveParsedJson"); assertThat(raw.eventId()).containsOnlyDigits(); assertThat(rawArchiveKey).endsWith("/" + raw.eventId() + ".bin"); assertThat(location.metadata()) @@ -102,7 +104,7 @@ class DispatcherRawArchiveMetadataTest { private static HandlerRegistry registryWithLocationHandler() throws NoSuchMethodException { LocationHandler bean = new LocationHandler(); - Method method = LocationHandler.class.getDeclaredMethod("onLocation", Object.class); + Method method = LocationHandler.class.getDeclaredMethod("onLocation", SamplePayload.class); HandlerRegistry registry = new HandlerRegistry(); registry.register(new HandlerDefinition( ProtocolId.JT808, @@ -169,7 +171,7 @@ class DispatcherRawArchiveMetadataTest { @ProtocolHandler(protocol = ProtocolId.JT808) static final class LocationHandler { @MessageMapping(command = 0x0200) - VehicleEvent.Location onLocation(Object ignored) { + VehicleEvent.Location onLocation(SamplePayload ignored) { return new VehicleEvent.Location( "location-1", "VIN001", @@ -182,6 +184,9 @@ class DispatcherRawArchiveMetadataTest { } } + record SamplePayload(String name, int value) { + } + @ProtocolHandler(protocol = ProtocolId.JT808) public static final class AsyncLocationHandler { @MessageMapping(command = 0x0201) diff --git a/modules/core/vehicle-identity/pom.xml b/modules/core/vehicle-identity/pom.xml index 00cdf569..2c6b9ad2 100644 --- a/modules/core/vehicle-identity/pom.xml +++ b/modules/core/vehicle-identity/pom.xml @@ -24,6 +24,10 @@ org.springframework.boot spring-boot-autoconfigure + + org.springframework + spring-jdbc + com.fasterxml.jackson.core jackson-databind diff --git a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/MySqlVehicleIdentityService.java b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/MySqlVehicleIdentityService.java index 41b8548d..90336fec 100644 --- a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/MySqlVehicleIdentityService.java +++ b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/MySqlVehicleIdentityService.java @@ -1,70 +1,64 @@ package com.lingniu.ingest.identity; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.lingniu.ingest.api.ProtocolId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.lingniu.ingest.identity.config.VehicleIdentityProperties; +import org.springframework.jdbc.core.ConnectionCallback; +import org.springframework.jdbc.core.JdbcTemplate; import javax.sql.DataSource; -import java.time.Duration; -import java.sql.Connection; -import java.sql.PreparedStatement; +import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; +import java.util.List; import java.util.Locale; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.Map; -public final class MySqlVehicleIdentityService implements VehicleIdentityResolver, VehicleIdentityRegistry, AutoCloseable { +public final class MySqlVehicleIdentityService implements VehicleIdentityResolver, VehicleIdentityRegistry { - private static final Logger log = LoggerFactory.getLogger(MySqlVehicleIdentityService.class); + private final JdbcTemplate jdbc; + private final ObjectMapper mapper; + private final String tableName; - private final DataSource dataSource; - private final String table; - private final String registrationTable; - private final ScheduledExecutorService refresher; - private volatile InMemoryVehicleIdentityService index; - - public MySqlVehicleIdentityService(DataSource dataSource, String table) { - this(dataSource, table, Duration.ZERO); - } - - public MySqlVehicleIdentityService(DataSource dataSource, String table, Duration refreshInterval) { - if (dataSource == null) { - throw new IllegalArgumentException("dataSource must not be null"); + public MySqlVehicleIdentityService(DataSource dataSource, + VehicleIdentityProperties.Mysql properties, + ObjectMapper mapper) { + this.jdbc = new JdbcTemplate(dataSource); + this.mapper = mapper == null ? new ObjectMapper() : mapper; + VehicleIdentityProperties.Mysql mysql = properties == null ? new VehicleIdentityProperties.Mysql() : properties; + this.tableName = safeTableName(mysql.getTableName()); + if (mysql.isInitializeSchema()) { + initializeSchema(); } - this.dataSource = dataSource; - this.table = sanitizeTable(table); - this.registrationTable = this.table + "_registration"; - initializeSchema(); - this.index = loadIndex(); - this.refresher = startRefresher(refreshInterval); } @Override public void bind(VehicleIdentityBinding binding) { - if (binding == null || !binding.hasResolvedVin()) { - return; - } - upsert(binding.plate(), binding.vin()); - index.bind(binding); - } - - @Override - public void register(VehicleRegistrationBinding registration) { - if (registration == null) { - return; - } - upsertRegistration(registration); - if (registration.hasResolvedVin()) { - bind(new VehicleIdentityBinding( - registration.protocol(), - registration.vin(), - registration.phone(), - registration.deviceId(), - registration.plate())); - } + String metadataJson = metadataJson(binding.metadata()); + jdbc.update(""" + INSERT INTO %s + (protocol, vin, phone, device_id, plate, province, city, maker, device_type, plate_color, metadata_json, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON DUPLICATE KEY UPDATE + vin = VALUES(vin), + device_id = VALUES(device_id), + province = VALUES(province), + city = VALUES(city), + maker = VALUES(maker), + device_type = VALUES(device_type), + plate_color = VALUES(plate_color), + metadata_json = VALUES(metadata_json), + updated_at = CURRENT_TIMESTAMP + """.formatted(tableName), + protocol(binding.protocol()), binding.vin(), binding.phone(), binding.deviceId(), binding.plate(), + binding.metadata().getOrDefault("province", ""), + binding.metadata().getOrDefault("city", ""), + binding.metadata().getOrDefault("maker", ""), + binding.metadata().getOrDefault("deviceType", ""), + binding.metadata().getOrDefault("plateColor", ""), + metadataJson); } @Override @@ -72,223 +66,167 @@ public final class MySqlVehicleIdentityService implements VehicleIdentityResolve if (lookup == null) { return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN); } - return index.resolve(lookup); - } - - public void refresh() { - index = loadIndex(); - } - - @Override - public void close() { - if (refresher != null) { - refresher.shutdownNow(); + if (!lookup.vin().isBlank()) { + return new VehicleIdentity(lookup.vin(), true, VehicleIdentitySource.EXPLICIT_VIN); } + VehicleIdentity byPhone = resolveBy("phone", lookup.protocol(), lookup.phone(), VehicleIdentitySource.BOUND_PHONE); + if (byPhone != null) { + return byPhone; + } + VehicleIdentity byDevice = resolveBy("device_id", lookup.protocol(), lookup.deviceId(), VehicleIdentitySource.BOUND_DEVICE_ID); + if (byDevice != null) { + return byDevice; + } + VehicleIdentity byPlate = resolveBy("plate", lookup.protocol(), lookup.plate(), VehicleIdentitySource.BOUND_PLATE); + if (byPlate != null) { + return byPlate; + } + if (!lookup.deviceId().isBlank()) { + return new VehicleIdentity(lookup.deviceId(), false, VehicleIdentitySource.FALLBACK_DEVICE_ID); + } + if (!lookup.phone().isBlank()) { + return new VehicleIdentity(lookup.phone(), false, VehicleIdentitySource.FALLBACK_PHONE); + } + if (!lookup.plate().isBlank()) { + return new VehicleIdentity(lookup.plate(), false, VehicleIdentitySource.FALLBACK_PLATE); + } + return new VehicleIdentity("unknown", false, VehicleIdentitySource.UNKNOWN); + } + + private VehicleIdentity resolveBy(String column, ProtocolId protocol, String externalId, VehicleIdentitySource source) { + if (externalId == null || externalId.isBlank()) { + return null; + } + String safeColumn = switch (column) { + case "phone", "device_id", "plate" -> column; + default -> throw new IllegalArgumentException("unsupported identity lookup column: " + column); + }; + List matches = jdbc.query(""" + SELECT vin + FROM %s + WHERE protocol = ? AND %s = ? + ORDER BY updated_at DESC + LIMIT 1 + """.formatted(tableName, safeColumn), + (rs, rowNum) -> identity(rs, source), + protocol(protocol), externalId.trim()); + return matches.isEmpty() ? null : matches.getFirst(); + } + + private VehicleIdentity identity(ResultSet rs, VehicleIdentitySource source) throws SQLException { + return new VehicleIdentity(rs.getString("vin"), true, source); } private void initializeSchema() { - String sql = """ + jdbc.execute(""" CREATE TABLE IF NOT EXISTS %s ( - plate VARCHAR(128) NOT NULL, - vin VARCHAR(64) NOT NULL, - PRIMARY KEY (plate), - KEY idx_vehicle_identity_vin (vin) + id BIGINT AUTO_INCREMENT PRIMARY KEY, + protocol VARCHAR(32) NOT NULL, + vin VARCHAR(32) NOT NULL, + phone VARCHAR(64) NOT NULL DEFAULT '', + device_id VARCHAR(64) NOT NULL DEFAULT '', + plate VARCHAR(64) NOT NULL DEFAULT '', + province VARCHAR(32) NOT NULL DEFAULT '', + city VARCHAR(32) NOT NULL DEFAULT '', + maker VARCHAR(64) NOT NULL DEFAULT '', + device_type VARCHAR(128) NOT NULL DEFAULT '', + plate_color VARCHAR(32) NOT NULL DEFAULT '', + metadata_json TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY uk_vehicle_identity_protocol_phone (protocol, phone), + KEY idx_vehicle_identity_protocol_device (protocol, device_id), + KEY idx_vehicle_identity_protocol_plate (protocol, plate) ) - """.formatted(table); - String registrationSql = """ - CREATE TABLE IF NOT EXISTS %s ( - protocol VARCHAR(32) NOT NULL, - phone VARCHAR(64) NOT NULL, - vin VARCHAR(64) NOT NULL DEFAULT 'unknown', - device_id VARCHAR(128) NOT NULL DEFAULT '', - plate VARCHAR(128) NOT NULL DEFAULT '', - province INT NULL, - city INT NULL, - maker VARCHAR(64) NOT NULL DEFAULT '', - device_type VARCHAR(128) NOT NULL DEFAULT '', - plate_color INT NULL, - last_registered_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (protocol, phone), - KEY idx_vehicle_registration_vin (vin), - KEY idx_vehicle_registration_device (protocol, device_id), - KEY idx_vehicle_registration_plate (protocol, plate) - ) - """.formatted(registrationTable); - try (Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement()) { - statement.execute(sql); - statement.execute(registrationSql); - } catch (SQLException e) { - throw new IllegalStateException("vehicle identity mysql schema initialize failed: " + table, e); + """.formatted(tableName)); + ensureColumn("phone", "phone VARCHAR(64) NOT NULL DEFAULT ''"); + ensureColumn("device_id", "device_id VARCHAR(64) NOT NULL DEFAULT ''"); + ensureColumn("plate", "plate VARCHAR(64) NOT NULL DEFAULT ''"); + ensureColumn("province", "province VARCHAR(32) NOT NULL DEFAULT ''"); + ensureColumn("city", "city VARCHAR(32) NOT NULL DEFAULT ''"); + ensureColumn("maker", "maker VARCHAR(64) NOT NULL DEFAULT ''"); + ensureColumn("device_type", "device_type VARCHAR(128) NOT NULL DEFAULT ''"); + ensureColumn("plate_color", "plate_color VARCHAR(32) NOT NULL DEFAULT ''"); + ensureColumn("metadata_json", "metadata_json TEXT"); + ensureColumn("created_at", "created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP"); + ensureColumn("updated_at", "updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP"); + ensureIndex("idx_vehicle_identity_protocol_device", "CREATE INDEX idx_vehicle_identity_protocol_device ON %s (protocol, device_id)"); + ensureIndex("idx_vehicle_identity_protocol_plate", "CREATE INDEX idx_vehicle_identity_protocol_plate ON %s (protocol, plate)"); + } + + private void ensureColumn(String columnName, String definition) { + if (!columnExists(columnName)) { + jdbc.execute("ALTER TABLE " + tableName + " ADD COLUMN " + definition); } } - private InMemoryVehicleIdentityService loadIndex() { - InMemoryVehicleIdentityService loaded = new InMemoryVehicleIdentityService(); - String sql = "SELECT plate, vin FROM " + table; - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(sql); - ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - bindGlobalPlate(loaded, resultSet.getString("vin"), resultSet.getString("plate")); + private boolean columnExists(String columnName) { + return Boolean.TRUE.equals(jdbc.execute((ConnectionCallback) connection -> { + DatabaseMetaData metaData = connection.getMetaData(); + for (String table : nameVariants(tableName)) { + for (String column : nameVariants(columnName)) { + try (ResultSet rs = metaData.getColumns(null, null, table, column)) { + if (rs.next()) { + return true; + } + } + } } - loadRegistrationIndex(loaded, connection); - return loaded; - } catch (SQLException e) { - throw new IllegalStateException("vehicle identity mysql index load failed: " + table, e); + return false; + })); + } + + private void ensureIndex(String indexName, String ddlTemplate) { + if (!indexExists(indexName)) { + jdbc.execute(ddlTemplate.formatted(tableName)); } } - private static void bindGlobalPlate(InMemoryVehicleIdentityService loaded, String vin, String plate) { - for (ProtocolId protocol : new ProtocolId[]{ - ProtocolId.GB32960, - ProtocolId.JT808, - ProtocolId.MQTT_YUTONG, - ProtocolId.XINDA_PUSH - }) { - loaded.bind(new VehicleIdentityBinding(protocol, vin, "", "", plate)); - } - } - - private void loadRegistrationIndex(InMemoryVehicleIdentityService loaded, Connection connection) throws SQLException { - String sql = "SELECT r.protocol, r.phone, r.device_id, r.plate, b.vin FROM " + registrationTable + " r " - + "JOIN " + table + " b ON b.plate = r.plate " - + "WHERE b.vin IS NOT NULL AND b.vin <> '' AND LOWER(b.vin) <> 'unknown'"; - try (PreparedStatement statement = connection.prepareStatement(sql); - ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - loaded.bind(new VehicleIdentityBinding( - protocol(resultSet.getString("protocol")), - resultSet.getString("vin"), - resultSet.getString("phone"), - resultSet.getString("device_id"), - resultSet.getString("plate"))); + private boolean indexExists(String indexName) { + return Boolean.TRUE.equals(jdbc.execute((ConnectionCallback) connection -> { + DatabaseMetaData metaData = connection.getMetaData(); + for (String table : nameVariants(tableName)) { + try (ResultSet rs = metaData.getIndexInfo(null, null, table, false, false)) { + while (rs.next()) { + String existing = rs.getString("INDEX_NAME"); + if (existing != null && existing.equalsIgnoreCase(indexName)) { + return true; + } + } + } } - } - loadLegacyRegistrationVinIndex(loaded, connection); + return false; + })); } - private void loadLegacyRegistrationVinIndex(InMemoryVehicleIdentityService loaded, Connection connection) throws SQLException { - String sql = "SELECT protocol, phone, device_id, plate, vin FROM " + registrationTable - + " WHERE vin IS NOT NULL AND vin <> '' AND LOWER(vin) <> 'unknown'"; - try (PreparedStatement statement = connection.prepareStatement(sql); - ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - loaded.bind(new VehicleIdentityBinding( - protocol(resultSet.getString("protocol")), - resultSet.getString("vin"), - resultSet.getString("phone"), - resultSet.getString("device_id"), - resultSet.getString("plate"))); - } - } + private static String[] nameVariants(String name) { + return new String[]{ + name, + name.toLowerCase(Locale.ROOT), + name.toUpperCase(Locale.ROOT) + }; } - private ScheduledExecutorService startRefresher(Duration refreshInterval) { - if (refreshInterval == null || refreshInterval.isZero() || refreshInterval.isNegative()) { - return null; + private String metadataJson(Map metadata) { + if (metadata == null || metadata.isEmpty()) { + return "{}"; } - long delayMillis = Math.max(refreshInterval.toMillis(), 1000L); - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(r -> { - Thread t = new Thread(r, "vehicle-identity-mysql-refresh"); - t.setDaemon(true); - return t; - }); - executor.scheduleWithFixedDelay(this::safeRefresh, delayMillis, delayMillis, TimeUnit.MILLISECONDS); - return executor; - } - - private void safeRefresh() { try { - refresh(); - } catch (RuntimeException e) { - log.warn("vehicle identity mysql refresh failed table={}", table, e); + return mapper.writeValueAsString(metadata); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("vehicle identity metadata serialize failed", e); } } - private void upsert(String plate, String vin) { - String normalizedPlate = normalize(plate); - if (normalizedPlate.isBlank()) { - return; - } - String sql = "INSERT INTO " + table + " (plate, vin) " - + "VALUES (?, ?) " - + "ON DUPLICATE KEY UPDATE vin = VALUES(vin)"; - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(sql)) { - statement.setString(1, normalizedPlate); - statement.setString(2, vin.trim()); - statement.executeUpdate(); - } catch (SQLException e) { - throw new IllegalStateException("vehicle identity mysql bind failed: " + table, e); - } - } - - private void upsertRegistration(VehicleRegistrationBinding registration) { - String phone = normalize(registration.phone()); - if (phone.isBlank()) { - return; - } - String vin = registration.hasResolvedVin() ? registration.vin().trim() : "unknown"; - String sql = "INSERT INTO " + registrationTable - + " (protocol, phone, vin, device_id, plate, province, city, maker, device_type, plate_color) " - + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " - + "ON DUPLICATE KEY UPDATE " - + "vin = CASE WHEN VALUES(vin) <> 'unknown' THEN VALUES(vin) ELSE vin END, " - + "device_id = VALUES(device_id), plate = VALUES(plate), province = VALUES(province), " - + "city = VALUES(city), maker = VALUES(maker), device_type = VALUES(device_type), " - + "plate_color = VALUES(plate_color), last_registered_at = CURRENT_TIMESTAMP"; - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(sql)) { - statement.setString(1, protocolName(registration.protocol())); - statement.setString(2, phone); - statement.setString(3, vin); - statement.setString(4, normalize(registration.deviceId())); - statement.setString(5, normalize(registration.plate())); - setInteger(statement, 6, registration.province()); - setInteger(statement, 7, registration.city()); - statement.setString(8, normalize(registration.maker())); - statement.setString(9, normalize(registration.deviceType())); - setInteger(statement, 10, registration.plateColor()); - statement.executeUpdate(); - } catch (SQLException e) { - throw new IllegalStateException("vehicle identity mysql registration bind failed: " + registrationTable, e); - } - } - - private static void setInteger(PreparedStatement statement, int index, Integer value) throws SQLException { - if (value == null) { - statement.setObject(index, null); - } else { - statement.setInt(index, value); - } - } - - private static String sanitizeTable(String table) { - String value = table == null || table.isBlank() ? "vehicle_identity_binding" : table.trim(); - if (!value.matches("[A-Za-z0-9_]+")) { - throw new IllegalArgumentException("vehicle identity mysql table must contain only letters, digits, and underscore"); - } - return value; - } - - private static String protocolName(ProtocolId protocol) { + private static String protocol(ProtocolId protocol) { return protocol == null ? "UNKNOWN" : protocol.name(); } - private static ProtocolId protocol(String value) { - if (value == null || value.isBlank()) { - return ProtocolId.UNKNOWN; + private static String safeTableName(String value) { + String name = value == null || value.isBlank() ? "vehicle_identity_bindings" : value.trim(); + if (!name.matches("[A-Za-z0-9_]+")) { + throw new IllegalArgumentException("invalid vehicle identity mysql table name: " + value); } - try { - return ProtocolId.valueOf(value.trim().toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException ignored) { - return ProtocolId.UNKNOWN; - } - } - - private static String normalize(String value) { - return value == null ? "" : value.trim().toUpperCase(Locale.ROOT); + return name.toLowerCase(Locale.ROOT); } } diff --git a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/VehicleIdentityBinding.java b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/VehicleIdentityBinding.java index f5b8f97c..3a31a26a 100644 --- a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/VehicleIdentityBinding.java +++ b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/VehicleIdentityBinding.java @@ -2,18 +2,26 @@ package com.lingniu.ingest.identity; import com.lingniu.ingest.api.ProtocolId; +import java.util.Map; + public record VehicleIdentityBinding( ProtocolId protocol, String vin, String phone, String deviceId, - String plate + String plate, + Map metadata ) { + public VehicleIdentityBinding(ProtocolId protocol, String vin, String phone, String deviceId, String plate) { + this(protocol, vin, phone, deviceId, plate, Map.of()); + } + public VehicleIdentityBinding { vin = normalize(vin); phone = normalize(phone); deviceId = normalize(deviceId); plate = normalize(plate); + metadata = metadata == null ? Map.of() : Map.copyOf(metadata); if (vin.isBlank()) { throw new IllegalArgumentException("vin must not be blank"); } diff --git a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/VehicleIdentitySource.java b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/VehicleIdentitySource.java index 3420788d..b35cfa69 100644 --- a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/VehicleIdentitySource.java +++ b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/VehicleIdentitySource.java @@ -2,6 +2,7 @@ package com.lingniu.ingest.identity; public enum VehicleIdentitySource { EXPLICIT_VIN, + REGISTERED, BOUND_PHONE, BOUND_DEVICE_ID, BOUND_PLATE, diff --git a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/config/VehicleIdentityAutoConfiguration.java b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/config/VehicleIdentityAutoConfiguration.java index 87407ffb..c45397fe 100644 --- a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/config/VehicleIdentityAutoConfiguration.java +++ b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/config/VehicleIdentityAutoConfiguration.java @@ -11,15 +11,10 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.datasource.DriverManagerDataSource; import javax.sql.DataSource; -import java.io.PrintWriter; import java.nio.file.Path; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; -import java.util.logging.Logger; @AutoConfiguration @EnableConfigurationProperties(VehicleIdentityProperties.class) @@ -40,22 +35,15 @@ public class VehicleIdentityAutoConfiguration { return new FileVehicleIdentityService(Path.of(properties.getFile().getPath()), objectMapper); } - @Bean - @ConditionalOnMissingBean(DataSource.class) - @ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql") - public DataSource vehicleIdentityDataSource(VehicleIdentityProperties properties) { - VehicleIdentityProperties.Mysql mysql = properties.getMysql(); - return new DriverManagerVehicleIdentityDataSource( - mysql.getDriverClassName(), mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword()); - } - @Bean @ConditionalOnMissingBean({VehicleIdentityResolver.class, VehicleIdentityRegistry.class}) @ConditionalOnProperty(prefix = "lingniu.ingest.identity", name = "store", havingValue = "mysql") - public MySqlVehicleIdentityService mySqlVehicleIdentityService(VehicleIdentityProperties properties, - DataSource dataSource) { + public MySqlVehicleIdentityService mysqlVehicleIdentityService(VehicleIdentityProperties properties, + ObjectMapper objectMapper) { VehicleIdentityProperties.Mysql mysql = properties.getMysql(); - return new MySqlVehicleIdentityService(dataSource, mysql.getTable(), mysql.getRefreshInterval()); + DataSource dataSource = new DriverManagerDataSource( + mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword()); + return new MySqlVehicleIdentityService(dataSource, mysql, objectMapper); } @Bean @@ -65,61 +53,4 @@ public class VehicleIdentityAutoConfiguration { public InMemoryVehicleIdentityService vehicleIdentityService() { return new InMemoryVehicleIdentityService(); } - - private static final class DriverManagerVehicleIdentityDataSource implements DataSource { - private final String jdbcUrl; - private final String username; - private final String password; - private PrintWriter logWriter; - private int loginTimeout; - - private DriverManagerVehicleIdentityDataSource(String driverClassName, - String jdbcUrl, - String username, - String password) { - if (jdbcUrl == null || jdbcUrl.isBlank()) { - throw new IllegalArgumentException("lingniu.ingest.identity.mysql.jdbc-url is required"); - } - if (driverClassName != null && !driverClassName.isBlank()) { - try { - Class.forName(driverClassName.trim()); - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("vehicle identity mysql driver class not found: " - + driverClassName, e); - } - } - this.jdbcUrl = jdbcUrl; - this.username = username == null ? "" : username; - this.password = password == null ? "" : password; - } - - @Override - public Connection getConnection() throws SQLException { - DriverManager.setLoginTimeout(loginTimeout); - return DriverManager.getConnection(jdbcUrl, username, password); - } - - @Override - public Connection getConnection(String username, String password) throws SQLException { - DriverManager.setLoginTimeout(loginTimeout); - return DriverManager.getConnection(jdbcUrl, username, password); - } - - @Override public PrintWriter getLogWriter() { return logWriter; } - @Override public void setLogWriter(PrintWriter out) { this.logWriter = out; } - @Override public void setLoginTimeout(int seconds) { this.loginTimeout = seconds; } - @Override public int getLoginTimeout() { return loginTimeout; } - @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { - throw new SQLFeatureNotSupportedException(); - } - @Override public T unwrap(Class iface) throws SQLException { - if (iface.isInstance(this)) { - return iface.cast(this); - } - throw new SQLException("not a wrapper for " + iface); - } - @Override public boolean isWrapperFor(Class iface) { - return iface.isInstance(this); - } - } } diff --git a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/config/VehicleIdentityProperties.java b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/config/VehicleIdentityProperties.java index 9b1a98d4..38aeb325 100644 --- a/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/config/VehicleIdentityProperties.java +++ b/modules/core/vehicle-identity/src/main/java/com/lingniu/ingest/identity/config/VehicleIdentityProperties.java @@ -2,8 +2,6 @@ package com.lingniu.ingest.identity.config; import org.springframework.boot.context.properties.ConfigurationProperties; -import java.time.Duration; - @ConfigurationProperties(prefix = "lingniu.ingest.identity") public class VehicleIdentityProperties { @@ -26,24 +24,21 @@ public class VehicleIdentityProperties { } public static class Mysql { - private String table = "vehicle_identity_binding"; - private String jdbcUrl = ""; - private String username = ""; + private String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/lingniu_vehicle?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai"; + private String username = "root"; private String password = ""; - private String driverClassName = "com.mysql.cj.jdbc.Driver"; - private Duration refreshInterval = Duration.ofSeconds(60); + private String tableName = "vehicle_identity_bindings"; + private boolean initializeSchema = true; - public String getTable() { return table; } - public void setTable(String table) { this.table = table; } public String getJdbcUrl() { return jdbcUrl; } public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } - public String getDriverClassName() { return driverClassName; } - public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; } - public Duration getRefreshInterval() { return refreshInterval; } - public void setRefreshInterval(Duration refreshInterval) { this.refreshInterval = refreshInterval; } + public String getTableName() { return tableName; } + public void setTableName(String tableName) { this.tableName = tableName; } + public boolean isInitializeSchema() { return initializeSchema; } + public void setInitializeSchema(boolean initializeSchema) { this.initializeSchema = initializeSchema; } } } diff --git a/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/MySqlVehicleIdentityServiceJdbcTest.java b/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/MySqlVehicleIdentityServiceJdbcTest.java index 0dd1a7d9..87eb5cfb 100644 --- a/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/MySqlVehicleIdentityServiceJdbcTest.java +++ b/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/MySqlVehicleIdentityServiceJdbcTest.java @@ -1,6 +1,8 @@ package com.lingniu.ingest.identity; +import com.fasterxml.jackson.databind.ObjectMapper; import com.lingniu.ingest.api.ProtocolId; +import com.lingniu.ingest.identity.config.VehicleIdentityProperties; import org.h2.jdbcx.JdbcDataSource; import org.junit.jupiter.api.Test; @@ -9,6 +11,7 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -21,62 +24,67 @@ class MySqlVehicleIdentityServiceJdbcTest { dataSource.setURL("jdbc:h2:mem:" + UUID.randomUUID() + ";MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1"); - try (MySqlVehicleIdentityService service = - new MySqlVehicleIdentityService(dataSource, "vehicle_identity_binding")) { - service.register(new VehicleRegistrationBinding( - ProtocolId.JT808, - "unknown", - "13079961001", - "dev61001", - "粤B61001", - 44, - 4401, - "maker-a", - "type-a", - 1)); + VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql(); + mysql.setTableName("vehicle_identity_binding"); + MySqlVehicleIdentityService service = new MySqlVehicleIdentityService( + dataSource, + mysql, + new ObjectMapper()); - try (Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement()) { - ResultSet registered = statement.executeQuery(""" - SELECT vin, device_id, plate, province, city, maker, device_type, plate_color - FROM vehicle_identity_binding_registration - WHERE protocol = 'JT808' AND phone = '13079961001' - """); - assertThat(registered.next()).isTrue(); - assertThat(registered.getString("vin")).isEqualTo("unknown"); - assertThat(registered.getString("device_id")).isEqualTo("DEV61001"); - assertThat(registered.getString("plate")).isEqualTo("粤B61001"); - assertThat(registered.getInt("province")).isEqualTo(44); - assertThat(registered.getInt("city")).isEqualTo(4401); - assertThat(registered.getString("maker")).isEqualTo("MAKER-A"); - assertThat(registered.getString("device_type")).isEqualTo("TYPE-A"); - assertThat(registered.getInt("plate_color")).isEqualTo(1); + service.bind(new VehicleIdentityBinding( + ProtocolId.JT808, + "LNVIN00000061001", + "13079961001", + "dev61001", + "粤B61001", + Map.of( + "province", "44", + "city", "4401", + "maker", "maker-a", + "deviceType", "type-a", + "plateColor", "1"))); - List columns = new ArrayList<>(); - ResultSet bindingColumns = connection.getMetaData() - .getColumns(null, null, "vehicle_identity_binding", null); - while (bindingColumns.next()) { - columns.add(bindingColumns.getString("COLUMN_NAME")); - } - assertThat(columns).containsExactly("plate", "vin"); + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + ResultSet binding = statement.executeQuery(""" + SELECT vin, phone, device_id, plate, province, city, maker, device_type, plate_color + FROM vehicle_identity_binding + WHERE protocol = 'JT808' AND phone = '13079961001' + """); + assertThat(binding.next()).isTrue(); + assertThat(binding.getString("vin")).isEqualTo("LNVIN00000061001"); + assertThat(binding.getString("phone")).isEqualTo("13079961001"); + assertThat(binding.getString("device_id")).isEqualTo("dev61001"); + assertThat(binding.getString("plate")).isEqualTo("粤B61001"); + assertThat(binding.getString("province")).isEqualTo("44"); + assertThat(binding.getString("city")).isEqualTo("4401"); + assertThat(binding.getString("maker")).isEqualTo("maker-a"); + assertThat(binding.getString("device_type")).isEqualTo("type-a"); + assertThat(binding.getString("plate_color")).isEqualTo("1"); - statement.executeUpdate(""" - INSERT INTO vehicle_identity_binding (plate, vin) - VALUES ('粤B61001', 'LNVIN00000061001') - """); + List columns = new ArrayList<>(); + ResultSet bindingColumns = connection.getMetaData() + .getColumns(null, null, "vehicle_identity_binding", null); + while (bindingColumns.next()) { + columns.add(bindingColumns.getString("COLUMN_NAME")); } - - service.refresh(); - - assertThat(service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "13079961001", "", "")).vin()) - .isEqualTo("LNVIN00000061001"); - assertThat(service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "", "dev61001", "")).vin()) - .isEqualTo("LNVIN00000061001"); - assertThat(service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "", "", "粤B61001")).vin()) - .isEqualTo("LNVIN00000061001"); + assertThat(columns).contains( + "protocol", + "phone", + "device_id", + "plate", + "vin", + "metadata_json"); } + + assertThat(service.resolve(new VehicleIdentityLookup( + ProtocolId.JT808, "", "13079961001", "", "")).vin()) + .isEqualTo("LNVIN00000061001"); + assertThat(service.resolve(new VehicleIdentityLookup( + ProtocolId.JT808, "", "", "dev61001", "")).vin()) + .isEqualTo("LNVIN00000061001"); + assertThat(service.resolve(new VehicleIdentityLookup( + ProtocolId.JT808, "", "", "", "粤B61001")).vin()) + .isEqualTo("LNVIN00000061001"); } } diff --git a/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/MySqlVehicleIdentityServiceTest.java b/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/MySqlVehicleIdentityServiceTest.java index b1ca6a11..1088a3f3 100644 --- a/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/MySqlVehicleIdentityServiceTest.java +++ b/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/MySqlVehicleIdentityServiceTest.java @@ -1,478 +1,157 @@ package com.lingniu.ingest.identity; +import com.fasterxml.jackson.databind.ObjectMapper; import com.lingniu.ingest.api.ProtocolId; +import com.lingniu.ingest.identity.config.VehicleIdentityProperties; import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; -import javax.sql.DataSource; -import java.io.PrintWriter; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Proxy; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.logging.Logger; import static org.assertj.core.api.Assertions.assertThat; class MySqlVehicleIdentityServiceTest { @Test - void bindsAndResolvesIdentityThroughJdbcStore() { - RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc(); - MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding"); + void upsertsRegistrationBindingAndResolvesByExternalIdentifiers() { + DriverManagerDataSource dataSource = new DriverManagerDataSource( + "jdbc:h2:mem:mysql-identity;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", ""); + VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql(); + mysql.setTableName("vehicle_identity_bindings"); + MySqlVehicleIdentityService service = new MySqlVehicleIdentityService( + dataSource, + mysql, + new ObjectMapper()); service.bind(new VehicleIdentityBinding( - ProtocolId.JT808, "LNVIN000000000123", "13079960001", "DEV001", "粤B00001")); - - VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "13079960001", "", "")); - VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "", "DEV001", "")); - VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "", "", "粤B00001")); - - assertThat(jdbc.createdTables) - .anySatisfy(sql -> assertThat(sql).contains("CREATE TABLE IF NOT EXISTS vehicle_identity_binding")); - assertThat(jdbc.bindings).containsEntry("粤B00001", "LNVIN000000000123"); - assertThat(byPhone.vin()).isEqualTo("LNVIN000000000123"); - assertThat(byPhone.resolved()).isTrue(); - assertThat(byPhone.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE); - assertThat(byDevice.vin()).isEqualTo("LNVIN000000000123"); - assertThat(byDevice.source()).isEqualTo(VehicleIdentitySource.BOUND_DEVICE_ID); - assertThat(byPlate.vin()).isEqualTo("LNVIN000000000123"); - assertThat(byPlate.source()).isEqualTo(VehicleIdentitySource.BOUND_PLATE); - } - - @Test - void ignoresUnknownVinBinding() { - RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc(); - MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding"); - - service.bind(new VehicleIdentityBinding( - ProtocolId.JT808, "unknown", "13079960001", "DEV001", "粤B00001")); - - assertThat(jdbc.bindings).isEmpty(); - } - - @Test - void resolvesFromStartupCacheWithoutPerFrameDatabaseLookup() { - RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc(); - jdbc.seedRegistration("JT808", "13079960001", "DEV001", "粤B00001"); - jdbc.seed("粤B00001", "LNVIN000000000123"); - MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding"); - - VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "13079960001", "", "")); - VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "", "DEV001", "")); - VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "", "", "粤B00001")); - - assertThat(byPhone.vin()).isEqualTo("LNVIN000000000123"); - assertThat(byDevice.vin()).isEqualTo("LNVIN000000000123"); - assertThat(byPlate.vin()).isEqualTo("LNVIN000000000123"); - assertThat(jdbc.loadAllSelects).isEqualTo(1); - assertThat(jdbc.lookupSelects).isZero(); - } - - @Test - void resolvesGlobalPlateBindingForXindaPush() { - RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc(); - jdbc.seed("浙F00780F", "LA9GG68L8PBAF4776"); - MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding"); - - VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup( - ProtocolId.XINDA_PUSH, "", "", "", "浙F00780F")); - - assertThat(byPlate.vin()).isEqualTo("LA9GG68L8PBAF4776"); - assertThat(byPlate.resolved()).isTrue(); - assertThat(byPlate.source()).isEqualTo(VehicleIdentitySource.BOUND_PLATE); - assertThat(jdbc.loadAllSelects).isEqualTo(1); - assertThat(jdbc.lookupSelects).isZero(); - } - - @Test - void refreshLoadsExternallyInsertedBindingWithoutPerFrameDatabaseLookup() { - RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc(); - jdbc.seedRegistration("JT808", "13079960002", "DEV002", "粤B00002"); - MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding"); - - VehicleIdentity before = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "13079960002", "", "")); - - jdbc.seed("粤B00002", "LNVIN000000000456"); - service.refresh(); - VehicleIdentity after = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "13079960002", "", "")); - - assertThat(before.resolved()).isFalse(); - assertThat(after.vin()).isEqualTo("LNVIN000000000456"); - assertThat(after.resolved()).isTrue(); - assertThat(after.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE); - assertThat(jdbc.loadAllSelects).isEqualTo(2); - assertThat(jdbc.lookupSelects).isZero(); - } - - @Test - void storesRegistrationWithoutVinAndResolvesAfterPlateVinIsWrittenBack() { - RecordingIdentityJdbc jdbc = new RecordingIdentityJdbc(); - MySqlVehicleIdentityService service = new MySqlVehicleIdentityService(jdbc.dataSource(), "vehicle_identity_binding"); - - service.register(new VehicleRegistrationBinding( ProtocolId.JT808, - "unknown", - "13079960003", - "DEV003", - "粤B00003", - 44, - 4401, - "MAKER", - "TYPE-A", - 1)); + "JT808AABBCCDDEE1", + "13079963289", + "9963289", + "沪A00113F", + Map.of( + "province", "31", + "city", "113", + "maker", "70112", + "deviceType", "SEG-9888G", + "plateColor", "2"))); - VehicleIdentity before = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "13079960003", "", "")); + assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "13079963289", "", ""))) + .extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source) + .containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_PHONE); + assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "", "9963289", ""))) + .extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source) + .containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_DEVICE_ID); + assertThat(service.resolve(new VehicleIdentityLookup(ProtocolId.JT808, "", "", "", "沪A00113F"))) + .extracting(VehicleIdentity::vin, VehicleIdentity::resolved, VehicleIdentity::source) + .containsExactly("JT808AABBCCDDEE1", true, VehicleIdentitySource.BOUND_PLATE); - jdbc.seed("粤B00003", "LNVIN000000000789"); - service.refresh(); - VehicleIdentity byPhone = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "13079960003", "", "")); - VehicleIdentity byDevice = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "", "DEV003", "")); - VehicleIdentity byPlate = service.resolve(new VehicleIdentityLookup( - ProtocolId.JT808, "", "", "", "粤B00003")); - - assertThat(jdbc.createdTables) - .anySatisfy(sql -> assertThat(sql).contains("CREATE TABLE IF NOT EXISTS vehicle_identity_binding_registration")); - assertThat(jdbc.registrations).containsKey("JT808|13079960003"); - assertThat(jdbc.bindings).containsEntry("粤B00003", "LNVIN000000000789"); - assertThat(before.resolved()).isFalse(); - assertThat(byPhone.vin()).isEqualTo("LNVIN000000000789"); - assertThat(byPhone.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE); - assertThat(byDevice.vin()).isEqualTo("LNVIN000000000789"); - assertThat(byPlate.vin()).isEqualTo("LNVIN000000000789"); + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + Map row = jdbc.queryForMap(""" + SELECT vin, phone, device_id, plate, province, city, maker, device_type, plate_color + FROM vehicle_identity_bindings + WHERE protocol = 'JT808' AND phone = '13079963289' + """); + assertThat(row) + .containsEntry("vin", "JT808AABBCCDDEE1") + .containsEntry("phone", "13079963289") + .containsEntry("device_id", "9963289") + .containsEntry("plate", "沪A00113F") + .containsEntry("province", "31") + .containsEntry("city", "113") + .containsEntry("maker", "70112") + .containsEntry("device_type", "SEG-9888G") + .containsEntry("plate_color", "2"); } - private static final class RecordingIdentityJdbc { - private final List createdTables = new ArrayList<>(); - private final Map bindings = new HashMap<>(); - private final Map registrations = new HashMap<>(); - private int loadAllSelects; - private int lookupSelects; + @Test + void initializesMissingColumnsOnExistingLegacyTable() { + DriverManagerDataSource dataSource = new DriverManagerDataSource( + "jdbc:h2:mem:mysql-identity-legacy;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", ""); + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + jdbc.execute(""" + CREATE TABLE vehicle_identity_binding ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + protocol VARCHAR(32) NOT NULL, + vin VARCHAR(32) NOT NULL, + legacy_key VARCHAR(64) NOT NULL DEFAULT '' + ) + """); + VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql(); + mysql.setTableName("vehicle_identity_binding"); - private void seed(String plate, String vin) { - bindings.put(normalize(plate), vin); - } + MySqlVehicleIdentityService service = new MySqlVehicleIdentityService( + dataSource, + mysql, + new ObjectMapper()); + service.bind(new VehicleIdentityBinding( + ProtocolId.JT808, + "JT808LEGACY00001", + "13079963289", + "9963289", + "沪A00113F", + Map.of("maker", "70112", "deviceType", "SEG-9888G"))); - private void seedRegistration(String protocol, String phone, String deviceId, String plate) { - registrations.put(protocol + "|" + normalize(phone), - new RegistrationRow(protocol, normalize(phone), "unknown", normalize(deviceId), normalize(plate))); - } + Map row = jdbc.queryForMap(""" + SELECT vin, maker, device_type, metadata_json + FROM vehicle_identity_binding + WHERE protocol = 'JT808' AND phone = '13079963289' + """); + assertThat(row) + .containsEntry("vin", "JT808LEGACY00001") + .containsEntry("maker", "70112") + .containsEntry("device_type", "SEG-9888G"); + assertThat((String) row.get("metadata_json")).contains("SEG-9888G"); + } - private DataSource dataSource() { - return new DataSource() { - @Override public Connection getConnection() { return connection(); } - @Override public Connection getConnection(String username, String password) { return connection(); } - @Override public PrintWriter getLogWriter() { return null; } - @Override public void setLogWriter(PrintWriter out) {} - @Override public void setLoginTimeout(int seconds) {} - @Override public int getLoginTimeout() { return 0; } - @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { throw new SQLFeatureNotSupportedException(); } - @Override public T unwrap(Class iface) { throw new UnsupportedOperationException(); } - @Override public boolean isWrapperFor(Class iface) { return false; } - }; - } + @Test + void upsertsByCompositeProtocolPhonePlatePrimaryKey() { + DriverManagerDataSource dataSource = new DriverManagerDataSource( + "jdbc:h2:mem:mysql-identity-composite;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", "sa", ""); + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + jdbc.execute(""" + CREATE TABLE vehicle_identity_binding ( + protocol VARCHAR(32) NOT NULL DEFAULT 'ALL', + phone VARCHAR(128) NOT NULL DEFAULT '', + plate VARCHAR(128) NOT NULL DEFAULT '', + vin VARCHAR(64) NOT NULL, + province VARCHAR(32) NOT NULL DEFAULT '', + city VARCHAR(32) NOT NULL DEFAULT '', + maker VARCHAR(64) NOT NULL DEFAULT '', + device_type VARCHAR(128) NOT NULL DEFAULT '', + plate_color VARCHAR(32) NOT NULL DEFAULT '', + metadata_json TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + device_id VARCHAR(64) NOT NULL DEFAULT '', + PRIMARY KEY (protocol, phone, plate) + ) + """); + VehicleIdentityProperties.Mysql mysql = new VehicleIdentityProperties.Mysql(); + mysql.setTableName("vehicle_identity_binding"); + MySqlVehicleIdentityService service = new MySqlVehicleIdentityService( + dataSource, + mysql, + new ObjectMapper()); + service.bind(new VehicleIdentityBinding(ProtocolId.JT808, + "VIN-A", "13079963289", "DEV-A", "沪A00113F")); - private Connection connection() { - InvocationHandler handler = (proxy, method, args) -> switch (method.getName()) { - case "createStatement" -> statement(); - case "prepareStatement" -> preparedStatement((String) args[0]); - case "close" -> null; - case "isClosed" -> false; - case "unwrap" -> throw new UnsupportedOperationException(); - case "isWrapperFor" -> false; - default -> defaultValue(method.getReturnType()); - }; - return (Connection) Proxy.newProxyInstance( - Connection.class.getClassLoader(), new Class[]{Connection.class}, handler); - } + service.bind(new VehicleIdentityBinding(ProtocolId.JT808, + "VIN-B", "13079963289", "DEV-B", "沪A00114F")); + service.bind(new VehicleIdentityBinding(ProtocolId.JT808, + "VIN-B2", "13079963289", "DEV-B2", "沪A00114F")); - private Statement statement() { - InvocationHandler handler = (proxy, method, args) -> { - if ("execute".equals(method.getName())) { - createdTables.add((String) args[0]); - return false; - } - if ("close".equals(method.getName())) { - return null; - } - return defaultValue(method.getReturnType()); - }; - return (Statement) Proxy.newProxyInstance( - Statement.class.getClassLoader(), new Class[]{Statement.class}, handler); - } - - private PreparedStatement preparedStatement(String sql) { - List params = new ArrayList<>(); - InvocationHandler handler = (proxy, method, args) -> { - switch (method.getName()) { - case "setString", "setInt", "setObject" -> { - set(params, (Integer) args[0], args[1]); - return null; - } - case "executeUpdate" -> { - if (sql.startsWith("INSERT INTO vehicle_identity_binding_registration ")) { - upsertRegistration(params); - } else { - upsert(params); - } - return 1; - } - case "executeQuery" -> { - if (sql.startsWith("SELECT plate, vin FROM ")) { - loadAllSelects++; - return allBindingsResultSet(); - } - if (sql.startsWith("SELECT protocol, identifier_type, identifier_value, vin FROM ")) { - loadAllSelects++; - return legacyAllBindingsResultSet(); - } - if (sql.startsWith("SELECT r.protocol, r.phone, r.device_id, r.plate, b.vin FROM ")) { - return registrationResultSet(); - } - if (sql.startsWith("SELECT protocol, phone, device_id, plate, vin FROM ")) { - return legacyRegistrationResultSet(); - } - lookupSelects++; - return resultSet(resolve(params)); - } - case "close" -> { - return null; - } - default -> { - return defaultValue(method.getReturnType()); - } - } - }; - return (PreparedStatement) Proxy.newProxyInstance( - PreparedStatement.class.getClassLoader(), new Class[]{PreparedStatement.class}, handler); - } - - private void upsert(List params) { - String plate; - String vin; - if (params.size() >= 4) { - plate = normalize((String) params.get(2)); - vin = (String) params.get(3); - } else { - plate = normalize((String) params.get(0)); - vin = (String) params.get(1); - } - bindings.put(plate, vin); - } - - private void upsertRegistration(List params) { - String protocol = (String) params.get(0); - String phone = normalize((String) params.get(1)); - String vin = (String) params.get(2); - String key = protocol + "|" + phone; - RegistrationRow existing = registrations.get(key); - String effectiveVin = existing != null && "unknown".equalsIgnoreCase(vin) ? existing.vin : vin; - registrations.put(key, new RegistrationRow( - protocol, - phone, - effectiveVin, - normalize((String) params.get(3)), - normalize((String) params.get(4)))); - } - - private String resolve(List params) { - String protocol = (String) params.get(0); - String identifierType = (String) params.get(1); - String identifierValue = normalize((String) params.get(2)); - if (!"PLATE".equals(identifierType)) { - return null; - } - return bindings.get(identifierValue); - } - - private ResultSet allBindingsResultSet() { - List> entries = new ArrayList<>(bindings.entrySet()); - InvocationHandler handler = new InvocationHandler() { - private int index = -1; - - @Override - public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) { - return switch (method.getName()) { - case "next" -> { - index++; - yield index < entries.size(); - } - case "getString" -> getColumn(entries.get(index), args[0]); - case "close" -> null; - default -> defaultValue(method.getReturnType()); - }; - } - }; - return (ResultSet) Proxy.newProxyInstance( - ResultSet.class.getClassLoader(), new Class[]{ResultSet.class}, handler); - } - - private ResultSet legacyAllBindingsResultSet() { - List> entries = new ArrayList<>(bindings.entrySet()); - InvocationHandler handler = new InvocationHandler() { - private int index = -1; - - @Override - public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) { - return switch (method.getName()) { - case "next" -> { - index++; - yield index < entries.size(); - } - case "getString" -> legacyColumn(entries.get(index), args[0]); - case "close" -> null; - default -> defaultValue(method.getReturnType()); - }; - } - }; - return (ResultSet) Proxy.newProxyInstance( - ResultSet.class.getClassLoader(), new Class[]{ResultSet.class}, handler); - } - - private static String getColumn(Map.Entry entry, Object column) { - String name = column.toString(); - return switch (name) { - case "plate" -> entry.getKey(); - case "vin" -> entry.getValue(); - default -> throw new IllegalArgumentException("unsupported column " + name); - }; - } - - private static String legacyColumn(Map.Entry entry, Object column) { - return switch (column.toString()) { - case "protocol" -> "JT808"; - case "identifier_type" -> "PLATE"; - case "identifier_value" -> entry.getKey(); - case "vin" -> entry.getValue(); - default -> throw new IllegalArgumentException("unsupported column " + column); - }; - } - - private ResultSet registrationResultSet() { - List rows = registrations.values().stream() - .filter(row -> bindings.containsKey(row.plate)) - .map(row -> new RegistrationRow(row.protocol, row.phone, bindings.get(row.plate), row.deviceId, row.plate)) - .toList(); - return registrationRowsResultSet(rows); - } - - private ResultSet legacyRegistrationResultSet() { - List rows = registrations.values().stream() - .filter(row -> row.vin != null && !row.vin.isBlank() && !"unknown".equalsIgnoreCase(row.vin)) - .toList(); - return registrationRowsResultSet(rows); - } - - private ResultSet registrationRowsResultSet(List rows) { - InvocationHandler handler = new InvocationHandler() { - private int index = -1; - - @Override - public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) { - return switch (method.getName()) { - case "next" -> { - index++; - yield index < rows.size(); - } - case "getString" -> registrationColumn(rows.get(index), args[0]); - case "close" -> null; - default -> defaultValue(method.getReturnType()); - }; - } - }; - return (ResultSet) Proxy.newProxyInstance( - ResultSet.class.getClassLoader(), new Class[]{ResultSet.class}, handler); - } - - private static String registrationColumn(RegistrationRow row, Object column) { - return switch (column.toString()) { - case "protocol" -> row.protocol; - case "phone" -> row.phone; - case "device_id" -> row.deviceId; - case "plate" -> row.plate; - case "vin" -> row.vin; - default -> throw new IllegalArgumentException("unsupported column " + column); - }; - } - - private static ResultSet resultSet(String vin) { - InvocationHandler handler = new InvocationHandler() { - private boolean read; - - @Override - public Object invoke(Object proxy, java.lang.reflect.Method method, Object[] args) { - return switch (method.getName()) { - case "next" -> { - boolean hasNext = !read && vin != null; - read = true; - yield hasNext; - } - case "getString" -> vin; - case "close" -> null; - default -> defaultValue(method.getReturnType()); - }; - } - }; - return (ResultSet) Proxy.newProxyInstance( - ResultSet.class.getClassLoader(), new Class[]{ResultSet.class}, handler); - } - - private static void set(List params, int oneBasedIndex, Object value) { - while (params.size() < oneBasedIndex) { - params.add(null); - } - params.set(oneBasedIndex - 1, value); - } - - private static String normalize(String value) { - return value == null ? "" : value.trim().toUpperCase(Locale.ROOT); - } - - private static Object defaultValue(Class type) { - if (type == Void.TYPE) return null; - if (type == Boolean.TYPE) return false; - if (type == Integer.TYPE) return 0; - if (type == Long.TYPE) return 0L; - if (type == Double.TYPE) return 0D; - if (type == Float.TYPE) return 0F; - if (type == Short.TYPE) return (short) 0; - if (type == Byte.TYPE) return (byte) 0; - return null; - } - - private static final class RegistrationRow { - private final String protocol; - private final String phone; - private String vin; - private final String deviceId; - private final String plate; - - private RegistrationRow(String protocol, String phone, String vin, String deviceId, String plate) { - this.protocol = protocol; - this.phone = phone; - this.vin = vin; - this.deviceId = deviceId; - this.plate = plate; - } - } + Integer rows = jdbc.queryForObject(""" + SELECT COUNT(*) + FROM vehicle_identity_binding + WHERE protocol = 'JT808' AND phone = '13079963289' + """, Integer.class); + assertThat(rows).isEqualTo(2); + assertThat(jdbc.queryForObject(""" + SELECT vin + FROM vehicle_identity_binding + WHERE protocol = 'JT808' AND phone = '13079963289' AND plate = '沪A00114F' + """, String.class)).isEqualTo("VIN-B2"); } } diff --git a/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/config/VehicleIdentityAutoConfigurationTest.java b/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/config/VehicleIdentityAutoConfigurationTest.java index 72588866..530646b7 100644 --- a/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/config/VehicleIdentityAutoConfigurationTest.java +++ b/modules/core/vehicle-identity/src/test/java/com/lingniu/ingest/identity/config/VehicleIdentityAutoConfigurationTest.java @@ -7,14 +7,7 @@ import org.junit.jupiter.api.io.TempDir; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import javax.sql.DataSource; import java.nio.file.Path; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.lang.reflect.Proxy; import static org.assertj.core.api.Assertions.assertThat; @@ -52,75 +45,19 @@ class VehicleIdentityAutoConfigurationTest { @Test void createsMysqlIdentityServiceWhenConfigured() { runner - .withBean(DataSource.class, StubDataSource::new) .withPropertyValues( "lingniu.ingest.identity.store=mysql", - "lingniu.ingest.identity.mysql.table=vehicle_identity_binding") + "lingniu.ingest.identity.mysql.jdbc-url=jdbc:h2:mem:auto-identity;MODE=MySQL;DATABASE_TO_UPPER=false;DB_CLOSE_DELAY=-1", + "lingniu.ingest.identity.mysql.username=sa", + "lingniu.ingest.identity.mysql.password=", + "lingniu.ingest.identity.mysql.table-name=vehicle_identity_bindings") .run(context -> { assertThat(context).hasSingleBean(VehicleIdentityResolver.class); - assertThat(context.getBean(VehicleIdentityResolver.class).getClass().getSimpleName()) - .isEqualTo("MySqlVehicleIdentityService"); + assertThat(context).hasSingleBean(VehicleIdentityRegistry.class); assertThat(context.getBean(VehicleIdentityResolver.class)) .isSameAs(context.getBean(VehicleIdentityRegistry.class)); + assertThat(context.getBean(VehicleIdentityResolver.class).getClass().getSimpleName()) + .isEqualTo("MySqlVehicleIdentityService"); }); } - - private static final class StubDataSource implements DataSource { - @Override public Connection getConnection() { return connection(); } - @Override public Connection getConnection(String username, String password) { return connection(); } - @Override public java.io.PrintWriter getLogWriter() { return null; } - @Override public void setLogWriter(java.io.PrintWriter out) {} - @Override public void setLoginTimeout(int seconds) {} - @Override public int getLoginTimeout() { return 0; } - @Override public java.util.logging.Logger getParentLogger() throws java.sql.SQLFeatureNotSupportedException { throw new java.sql.SQLFeatureNotSupportedException(); } - @Override public T unwrap(Class iface) { throw new UnsupportedOperationException(); } - @Override public boolean isWrapperFor(Class iface) { return false; } - - private static Connection connection() { - return (Connection) Proxy.newProxyInstance(Connection.class.getClassLoader(), new Class[]{Connection.class}, - (proxy, method, args) -> switch (method.getName()) { - case "createStatement" -> statement(); - case "prepareStatement" -> preparedStatement(); - case "close" -> null; - case "isClosed" -> false; - default -> defaultValue(method.getReturnType()); - }); - } - - private static Statement statement() { - return (Statement) Proxy.newProxyInstance(Statement.class.getClassLoader(), new Class[]{Statement.class}, - (proxy, method, args) -> switch (method.getName()) { - case "execute" -> false; - case "close" -> null; - default -> defaultValue(method.getReturnType()); - }); - } - - private static PreparedStatement preparedStatement() { - return (PreparedStatement) Proxy.newProxyInstance( - PreparedStatement.class.getClassLoader(), new Class[]{PreparedStatement.class}, - (proxy, method, args) -> switch (method.getName()) { - case "executeQuery" -> emptyResultSet(); - case "close" -> null; - default -> defaultValue(method.getReturnType()); - }); - } - - private static ResultSet emptyResultSet() { - return (ResultSet) Proxy.newProxyInstance(ResultSet.class.getClassLoader(), new Class[]{ResultSet.class}, - (proxy, method, args) -> switch (method.getName()) { - case "next" -> false; - case "close" -> null; - default -> defaultValue(method.getReturnType()); - }); - } - - private static Object defaultValue(Class type) { - if (type == Void.TYPE) return null; - if (type == Boolean.TYPE) return false; - if (type == Integer.TYPE) return 0; - if (type == Long.TYPE) return 0L; - return null; - } - } } diff --git a/modules/inbound/inbound-mqtt/pom.xml b/modules/inbound/inbound-mqtt/pom.xml index 06316ec8..8bcbe7b3 100644 --- a/modules/inbound/inbound-mqtt/pom.xml +++ b/modules/inbound/inbound-mqtt/pom.xml @@ -9,7 +9,7 @@ inbound-mqtt inbound-mqtt - MQTT 接入模块（支持多 endpoint，宇通等），Fusesource MQTT Client + 双向 TLS。 + MQTT 接入模块（支持多 endpoint，宇通等），Eclipse Paho MQTT Client + 双向 TLS。 @@ -34,8 +34,8 @@ true - org.fusesource.mqtt-client - mqtt-client + org.eclipse.paho + org.eclipse.paho.client.mqttv3 com.fasterxml.jackson.core diff --git a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttEndpointManager.java b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttEndpointManager.java index 66f8dd1d..ae2989bf 100644 --- a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttEndpointManager.java +++ b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttEndpointManager.java @@ -11,30 +11,33 @@ import com.lingniu.ingest.identity.VehicleIdentitySource; import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties; import com.lingniu.ingest.inbound.mqtt.model.MqttPayload; import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry; -import org.fusesource.mqtt.client.Future; -import org.fusesource.mqtt.client.FutureConnection; -import org.fusesource.mqtt.client.MQTT; -import org.fusesource.mqtt.client.Message; -import org.fusesource.mqtt.client.QoS; -import org.fusesource.mqtt.client.Topic; +import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; +import org.eclipse.paho.client.mqttv3.MqttAsyncClient; +import org.eclipse.paho.client.mqttv3.MqttCallbackExtended; +import org.eclipse.paho.client.mqttv3.MqttConnectOptions; +import org.eclipse.paho.client.mqttv3.MqttException; +import org.eclipse.paho.client.mqttv3.MqttMessage; +import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.EOFException; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; /** * MQTT 多 endpoint 生命周期管理。 * - *

为每个 endpoint 创建一个 fusesource MQTT FutureConnection → 连接 → 订阅 → 接收循环 → + *

为每个 endpoint 创建一个 Paho 异步客户端 → 连接 → 订阅 → 注册回调 → * 回调里解析 payload → {@link Dispatcher#dispatch(RawFrame)}。 * - *

每个 endpoint 在独立虚拟线程中阻塞接收，业务处理由 Dispatcher 侧的 Disruptor 承接。 + *

MQTT 回调不做业务处理，虚拟线程由 Dispatcher 侧的 Disruptor 承接。 */ public final class MqttEndpointManager implements AutoCloseable { @@ -44,9 +47,8 @@ public final class MqttEndpointManager implements AutoCloseable { private final MqttProfileRegistry profileRegistry; private final VehicleIdentityResolver identityResolver; private final Dispatcher dispatcher; - private final List connections = new ArrayList<>(); - private final List workers = new ArrayList<>(); - private volatile boolean running; + private final List clients = new ArrayList<>(); + private boolean started; public MqttEndpointManager(MqttInboundProperties props, MqttProfileRegistry profileRegistry, @@ -64,111 +66,78 @@ public final class MqttEndpointManager implements AutoCloseable { this.dispatcher = dispatcher; } - public void start() { - running = true; - log.info("mqtt endpoint manager starting, endpoints={}", props.getEndpoints().size()); + public synchronized void start() { + if (started) { + return; + } + started = true; + if (props.getEndpoints().isEmpty()) { + log.warn("mqtt inbound enabled but no endpoints configured"); + return; + } + log.info("mqtt inbound starting endpoints={}", props.getEndpoints().size()); for (MqttInboundProperties.Endpoint ep : props.getEndpoints()) { - Thread worker = Thread.ofVirtual() - .name("mqtt-endpoint-" + safeName(ep.getName())) - .start(() -> runEndpoint(ep)); - workers.add(worker); + try { + // 多 endpoint 互不影响：单个连接初始化失败会派发 operationalError，但不会阻止其他 endpoint。 + log.info("mqtt endpoint [{}] initializing uri={} topic={} profile={} qos={} cleanSession={} tlsHostnameVerification={}", + ep.getName(), ep.getUri(), ep.getTopic(), ep.getProfile(), ep.getQos(), ep.isCleanSession(), + ep.getTls() == null || ep.getTls().isHostnameVerificationEnabled()); + MqttAsyncClient client = buildClient(ep); + clients.add(client); + connectAndSubscribe(client, ep); + } catch (Exception e) { + log.error("mqtt endpoint [{}] init failed, skipped", ep.getName(), e); + publishOperationalError(ep, "init", e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage()); + } } } - private void runEndpoint(MqttInboundProperties.Endpoint ep) { - while (running && !Thread.currentThread().isInterrupted()) { - FutureConnection connection = null; - try { - log.info("mqtt endpoint [{}] initializing uri={} topic={} qos={} tls={}", - ep.getName(), ep.getUri(), ep.getTopic(), ep.getQos(), hasCustomTls(ep.getTls())); - MQTT mqtt = buildClient(ep); - connection = mqtt.futureConnection(); - connections.add(connection); - log.info("mqtt endpoint [{}] connect sending", ep.getName()); - connection.connect().await(); - log.info("mqtt endpoint [{}] connected", ep.getName()); - connection.subscribe(new Topic[]{new Topic(ep.getTopic(), mapQos(ep.getQos()))}).await(); - log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos()); - receiveLoop(ep, connection); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } catch (Exception e) { - if (isPeerDisconnect(e)) { - log.warn("mqtt endpoint [{}] peer disconnected, reconnecting: {}", ep.getName(), e.getMessage()); - } else { - log.error("mqtt endpoint [{}] connect/receive failed", ep.getName(), e); - publishOperationalError(ep, "connect", + private MqttAsyncClient buildClient(MqttInboundProperties.Endpoint ep) throws MqttException { + String clientId = ep.getClientId() == null || ep.getClientId().isBlank() + ? "lingniu-" + UUID.randomUUID() : ep.getClientId(); + return new MqttAsyncClient(pahoServerUri(ep.getUri()), clientId, new MemoryPersistence()); + } + + private void connectAndSubscribe(MqttAsyncClient client, MqttInboundProperties.Endpoint ep) throws MqttException { + client.setCallback(new MqttCallbackExtended() { + @Override + public void connectComplete(boolean reconnect, String serverURI) { + if (!reconnect) { + return; + } + try { + client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep)); + log.info("mqtt endpoint [{}] re-subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos()); + } catch (Exception e) { + log.error("mqtt endpoint [{}] re-subscribe failed topic={}", ep.getName(), ep.getTopic(), e); + publishOperationalError(ep, "subscribe", e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage()); } - sleepBeforeReconnect(ep); - } finally { - if (connection != null) { - connections.remove(connection); - try { - connection.disconnect().await(); - } catch (Exception ignored) { - // best-effort close - } - } } - } - } - private MQTT buildClient(MqttInboundProperties.Endpoint ep) throws Exception { - MQTT mqtt = new MQTT(); - mqtt.setHost(ep.getUri()); - if (ep.getClientId() != null && !ep.getClientId().isBlank()) { - mqtt.setClientId(ep.getClientId()); - } - mqtt.setCleanSession(ep.isCleanSession()); - if (ep.getUsername() != null && !ep.getUsername().isBlank()) { - mqtt.setUserName(ep.getUsername()); - mqtt.setPassword(ep.getPassword() == null ? "" : ep.getPassword()); - } - mqtt.setConnectAttemptsMax(1); - mqtt.setReconnectAttemptsMax(0); - if (ep.getKeepAliveSeconds() > 0) { - mqtt.setKeepAlive((short) ep.getKeepAliveSeconds()); - } - mqtt.setVersion("3.1.1"); - if (hasCustomTls(ep.getTls())) { - mqtt.setSslContext(MqttTlsSupport.buildSslContext(ep.getTls())); - } - return mqtt; - } - - private void receiveLoop(MqttInboundProperties.Endpoint ep, FutureConnection connection) throws Exception { - while (running && !Thread.currentThread().isInterrupted()) { - Future futureMessage = connection.receive(); - Message message = futureMessage.await(); - try { - message.ack(); - onMessage(ep, message.getTopic(), message.getPayload()); - } catch (Exception e) { - log.warn("mqtt endpoint [{}] message ack/dispatch failed topic={} len={}", - ep.getName(), message.getTopic(), message.getPayload() == null ? 0 : message.getPayload().length, e); + @Override + public void connectionLost(Throwable cause) { + log.warn("mqtt endpoint [{}] connection lost", ep.getName(), cause); } - } - } - private boolean isPeerDisconnect(Exception e) { - Throwable current = e; - while (current != null) { - if (current instanceof EOFException) { - return true; + @Override + public void messageArrived(String topic, MqttMessage message) { + log.info("mqtt endpoint [{}] received topic={} bytes={}", ep.getName(), topic, message.getPayload().length); + onMessage(ep, topic, message.getPayload()); } - current = current.getCause(); - } - return false; - } - private void sleepBeforeReconnect(MqttInboundProperties.Endpoint ep) { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + @Override + public void deliveryComplete(IMqttDeliveryToken token) { + // inbound only + } + }); + + MqttConnectOptions options = connectOptions(ep); + client.connect(options).waitForCompletion(timeoutMs(ep)); + log.info("mqtt endpoint [{}] connected serverUri={} clientId={}", + ep.getName(), client.getServerURI(), client.getClientId()); + client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep)); + log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos()); } private void onMessage(MqttInboundProperties.Endpoint ep, String topic, byte[] payload) { @@ -319,11 +288,58 @@ public final class MqttEndpointManager implements AutoCloseable { return value.replace("\\", "\\\\").replace("\"", "\\\""); } - private static QoS mapQos(int level) { + private static MqttConnectOptions connectOptions(MqttInboundProperties.Endpoint ep) { + MqttConnectOptions options = new MqttConnectOptions(); + options.setAutomaticReconnect(true); + options.setCleanSession(ep.isCleanSession()); + if (ep.getKeepAliveSeconds() > 0) { + options.setKeepAliveInterval(ep.getKeepAliveSeconds()); + } + if (ep.getConnectionTimeoutSeconds() > 0) { + options.setConnectionTimeout(ep.getConnectionTimeoutSeconds()); + } + if (hasText(ep.getUsername())) { + options.setUserName(ep.getUsername()); + options.setPassword(ep.getPassword() == null ? new char[0] : ep.getPassword().toCharArray()); + } + if (isSslUri(ep.getUri()) && hasCustomTls(ep.getTls())) { + options.setSocketFactory(MqttTlsSupport.socketFactory(ep.getTls())); + if (ep.getTls() != null && !ep.getTls().isHostnameVerificationEnabled()) { + options.setSSLHostnameVerifier((hostname, session) -> true); + } + } + return options; + } + + private static String pahoServerUri(String value) { + URI uri = URI.create(value); + String scheme = uri.getScheme(); + String pahoScheme = switch (scheme == null ? "" : scheme.toLowerCase()) { + case "ssl", "tls", "mqtts" -> "ssl"; + default -> "tcp"; + }; + int port = uri.getPort() > 0 ? uri.getPort() : defaultPort(scheme); + return pahoScheme + "://" + uri.getHost() + ":" + port; + } + + private static int mapQos(int level) { return switch (level) { - case 0 -> QoS.AT_MOST_ONCE; - case 2 -> QoS.EXACTLY_ONCE; - default -> QoS.AT_LEAST_ONCE; + case 0 -> 0; + case 2 -> 2; + default -> 1; + }; + } + + private static long timeoutMs(MqttInboundProperties.Endpoint ep) { + int seconds = ep.getConnectionTimeoutSeconds() > 0 ? ep.getConnectionTimeoutSeconds() : 10; + return TimeUnit.SECONDS.toMillis(seconds); + } + + private static int defaultPort(String scheme) { + if (scheme == null) return 1883; + return switch (scheme.toLowerCase()) { + case "ssl", "tls", "mqtts" -> 8883; + default -> 1883; }; } @@ -331,13 +347,6 @@ public final class MqttEndpointManager implements AutoCloseable { return value != null && value.trim().length() == 17; } - private static String safeName(String value) { - if (value == null || value.isBlank()) { - return "unnamed"; - } - return value.replaceAll("[^a-zA-Z0-9_.-]", "_"); - } - private record IdentityResolution(VehicleIdentity identity, String errorMessage) {} private static boolean hasCustomTls(MqttInboundProperties.Tls tls) { @@ -345,23 +354,30 @@ public final class MqttEndpointManager implements AutoCloseable { || (hasText(tls.getClientPem()) && hasText(tls.getClientKey()))); } + private static boolean isSslUri(String value) { + String scheme = URI.create(value).getScheme(); + return scheme != null && switch (scheme.toLowerCase()) { + case "ssl", "tls", "mqtts" -> true; + default -> false; + }; + } + private static boolean hasText(String value) { return value != null && !value.isBlank(); } @Override public void close() { - running = false; - for (FutureConnection connection : List.copyOf(connections)) { + for (MqttAsyncClient c : clients) { try { - connection.disconnect().await(); + if (c.isConnected()) { + c.disconnectForcibly(1000, 1000, false); + } + c.close(true); } catch (Exception ignored) { // best-effort } } - for (Thread worker : workers) { - worker.interrupt(); - } - log.info("mqtt endpoint manager stopped, connections={}", connections.size()); + log.info("mqtt endpoint manager stopped, clients={}", clients.size()); } } diff --git a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttTlsSupport.java b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttTlsSupport.java index 527b3d1d..2983c7a9 100644 --- a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttTlsSupport.java +++ b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttTlsSupport.java @@ -2,19 +2,31 @@ package com.lingniu.ingest.inbound.mqtt.client; import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties; +import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; -import java.io.InputStream; +import javax.net.ssl.X509ExtendedTrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyFactory; import java.security.KeyStore; import java.security.PrivateKey; -import java.security.SecureRandom; import java.security.cert.Certificate; +import java.security.cert.CertificateException; import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; import java.security.spec.PKCS8EncodedKeySpec; import java.util.ArrayList; import java.util.Base64; @@ -25,93 +37,81 @@ public final class MqttTlsSupport { private MqttTlsSupport() {} - public static SSLContext buildSslContext(MqttInboundProperties.Tls tls) { + public static SSLSocketFactory socketFactory(MqttInboundProperties.Tls tls) { try { - TrustManagerFactory tmf = null; - KeyManagerFactory kmf = null; + KeyManager[] keyManagers = null; + TrustManager[] trustManagers = null; if (tls != null && hasText(tls.getCaPem())) { - tmf = trustManagerFactory(Path.of(tls.getCaPem())); + trustManagers = trustManagerFactory(Path.of(tls.getCaPem())).getTrustManagers(); + if (!tls.isHostnameVerificationEnabled()) { + trustManagers = withoutHostnameVerification(trustManagers); + } } if (tls != null && hasText(tls.getClientPem()) && hasText(tls.getClientKey())) { - kmf = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey())); + keyManagers = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey())).getKeyManagers(); } - SSLContext context = SSLContext.getInstance("TLSv1.2"); - context.init( - kmf == null ? null : kmf.getKeyManagers(), - tmf == null ? null : tmf.getTrustManagers(), - new SecureRandom()); - return context; + SSLContext context = SSLContext.getInstance("TLS"); + context.init(keyManagers, trustManagers, null); + SSLSocketFactory factory = context.getSocketFactory(); + if (tls != null && !tls.isHostnameVerificationEnabled()) { + return new HostnameVerificationDisablingSocketFactory(factory); + } + return factory; } catch (Exception e) { throw new IllegalStateException("mqtt tls config build failed", e); } } - private static TrustManagerFactory trustManagerFactory(Path caPath) throws Exception { + private static TrustManagerFactory trustManagerFactory(Path caPem) throws Exception { KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load(null, null); int i = 0; - for (Certificate cert : readCertificates(caPath)) { + for (Certificate cert : readCertificates(caPem)) { trustStore.setCertificateEntry("ca-" + i++, cert); } - TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX"); + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); tmf.init(trustStore); return tmf; } - private static KeyManagerFactory keyManagerFactory(Path clientCertPath, Path clientKeyPath) throws Exception { - List chain = readCertificates(clientCertPath); - PrivateKey key = readPrivateKey(clientKeyPath); + private static KeyManagerFactory keyManagerFactory(Path clientPem, Path clientKey) throws Exception { + List chain = readCertificates(clientPem); + PrivateKey key = readPrivateKey(clientKey); KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); keyStore.load(null, null); char[] password = new char[0]; - keyStore.setCertificateEntry("client", chain.getFirst()); - keyStore.setKeyEntry("clientKey", key, password, chain.toArray(Certificate[]::new)); - KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX"); + keyStore.setKeyEntry("mqtt-client", key, password, chain.toArray(Certificate[]::new)); + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); kmf.init(keyStore, password); return kmf; } - private static List readCertificates(Path path) throws Exception { + private static List readCertificates(Path pem) throws Exception { + String text = Files.readString(pem, StandardCharsets.UTF_8); + List certificates = new ArrayList<>(); CertificateFactory factory = CertificateFactory.getInstance("X.509"); - String text = readUtf8IfPossible(path); - if (text != null && text.contains("-----BEGIN CERTIFICATE-----")) { - List certificates = new ArrayList<>(); - for (String block : pemBlocks(text, "CERTIFICATE")) { - certificates.add(factory.generateCertificate( - new java.io.ByteArrayInputStream(Base64.getMimeDecoder().decode(block)))); - } - if (!certificates.isEmpty()) { - return certificates; + for (String block : pemBlocks(text, "CERTIFICATE")) { + try (ByteArrayInputStream in = new ByteArrayInputStream(Base64.getMimeDecoder().decode(block))) { + certificates.add(factory.generateCertificate(in)); } } - try (InputStream in = Files.newInputStream(path)) { - Collection certificates = factory.generateCertificates(in); - if (certificates.isEmpty()) { - throw new IllegalArgumentException("no certificate found in " + path); - } - return List.copyOf(certificates); + if (certificates.isEmpty()) { + throw new IllegalArgumentException("no certificate found in " + pem); } + return certificates; } - private static PrivateKey readPrivateKey(Path path) throws Exception { - String text = Files.readString(path, StandardCharsets.UTF_8); + private static PrivateKey readPrivateKey(Path pem) throws Exception { + String text = Files.readString(pem, StandardCharsets.UTF_8); List blocks = pemBlocks(text, "PRIVATE KEY"); if (blocks.isEmpty()) { - throw new IllegalArgumentException("no PKCS#8 private key found in " + path); + throw new IllegalArgumentException("no PKCS#8 private key found in " + pem); } byte[] der = Base64.getMimeDecoder().decode(blocks.getFirst()); PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der); return KeyFactory.getInstance("RSA").generatePrivate(spec); } - private static String readUtf8IfPossible(Path path) { - try { - return Files.readString(path, StandardCharsets.UTF_8); - } catch (Exception ignored) { - return null; - } - } - private static List pemBlocks(String text, String type) { String begin = "-----BEGIN " + type + "-----"; String end = "-----END " + type + "-----"; @@ -132,4 +132,129 @@ public final class MqttTlsSupport { private static boolean hasText(String value) { return value != null && !value.isBlank(); } + + private static TrustManager[] withoutHostnameVerification(TrustManager[] managers) { + TrustManager[] wrapped = new TrustManager[managers.length]; + for (int i = 0; i < managers.length; i++) { + TrustManager manager = managers[i]; + if (manager instanceof X509TrustManager x509) { + wrapped[i] = new ChainOnlyTrustManager(x509); + } else { + wrapped[i] = manager; + } + } + return wrapped; + } + + private static final class ChainOnlyTrustManager extends X509ExtendedTrustManager { + private final X509TrustManager delegate; + + private ChainOnlyTrustManager(X509TrustManager delegate) { + this.delegate = delegate; + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) + throws CertificateException { + if (delegate instanceof X509ExtendedTrustManager extended) { + extended.checkClientTrusted(chain, authType, socket); + } else { + delegate.checkClientTrusted(chain, authType); + } + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) + throws CertificateException { + delegate.checkServerTrusted(chain, authType); + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) + throws CertificateException { + if (delegate instanceof X509ExtendedTrustManager extended) { + extended.checkClientTrusted(chain, authType, engine); + } else { + delegate.checkClientTrusted(chain, authType); + } + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) + throws CertificateException { + delegate.checkServerTrusted(chain, authType); + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + delegate.checkClientTrusted(chain, authType); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + delegate.checkServerTrusted(chain, authType); + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return delegate.getAcceptedIssuers(); + } + } + + private static final class HostnameVerificationDisablingSocketFactory extends SSLSocketFactory { + private final SSLSocketFactory delegate; + + private HostnameVerificationDisablingSocketFactory(SSLSocketFactory delegate) { + this.delegate = delegate; + } + + @Override + public String[] getDefaultCipherSuites() { + return delegate.getDefaultCipherSuites(); + } + + @Override + public String[] getSupportedCipherSuites() { + return delegate.getSupportedCipherSuites(); + } + + @Override + public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException { + return prepare(delegate.createSocket(socket, host, port, autoClose)); + } + + @Override + public Socket createSocket() throws IOException { + return prepare(delegate.createSocket()); + } + + @Override + public Socket createSocket(String host, int port) throws IOException { + return prepare(delegate.createSocket(host, port)); + } + + @Override + public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException { + return prepare(delegate.createSocket(host, port, localHost, localPort)); + } + + @Override + public Socket createSocket(InetAddress host, int port) throws IOException { + return prepare(delegate.createSocket(host, port)); + } + + @Override + public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException { + return prepare(delegate.createSocket(address, port, localAddress, localPort)); + } + + private static Socket prepare(Socket socket) { + if (socket instanceof SSLSocket sslSocket) { + SSLParameters parameters = sslSocket.getSSLParameters(); + parameters.setEndpointIdentificationAlgorithm(null); + sslSocket.setSSLParameters(parameters); + } + return socket; + } + } } diff --git a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundAutoConfiguration.java b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundAutoConfiguration.java index e0f72f57..1d4d2839 100644 --- a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundAutoConfiguration.java +++ b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundAutoConfiguration.java @@ -10,19 +10,20 @@ import com.lingniu.ingest.inbound.mqtt.parser.MqttPayloadParser; import com.lingniu.ingest.inbound.mqtt.profile.MqttProfile; import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry; import com.lingniu.ingest.inbound.mqtt.profile.YutongMqttProfile; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.ApplicationRunner; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.Environment; +import java.util.ArrayList; import java.util.List; @AutoConfiguration @EnableConfigurationProperties(MqttInboundProperties.class) -@ConditionalOnProperty(prefix = "lingniu.ingest.mqtt", name = "enabled", havingValue = "true") +@Conditional(MqttInboundEnabledCondition.class) public class MqttInboundAutoConfiguration { @Bean @@ -61,25 +62,144 @@ public class MqttInboundAutoConfiguration { return new MqttRealtimeHandler(profileRegistry); } - @Bean(destroyMethod = "close") + @Bean(initMethod = "start", destroyMethod = "close") + @Lazy(false) @ConditionalOnMissingBean public MqttEndpointManager mqttEndpointManager(MqttInboundProperties props, MqttProfileRegistry profileRegistry, VehicleIdentityResolver identityResolver, - Dispatcher dispatcher) { + Dispatcher dispatcher, + Environment environment) { + applyLegacySingleEndpointFallback(props, environment); return new MqttEndpointManager(props, profileRegistry, identityResolver, dispatcher); } - /** - * Spring Boot 启动完成后触发 MQTT 连接，避免与 Netty / Kafka 启动竞争。 - */ - @Bean - public ApplicationRunner mqttStartupRunner(MqttEndpointManager manager) { - return new ApplicationRunner() { - @Override - public void run(ApplicationArguments args) { - manager.start(); + private static void applyLegacySingleEndpointFallback(MqttInboundProperties props, Environment env) { + if (!props.getEndpoints().isEmpty()) { + return; + } + String uri = firstText(env, + "lingniu.ingest.mqtt.endpoints[0].uri", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_URI", + "MQTT_URI", + "YUTONG_MQTT_URI"); + if (uri.isBlank() || uri.contains("${")) { + return; + } + MqttInboundProperties.Endpoint endpoint = new MqttInboundProperties.Endpoint(); + endpoint.setName(defaultText(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].name", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_NAME", + "MQTT_ENDPOINT_NAME", + "YUTONG_MQTT_ENDPOINT_NAME"), "yutong")); + endpoint.setUri(uri); + endpoint.setTopic(defaultText(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].topic", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_TOPIC", + "MQTT_TOPIC", + "YUTONG_MQTT_TOPIC"), "#")); + endpoint.setQos(intValue(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].qos", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_QOS", + "MQTT_QOS", + "YUTONG_MQTT_QOS"), 1)); + endpoint.setClientId(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].client-id", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLIENT_ID", + "MQTT_CLIENT_ID", + "YUTONG_MQTT_CLIENT_ID")); + endpoint.setUsername(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].username", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_USERNAME", + "MQTT_USERNAME", + "YUTONG_MQTT_USERNAME")); + endpoint.setPassword(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].password", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_PASSWORD", + "MQTT_PASSWORD", + "YUTONG_MQTT_PASSWORD")); + endpoint.setCleanSession(booleanValue(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].clean-session", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLEAN_SESSION", + "MQTT_CLEAN_SESSION", + "YUTONG_MQTT_CLEAN_SESSION"), false)); + endpoint.setKeepAliveSeconds(intValue(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].keep-alive-seconds", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_KEEP_ALIVE_SECONDS", + "MQTT_KEEP_ALIVE_SECONDS", + "YUTONG_MQTT_KEEP_ALIVE_SECONDS"), 20)); + endpoint.setConnectionTimeoutSeconds(intValue(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].connection-timeout-seconds", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_CONNECTION_TIMEOUT_SECONDS", + "MQTT_CONNECTION_TIMEOUT_SECONDS", + "YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS"), 10)); + endpoint.setProfile(defaultText(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].profile", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_PROFILE", + "MQTT_PROFILE", + "YUTONG_MQTT_PROFILE"), "yutong")); + MqttInboundProperties.Tls tls = endpoint.getTls(); + tls.setCaPem(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].tls.ca-pem", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CA_PEM", + "MQTT_TLS_CA_PEM", + "YUTONG_MQTT_TLS_CA_PEM")); + tls.setClientPem(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].tls.client-pem", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_PEM", + "MQTT_TLS_CLIENT_PEM", + "YUTONG_MQTT_TLS_CLIENT_PEM")); + tls.setClientKey(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].tls.client-key", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_KEY", + "MQTT_TLS_CLIENT_KEY", + "YUTONG_MQTT_TLS_CLIENT_KEY")); + tls.setHostnameVerificationEnabled(booleanValue(firstText(env, + "lingniu.ingest.mqtt.endpoints[0].tls.hostname-verification-enabled", + "LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_HOSTNAME_VERIFICATION_ENABLED", + "MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", + "YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED"), true)); + props.setEndpoints(new ArrayList<>(List.of(endpoint))); + } + + private static String firstText(Environment env, String... keys) { + for (String key : keys) { + String value = env.getProperty(key); + if (value != null && !value.isBlank() && !value.contains("${")) { + return value; } - }; + } + return ""; + } + + private static String defaultText(String value, String fallback) { + return value == null || value.isBlank() ? fallback : value; + } + + private static int intValue(String value, int fallback) { + if (value == null || value.isBlank()) { + return fallback; + } + try { + return Integer.parseInt(value.trim()); + } catch (NumberFormatException ex) { + return fallback; + } + } + + private static boolean booleanValue(String value, boolean fallback) { + if (value == null || value.isBlank()) { + return fallback; + } + String normalized = value.trim().toLowerCase(); + if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized) + || "on".equals(normalized)) { + return true; + } + if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized) + || "off".equals(normalized)) { + return false; + } + return fallback; } } diff --git a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundEnabledCondition.java b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundEnabledCondition.java new file mode 100644 index 00000000..a7397679 --- /dev/null +++ b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundEnabledCondition.java @@ -0,0 +1,41 @@ +package com.lingniu.ingest.inbound.mqtt.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +final class MqttInboundEnabledCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + Environment env = context.getEnvironment(); + Boolean direct = parseBoolean(env.getProperty("lingniu.ingest.mqtt.enabled")); + if (direct != null) { + return direct; + } + return isTrue(env.getProperty("MQTT_ENABLED")) + || isTrue(env.getProperty("YUTONG_MQTT_ENABLED")); + } + + private static boolean isTrue(String value) { + Boolean parsed = parseBoolean(value); + return parsed != null && parsed; + } + + private static Boolean parseBoolean(String value) { + if (value == null || value.isBlank() || value.contains("${")) { + return null; + } + String normalized = value.trim().toLowerCase(); + if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized) + || "on".equals(normalized)) { + return true; + } + if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized) + || "off".equals(normalized)) { + return false; + } + return null; + } +} diff --git a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundProperties.java b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundProperties.java index d5241893..ca1535fc 100644 --- a/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundProperties.java +++ b/modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundProperties.java @@ -82,6 +82,8 @@ public class MqttInboundProperties { public String getClientKey() { return clientKey; } public void setClientKey(String clientKey) { this.clientKey = clientKey; } public boolean isHostnameVerificationEnabled() { return hostnameVerificationEnabled; } - public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) { this.hostnameVerificationEnabled = hostnameVerificationEnabled; } + public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) { + this.hostnameVerificationEnabled = hostnameVerificationEnabled; + } } } diff --git a/modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttEndpointManagerTest.java b/modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttEndpointManagerTest.java index a1c94db6..138c7392 100644 --- a/modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttEndpointManagerTest.java +++ b/modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttEndpointManagerTest.java @@ -172,7 +172,7 @@ class MqttEndpointManagerTest { .containsEntry("endpoint", "bad-endpoint") .containsEntry("profile", "yutong") .containsEntry("operationalError", "true") - .containsEntry("phase", "connect") + .containsEntry("phase", "init") .containsEntry("identityResolved", "false") .containsEntry("identitySource", "UNKNOWN"); }); @@ -184,7 +184,7 @@ class MqttEndpointManagerTest { .containsEntry("operationalError", "true") .containsEntry("endpoint", "bad-endpoint") .containsEntry("profile", "yutong") - .containsEntry("phase", "connect"); + .containsEntry("phase", "init"); }); batchExecutor.close(); diff --git a/modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttTlsSupportTest.java b/modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttTlsSupportTest.java index 769b04c1..1e9ab16e 100644 --- a/modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttTlsSupportTest.java +++ b/modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttTlsSupportTest.java @@ -5,11 +5,10 @@ import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.net.ssl.SSLSocketFactory; import java.nio.file.Files; import java.nio.file.Path; -import javax.net.ssl.SSLContext; - import static org.assertj.core.api.Assertions.assertThat; class MqttTlsSupportTest { @@ -24,9 +23,9 @@ class MqttTlsSupportTest { MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls(); tls.setCaPem(ca.toString()); - SSLContext config = MqttTlsSupport.buildSslContext(tls); + SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls); - assertThat(config.getProtocol()).isEqualTo("TLSv1.2"); + assertThat(factory).isNotNull(); } @Test @@ -42,22 +41,9 @@ class MqttTlsSupportTest { tls.setClientPem(cert.toString()); tls.setClientKey(key.toString()); - SSLContext config = MqttTlsSupport.buildSslContext(tls); + SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls); - assertThat(config.getProtocol()).isEqualTo("TLSv1.2"); - } - - @Test - void hostnameVerificationFlagDoesNotBlockSslContextCreation() throws Exception { - Path ca = tempDir.resolve("ca.pem"); - Files.writeString(ca, CERT); - MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls(); - tls.setCaPem(ca.toString()); - tls.setHostnameVerificationEnabled(false); - - SSLContext config = MqttTlsSupport.buildSslContext(tls); - - assertThat(config.getProtocol()).isEqualTo("TLSv1.2"); + assertThat(factory).isNotNull(); } private static final String CERT = """ diff --git a/modules/inbound/inbound-xinda-push/pom.xml b/modules/inbound/inbound-xinda-push/pom.xml index 00e6e4ed..27e97eb3 100644 --- a/modules/inbound/inbound-xinda-push/pom.xml +++ b/modules/inbound/inbound-xinda-push/pom.xml @@ -41,6 +41,30 @@ io.netty netty-all + + io.netty + netty-common + + + io.netty + netty-buffer + + + io.netty + netty-transport + + + io.netty + netty-resolver + + + io.netty + netty-codec + + + io.netty + netty-handler + com.fasterxml.jackson.core jackson-databind diff --git a/modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushClient.java b/modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushClient.java index 76d043d2..9715b9a1 100644 --- a/modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushClient.java +++ b/modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushClient.java @@ -6,6 +6,7 @@ import com.gps31.push.netty.PushClient; import com.gps31.push.netty.PushMsg; import com.gps31.push.netty.client.TcpClient; import com.lingniu.ingest.api.ProtocolId; +import com.lingniu.ingest.api.event.RawArchiveKeys; import com.lingniu.ingest.api.pipeline.RawFrame; import com.lingniu.ingest.core.dispatcher.Dispatcher; import com.lingniu.ingest.identity.InMemoryVehicleIdentityService; @@ -220,6 +221,7 @@ public class XindaPushClient extends PushClient { meta.put("parseError", "true"); meta.put("parseErrorMessage", parsed.errorMessage()); } + meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(payload.command(), json, parsed)); dispatcher.dispatch(new RawFrame( ProtocolId.XINDA_PUSH, payload.commandCode(), @@ -247,6 +249,8 @@ public class XindaPushClient extends PushClient { meta.put("operationalError", "true"); meta.put("phase", phase == null ? "" : phase); meta.put("reason", reason == null ? "" : reason); + meta.put(RawArchiveKeys.META_PARSED_JSON, + parsedJson(payload.command(), payload.json(), new JsonParseResult(json, false, ""))); dispatcher.dispatch(new RawFrame( ProtocolId.XINDA_PUSH, payload.commandCode(), @@ -271,6 +275,8 @@ public class XindaPushClient extends PushClient { meta.put("operationalError", "true"); meta.put("phase", "message"); meta.put("reason", reason == null ? "" : reason); + JsonParseResult parsed = parseForMetadata(json); + meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(cmd, json, parsed)); dispatcher.dispatch(new RawFrame( ProtocolId.XINDA_PUSH, 0, @@ -388,6 +394,19 @@ public class XindaPushClient extends PushClient { } } + private static String parsedJson(String command, String rawJson, JsonParseResult parsed) { + JSONObject root = new JSONObject(true); + root.put("command", command == null ? "" : command); + root.put("parseError", parsed != null && parsed.parseError()); + if (parsed != null && parsed.parseError()) { + root.put("parseErrorMessage", parsed.errorMessage() == null ? "" : parsed.errorMessage()); + root.put("rawJson", rawJson == null ? "" : rawJson); + } else { + root.put("body", parsed == null || parsed.json() == null ? new JSONObject() : parsed.json()); + } + return root.toJSONString(); + } + private record JsonParseResult(JSONObject json, boolean parseError, String errorMessage) {} private record IdentityResolution(VehicleIdentity identity, String errorMessage) {} diff --git a/modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapper.java b/modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapper.java index 1956a727..e9002546 100644 --- a/modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapper.java +++ b/modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapper.java @@ -185,7 +185,6 @@ public final class XindaPushEventMapper implements EventMapper Map out = new java.util.LinkedHashMap<>(); out.put("source", "xinda-push"); out.put("cmd", message.command()); - out.put("rawJson", message.json()); out.put("vin", normalizedVin(identity)); out.put("externalVin", firstNonBlank(json, "vin")); out.put("phone", phone(json)); diff --git a/modules/inbound/inbound-xinda-push/src/test/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapperTest.java b/modules/inbound/inbound-xinda-push/src/test/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapperTest.java index 74da45ed..6d8802c4 100644 --- a/modules/inbound/inbound-xinda-push/src/test/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapperTest.java +++ b/modules/inbound/inbound-xinda-push/src/test/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapperTest.java @@ -37,7 +37,8 @@ class XindaPushEventMapperTest { .containsEntry("externalVin", "LNVIN000000000101") .containsEntry("phone", "13800138000") .containsEntry("deviceId", "XD-101") - .containsEntry("plateNo", "粤B10101"); + .containsEntry("plateNo", "粤B10101") + .doesNotContainKey("rawJson"); } @Test diff --git a/modules/protocols/protocol-gb32960/src/test/java/com/lingniu/ingest/protocol/gb32960/inbound/Gb32960ChannelHandlerAckBoundaryTest.java b/modules/protocols/protocol-gb32960/src/test/java/com/lingniu/ingest/protocol/gb32960/inbound/Gb32960ChannelHandlerAckBoundaryTest.java index 4522bb79..4ce6e1a4 100644 --- a/modules/protocols/protocol-gb32960/src/test/java/com/lingniu/ingest/protocol/gb32960/inbound/Gb32960ChannelHandlerAckBoundaryTest.java +++ b/modules/protocols/protocol-gb32960/src/test/java/com/lingniu/ingest/protocol/gb32960/inbound/Gb32960ChannelHandlerAckBoundaryTest.java @@ -43,7 +43,7 @@ class Gb32960ChannelHandlerAckBoundaryTest { dispatch.awaitPublishCount(1); dispatch.completeDurability(); - ByteBuf ack = readOutboundAfterRunningTasks(channel); + ByteBuf ack = awaitOutbound(channel); assertThat(ack).isNotNull(); assertThat(ack.getUnsignedByte(2)).isEqualTo((short) 0x02); assertThat(ack.getUnsignedByte(3)).isEqualTo((short) 0x01); @@ -85,8 +85,8 @@ class Gb32960ChannelHandlerAckBoundaryTest { dispatch.completeDurability(0); - ByteBuf firstAck = readOutboundAfterRunningTasks(channel); - ByteBuf secondAck = readOutboundAfterRunningTasks(channel); + ByteBuf firstAck = awaitOutbound(channel); + ByteBuf secondAck = awaitOutbound(channel); assertThat(firstAck).isNotNull(); assertThat(secondAck).isNotNull(); assertThat(firstAck.getUnsignedByte(2)).isEqualTo((short) 0x02); @@ -97,42 +97,6 @@ class Gb32960ChannelHandlerAckBoundaryTest { } } - @Test - void decodeFailureIsArchivedWithParseErrorMetadata() { - try (DispatchHarness dispatch = new DispatchHarness()) { - EmbeddedChannel channel = new EmbeddedChannel(handlerWith(dispatch.dispatcher())); - byte[] malformed = new byte[]{0x23, 0x23, 0x02}; - - assertThat(channel.writeInbound(malformed)).isFalse(); - - dispatch.awaitPublishCount(1); - VehicleEvent event = dispatch.event(0); - assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class); - VehicleEvent.RawArchive raw = (VehicleEvent.RawArchive) event; - assertThat(raw.source()).isEqualTo(com.lingniu.ingest.api.ProtocolId.GB32960); - assertThat(raw.command()).isZero(); - assertThat(raw.rawBytes()).containsExactly(malformed); - assertThat(raw.metadata()) - .containsEntry("vin", "unknown") - .containsEntry("parseError", "true") - .containsKey("parseErrorMessage"); - channel.finishAndReleaseAll(); - } - } - - private static ByteBuf readOutboundAfterRunningTasks(EmbeddedChannel channel) { - long deadline = System.currentTimeMillis() + 3000; - while (System.currentTimeMillis() < deadline) { - channel.runPendingTasks(); - ByteBuf outbound = channel.readOutbound(); - if (outbound != null) { - return outbound; - } - Thread.onSpinWait(); - } - return null; - } - private static Gb32960ChannelHandler handlerWith(Dispatcher dispatcher) { return new Gb32960ChannelHandler( decoder(), @@ -174,6 +138,19 @@ class Gb32960ChannelHandlerAckBoundaryTest { return out.toByteArray(); } + private static ByteBuf awaitOutbound(EmbeddedChannel channel) { + long deadline = System.currentTimeMillis() + 3000; + while (System.currentTimeMillis() < deadline) { + channel.runPendingTasks(); + ByteBuf outbound = channel.readOutbound(); + if (outbound != null) { + return outbound; + } + Thread.onSpinWait(); + } + return null; + } + private static final class DispatchHarness implements AutoCloseable { private final ControlledSink sink = new ControlledSink(); private final DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink)); @@ -214,12 +191,6 @@ class Gb32960ChannelHandlerAckBoundaryTest { throw new AssertionError("expected " + count + " sink publishes, actual=" + sink.futures.size()); } - VehicleEvent event(int index) { - synchronized (sink.futures) { - return sink.events.get(index); - } - } - @Override public void close() { batchExecutor.close(); @@ -229,7 +200,6 @@ class Gb32960ChannelHandlerAckBoundaryTest { private static final class ControlledSink implements EventSink { private final List> futures = new ArrayList<>(); - private final List events = new ArrayList<>(); @Override public String name() { @@ -240,7 +210,6 @@ class Gb32960ChannelHandlerAckBoundaryTest { public CompletableFuture publish(VehicleEvent event) { CompletableFuture future = new CompletableFuture<>(); synchronized (futures) { - events.add(event); futures.add(future); } return future; diff --git a/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandler.java b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandler.java index 43a15f68..efc26b63 100644 --- a/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandler.java +++ b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandler.java @@ -9,13 +9,14 @@ import com.lingniu.ingest.identity.VehicleIdentityLookup; import com.lingniu.ingest.identity.VehicleIdentityRegistry; import com.lingniu.ingest.identity.VehicleIdentityResolver; import com.lingniu.ingest.identity.VehicleIdentitySource; -import com.lingniu.ingest.identity.VehicleRegistrationBinding; import com.lingniu.ingest.protocol.jt808.codec.Jt808FrameEncoder; import com.lingniu.ingest.protocol.jt808.codec.Jt808MalformedFrame; import com.lingniu.ingest.protocol.jt808.codec.Jt808MessageDecoder; import com.lingniu.ingest.protocol.jt808.downlink.Jt808Commands; import com.lingniu.ingest.protocol.jt808.model.Jt808Body; import com.lingniu.ingest.protocol.jt808.model.Jt808Header; +import com.lingniu.ingest.protocol.jt808.model.Jt808LocationAdditionalInfo; +import com.lingniu.ingest.protocol.jt808.model.Jt808LocationFlagInfo; import com.lingniu.ingest.protocol.jt808.model.Jt808Message; import com.lingniu.ingest.protocol.jt808.model.Jt808MessageId; import com.lingniu.ingest.protocol.jt808.session.Jt808ChannelRegistry; @@ -25,17 +26,18 @@ import com.lingniu.ingest.session.SessionStore; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.util.AttributeKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.UUID; -import java.util.concurrent.CompletableFuture; /** * JT808 Netty 入站处理器。职责： @@ -51,8 +53,6 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { private static final Logger log = LoggerFactory.getLogger(Jt808ChannelHandler.class); private static final String PROCESSING_ERROR_PREFIX = "processing failed: "; - private static final AttributeKey> DURABLE_ACK_CHAIN_ATTR = - AttributeKey.valueOf("jt808.durableAckChain"); private final Jt808MessageDecoder decoder; private final Dispatcher dispatcher; @@ -123,21 +123,29 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { // ====== 会话维护 & 同步应答匹配 ====== String phone = msg.header().phone(); IdentityResolution identity = resolveIdentity(msg); - byte[] durableAck = null; + if (msg.body() instanceof Jt808Body.Register reg) { + identity = registerIdentity(msg, reg, identity); + } + byte[] registerAck = null; switch (msg.header().messageId()) { case Jt808MessageId.TERMINAL_REGISTER -> { DeviceSession session = upsertSession(ctx, msg, identity.identity()); channelRegistry.bind(phone, ctx.channel()); - durableAck = registerAckFrame(phone, msg.header().serialNo(), session.token()); + var command = Jt808Commands.registerAck(msg.header().serialNo(), 0, session.token()); + registerAck = Jt808FrameEncoder.encode( + command.messageId(), + phone, + channelRegistry.nextSerial(phone), + command.body()); } case Jt808MessageId.TERMINAL_AUTH -> { upsertSession(ctx, msg, identity.identity()); channelRegistry.bind(phone, ctx.channel()); - durableAck = terminalAckFrame(phone, msg.header().serialNo(), msg.header().messageId(), 0); + ackTerminal(ctx, phone, msg.header().serialNo(), msg.header().messageId(), 0); } case Jt808MessageId.TERMINAL_HEARTBEAT -> { channelRegistry.bind(phone, ctx.channel()); - durableAck = terminalAckFrame(phone, msg.header().serialNo(), msg.header().messageId(), 0); + ackTerminal(ctx, phone, msg.header().serialNo(), msg.header().messageId(), 0); } case Jt808MessageId.TERMINAL_GENERAL_RESPONSE -> { // body: ackSerial(2) + ackMsgId(2) + result(1) @@ -153,7 +161,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { } // ====== 派发到 Dispatcher ====== - Map meta = new HashMap<>(6); + Map meta = new HashMap<>(32); meta.put("vin", identity.identity().vin()); meta.put("phone", phone); meta.put("identityResolved", Boolean.toString(identity.identity().resolved())); @@ -163,8 +171,11 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { meta.put("identityErrorMessage", identity.errorMessage()); } meta.put("seq", Integer.toString(msg.header().serialNo())); + meta.put("messageId", "0x" + Jt808MessageId.hex(msg.header().messageId())); + meta.put("messageName", Jt808MessageId.nameOf(msg.header().messageId())); + meta.put("reportType", Jt808MessageId.reportTypeOf(msg.header().messageId())); + putBodyMetadata(meta, msg.body()); meta.put("peer", addr(ctx)); - addRegistrationMetadata(meta, msg.body()); RawFrame rf = new RawFrame( ProtocolId.JT808, msg.header().messageId(), @@ -173,63 +184,19 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { frame, meta, Instant.now()); - if (durableAck == null) { - dispatcher.dispatch(rf); + if (registerAck != null) { + byte[] ack = registerAck; + dispatcher.dispatchAndAwait(rf) + .thenRun(() -> ctx.writeAndFlush(Unpooled.wrappedBuffer(ack))) + .exceptionally(ex -> { + log.warn("[jt808] register ack withheld because dispatch durability failed peer={} phone={} seq={} error={}", + addr(ctx), phone, msg.header().serialNo(), + ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage()); + return null; + }); return; } - enqueueDurableAck(ctx, phone, msg.header().messageId(), dispatcher.dispatchAndAwait(rf), durableAck); - } - - private void enqueueDurableAck(ChannelHandlerContext ctx, - String phone, - int messageId, - CompletableFuture dispatchFuture, - byte[] ackFrame) { - CompletableFuture previous = ctx.channel().attr(DURABLE_ACK_CHAIN_ATTR).get(); - if (previous == null) { - previous = CompletableFuture.completedFuture(null); - } - CompletableFuture next = previous - .handle((ignored, previousFailure) -> null) - .thenCompose(ignored -> dispatchFuture.handle((ok, failure) -> failure)) - .thenCompose(failure -> runOnEventLoop(ctx, - () -> handleDurableAckResult(ctx, phone, messageId, failure, ackFrame))); - ctx.channel().attr(DURABLE_ACK_CHAIN_ATTR).set(next); - } - - private CompletableFuture runOnEventLoop(ChannelHandlerContext ctx, Runnable task) { - CompletableFuture done = new CompletableFuture<>(); - Runnable wrapped = () -> { - try { - task.run(); - done.complete(null); - } catch (Throwable t) { - done.completeExceptionally(t); - } - }; - if (ctx.executor().inEventLoop()) { - wrapped.run(); - } else { - ctx.executor().execute(wrapped); - } - return done; - } - - private void handleDurableAckResult(ChannelHandlerContext ctx, - String phone, - int messageId, - Throwable failure, - byte[] ackFrame) { - if (!ctx.channel().isActive()) { - return; - } - if (failure != null) { - log.warn("[jt808] dispatch failed before ack peer={} phone={} msgId=0x{}", - addr(ctx), phone, Integer.toHexString(messageId), failure); - ctx.close(); - return; - } - ctx.writeAndFlush(Unpooled.wrappedBuffer(ackFrame)); + dispatcher.dispatch(rf); } private IdentityResolution resolveIdentity(Jt808Message msg) { @@ -267,6 +234,28 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { return new VehicleIdentity("unknown", false, identity.source()); } + private IdentityResolution registerIdentity(Jt808Message msg, + Jt808Body.Register reg, + IdentityResolution current) { + VehicleIdentity currentIdentity = current.identity(); + boolean alreadyResolved = currentIdentity != null && currentIdentity.resolved(); + String vin = alreadyResolved ? currentIdentity.vin() : generatedInternalVin(msg.header().phone(), reg); + identityRegistry.bind(new VehicleIdentityBinding( + ProtocolId.JT808, + vin, + msg.header().phone(), + reg.deviceId(), + reg.plate(), + Map.of( + "province", Integer.toString(reg.province()), + "city", Integer.toString(reg.city()), + "maker", reg.maker(), + "deviceType", reg.deviceType(), + "plateColor", Integer.toString(reg.plateColor())))); + VehicleIdentitySource source = alreadyResolved ? currentIdentity.source() : VehicleIdentitySource.REGISTERED; + return new IdentityResolution(new VehicleIdentity(vin, true, source), current.errorMessage()); + } + private void dispatchFrameError(ChannelHandlerContext ctx, Jt808MalformedFrame malformed) { String peer = malformed.peer() == null || malformed.peer().isBlank() ? addr(ctx) : malformed.peer(); dispatchMalformed(peer, malformed.rawBytes(), malformed.reason(), true); @@ -283,7 +272,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { new Jt808Header(0, frame == null ? 0 : frame.length, 0, false, Jt808Header.ProtocolVersion.V2013, "unknown", 0, 0, 0), new Jt808Body.Malformed(frame == null ? new byte[0] : frame, peer, message)); - Map meta = new HashMap<>(4); + Map meta = new HashMap<>(8); meta.put("vin", "unknown"); meta.put("peer", peer); meta.put("identityResolved", "false"); @@ -295,6 +284,7 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { meta.put("parseError", "true"); meta.put("parseErrorMessage", message); } + putBodyMetadata(meta, malformed.body()); dispatcher.dispatch(new RawFrame( ProtocolId.JT808, 0, @@ -312,15 +302,19 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { msg.header().version(), msg.header().phone(), msg.header().serialNo(), 0, 0), new Jt808Body.Malformed(frame == null ? new byte[0] : frame, addr(ctx), PROCESSING_ERROR_PREFIX + message)); - Map meta = new HashMap<>(8); + Map meta = new HashMap<>(16); meta.put("vin", "unknown"); meta.put("phone", msg.header().phone()); meta.put("peer", addr(ctx)); meta.put("seq", Integer.toString(msg.header().serialNo())); + meta.put("messageId", "0x" + Jt808MessageId.hex(msg.header().messageId())); + meta.put("messageName", Jt808MessageId.nameOf(msg.header().messageId())); + meta.put("reportType", Jt808MessageId.reportTypeOf(msg.header().messageId())); meta.put("identityResolved", "false"); meta.put("identitySource", VehicleIdentitySource.UNKNOWN.name()); meta.put("processingError", "true"); meta.put("processingErrorMessage", message); + putBodyMetadata(meta, failed.body()); dispatcher.dispatch(new RawFrame( ProtocolId.JT808, 0, @@ -343,25 +337,6 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { Instant.now(), Instant.now(), Map.of("protocolVersion", msg.header().version().name()))); session = session.withVin(identity.vin()); - if (msg.body() instanceof Jt808Body.Register reg) { - identityRegistry.register(new VehicleRegistrationBinding( - ProtocolId.JT808, - session.vin(), - phone, - reg.deviceId(), - reg.plate(), - reg.province(), - reg.city(), - reg.maker(), - reg.deviceType(), - reg.plateColor())); - if (identity.resolved() && session.vin() != null - && !session.vin().isBlank() - && !"unknown".equalsIgnoreCase(session.vin())) { - identityRegistry.bind(new VehicleIdentityBinding( - ProtocolId.JT808, session.vin(), phone, reg.deviceId(), reg.plate())); - } - } sessions.put(session); return session; } @@ -380,37 +355,175 @@ public class Jt808ChannelHandler extends SimpleChannelInboundHandler { return body instanceof Jt808Body.Register reg ? reg.plate() : ""; } - private static void addRegistrationMetadata(Map meta, Jt808Body body) { - if (!(body instanceof Jt808Body.Register reg)) { - return; + private static void putBodyMetadata(Map meta, Jt808Body body) { + switch (body) { + case Jt808Body.Register reg -> { + put(meta, "body.province", reg.province()); + put(meta, "body.city", reg.city()); + put(meta, "body.maker", reg.maker()); + put(meta, "body.deviceType", reg.deviceType()); + put(meta, "body.deviceId", reg.deviceId()); + put(meta, "body.plateColor", reg.plateColor()); + put(meta, "body.plate", reg.plate()); + } + case Jt808Body.Auth auth -> { + put(meta, "body.token", auth.token()); + put(meta, "body.imei", auth.imei()); + put(meta, "body.softwareVersion", auth.softwareVersion()); + } + case Jt808Body.Location location -> putLocationMetadata(meta, "body.", location); + case Jt808Body.LocationBatch batch -> { + put(meta, "body.batchType", batch.batchType()); + put(meta, "body.locationCount", batch.locations() == null ? 0 : batch.locations().size()); + } + case Jt808Body.ParamsReport report -> { + put(meta, "body.responseSerialNo", report.responseSerialNo()); + put(meta, "body.parameterCount", report.parameters() == null ? 0 : report.parameters().size()); + if (report.parameters() != null) { + report.parameters().forEach((id, bytes) -> + put(meta, "body.param.0x" + String.format("%08X", id), toHex(bytes))); + } + } + case Jt808Body.TerminalAttrs attrs -> { + put(meta, "body.terminalType", attrs.terminalType()); + put(meta, "body.maker", attrs.maker()); + put(meta, "body.terminalModel", attrs.terminalModel()); + put(meta, "body.terminalId", attrs.terminalId()); + put(meta, "body.iccid", attrs.iccid()); + put(meta, "body.hardwareVersion", attrs.hardwareVersion()); + put(meta, "body.firmwareVersion", attrs.firmwareVersion()); + put(meta, "body.gnssProperty", attrs.gnssProperty()); + put(meta, "body.commProperty", attrs.commProperty()); + } + case Jt808Body.MediaEvent event -> { + put(meta, "body.mediaId", event.mediaId()); + put(meta, "body.mediaType", event.mediaType()); + put(meta, "body.mediaFormat", event.mediaFormat()); + put(meta, "body.eventCode", event.eventCode()); + put(meta, "body.channelId", event.channelId()); + } + case Jt808Body.MediaUpload upload -> { + put(meta, "body.mediaId", upload.mediaId()); + put(meta, "body.mediaType", upload.mediaType()); + put(meta, "body.mediaFormat", upload.mediaFormat()); + put(meta, "body.eventCode", upload.eventCode()); + put(meta, "body.channelId", upload.channelId()); + put(meta, "body.dataSizeBytes", upload.data() == null ? 0 : upload.data().length); + if (upload.location() != null) { + putLocationMetadata(meta, "body.location.", upload.location()); + } + } + case Jt808Body.MediaAttributes attrs -> { + put(meta, "body.inputAudioEncoding", attrs.inputAudioEncoding()); + put(meta, "body.inputAudioChannels", attrs.inputAudioChannels()); + put(meta, "body.inputAudioSampleRate", attrs.inputAudioSampleRate()); + put(meta, "body.inputAudioBitDepth", attrs.inputAudioBitDepth()); + put(meta, "body.audioFrameLength", attrs.audioFrameLength()); + put(meta, "body.audioOutputSupported", attrs.audioOutputSupported()); + put(meta, "body.videoEncoding", attrs.videoEncoding()); + put(meta, "body.maxAudioChannels", attrs.maxAudioChannels()); + put(meta, "body.maxVideoChannels", attrs.maxVideoChannels()); + put(meta, "body.rawSizeBytes", attrs.raw() == null ? 0 : attrs.raw().length); + } + case Jt808Body.FileUploadComplete complete -> { + put(meta, "body.responseSerialNo", complete.responseSerialNo()); + put(meta, "body.result", complete.result()); + put(meta, "body.rawSizeBytes", complete.raw() == null ? 0 : complete.raw().length); + } + case Jt808Body.FileList files -> { + put(meta, "body.responseSerialNo", files.responseSerialNo()); + put(meta, "body.fileCount", files.fileCount()); + put(meta, "body.rawSizeBytes", files.raw() == null ? 0 : files.raw().length); + } + case Jt808Body.PassengerVolume volume -> { + put(meta, "body.channelId", volume.channelId()); + put(meta, "body.startTime", volume.startTime()); + put(meta, "body.endTime", volume.endTime()); + put(meta, "body.passengerGetOn", volume.passengerGetOn()); + put(meta, "body.passengerGetOff", volume.passengerGetOff()); + put(meta, "body.rawSizeBytes", volume.raw() == null ? 0 : volume.raw().length); + } + case Jt808Body.Passthrough passthrough -> { + put(meta, "body.passthroughType", "0x" + String.format("%02X", passthrough.passthroughType())); + put(meta, "body.dataSizeBytes", passthrough.data() == null ? 0 : passthrough.data().length); + put(meta, "body.dataHex", toHex(passthrough.data())); + } + case Jt808Body.Raw raw -> { + put(meta, "body.rawMessageId", "0x" + Jt808MessageId.hex(raw.messageId())); + put(meta, "body.rawSizeBytes", raw.bytes() == null ? 0 : raw.bytes().length); + put(meta, "body.rawHex", toHex(raw.bytes())); + } + case Jt808Body.Malformed malformed -> { + put(meta, "body.peer", malformed.peer()); + put(meta, "body.errorMessage", malformed.errorMessage()); + put(meta, "body.rawSizeBytes", malformed.bytes() == null ? 0 : malformed.bytes().length); + } + case Jt808Body.Heartbeat ignored -> { + } + case Jt808Body.Unregister ignored -> { + } } - meta.put("jt808.register.province", Integer.toString(reg.province())); - meta.put("jt808.register.city", Integer.toString(reg.city())); - meta.put("jt808.register.maker", nullToEmpty(reg.maker())); - meta.put("jt808.register.deviceType", nullToEmpty(reg.deviceType())); - meta.put("jt808.register.deviceId", nullToEmpty(reg.deviceId())); - meta.put("jt808.register.plateColor", Integer.toString(reg.plateColor())); - meta.put("jt808.register.plate", nullToEmpty(reg.plate())); } - private static String nullToEmpty(String value) { - return value == null ? "" : value; + private static void putLocationMetadata(Map meta, String prefix, Jt808Body.Location location) { + put(meta, prefix + "alarmFlag", location.alarmFlag()); + put(meta, prefix + "statusFlag", location.statusFlag()); + meta.putAll(Jt808LocationFlagInfo.decodeStatus(location.statusFlag(), prefix + "status.")); + meta.putAll(Jt808LocationFlagInfo.decodeAlarm(location.alarmFlag(), prefix + "alarm.")); + put(meta, prefix + "longitude", location.longitude()); + put(meta, prefix + "latitude", location.latitude()); + put(meta, prefix + "altitudeM", location.altitudeM()); + put(meta, prefix + "speedKmh", location.speedKmh()); + put(meta, prefix + "directionDeg", location.directionDeg()); + put(meta, prefix + "deviceTime", location.deviceTime()); + if (location.extensionItems() != null) { + location.extensionItems().forEach((id, bytes) -> + put(meta, prefix + "extra.0x" + String.format("%02X", id), toHex(bytes))); + meta.putAll(Jt808LocationAdditionalInfo.decode(location.extensionItems(), prefix + "extra.")); + } } - private byte[] registerAckFrame(String phone, int ackSerial, String token) { - var cmd = Jt808Commands.registerAck(ackSerial, 0, token); - return Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body()); + private static void put(Map meta, String key, Object value) { + if (value != null) { + meta.put(key, String.valueOf(value)); + } } - private byte[] terminalAckFrame(String phone, int ackSerial, int ackMsgId, int result) { + private static String toHex(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return ""; + } + StringBuilder out = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + out.append(String.format("%02X", b & 0xFF)); + } + return out.toString(); + } + + private void ackTerminal(ChannelHandlerContext ctx, String phone, int ackSerial, int ackMsgId, int result) { var cmd = Jt808Commands.platformAck(ackSerial, ackMsgId, result); - return Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body()); + byte[] frame = Jt808FrameEncoder.encode(cmd.messageId(), phone, channelRegistry.nextSerial(phone), cmd.body()); + ctx.writeAndFlush(Unpooled.wrappedBuffer(frame)); } private static String generateToken(String phone) { return UUID.nameUUIDFromBytes(("jt808:" + phone).getBytes()).toString().replace("-", "").substring(0, 16); } + private static String generatedInternalVin(String phone, Jt808Body.Register reg) { + String material = String.join("|", phone, reg.deviceId(), reg.plate(), reg.maker(), reg.deviceType()); + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(material.getBytes(StandardCharsets.UTF_8)); + StringBuilder suffix = new StringBuilder(12); + for (int i = 0; i < digest.length && suffix.length() < 12; i++) { + suffix.append(String.format("%02X", digest[i] & 0xFF)); + } + return "JT808" + suffix.substring(0, 12); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 not available", e); + } + } + @Override public void channelInactive(ChannelHandlerContext ctx) { channelRegistry.unbind(ctx.channel()) diff --git a/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapper.java b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapper.java index 93578108..931209ea 100644 --- a/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapper.java +++ b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapper.java @@ -10,6 +10,8 @@ import com.lingniu.ingest.identity.VehicleIdentityResolver; import com.lingniu.ingest.identity.VehicleIdentitySource; import com.lingniu.ingest.protocol.jt808.model.Jt808Body; import com.lingniu.ingest.protocol.jt808.model.Jt808Header; +import com.lingniu.ingest.protocol.jt808.model.Jt808LocationAdditionalInfo; +import com.lingniu.ingest.protocol.jt808.model.Jt808LocationFlagInfo; import com.lingniu.ingest.protocol.jt808.model.Jt808Message; import java.time.Instant; @@ -276,12 +278,14 @@ public final class Jt808EventMapper implements EventMapper { } private static Map locationMeta(Map meta, Jt808Body.Location loc) { - if (loc.extensionItems().isEmpty()) { - return meta; - } java.util.HashMap copy = new java.util.HashMap<>(meta); - loc.extensionItems().forEach((id, bytes) -> - copy.put("jt808.extra.0x" + String.format("%02X", id), hex(bytes))); + copy.putAll(Jt808LocationFlagInfo.decodeStatus(loc.statusFlag(), "jt808.status.")); + copy.putAll(Jt808LocationFlagInfo.decodeAlarm(loc.alarmFlag(), "jt808.alarm.")); + if (!loc.extensionItems().isEmpty()) { + loc.extensionItems().forEach((id, bytes) -> + copy.put("jt808.extra.0x" + String.format("%02X", id), hex(bytes))); + copy.putAll(Jt808LocationAdditionalInfo.decode(loc.extensionItems())); + } return Map.copyOf(copy); } @@ -304,15 +308,7 @@ public final class Jt808EventMapper implements EventMapper { } private static Double totalMileageKm(Jt808Body.Location loc) { - byte[] bytes = loc.extensionItems().get(0x01); - if (bytes == null || bytes.length != 4) { - return null; - } - long raw = ((long) bytes[0] & 0xFF) << 24 - | ((long) bytes[1] & 0xFF) << 16 - | ((long) bytes[2] & 0xFF) << 8 - | ((long) bytes[3] & 0xFF); - return raw * 0.1; + return Jt808LocationAdditionalInfo.mileageKm(loc.extensionItems()); } private static String hex(byte[] bytes) { diff --git a/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808LocationAdditionalInfo.java b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808LocationAdditionalInfo.java new file mode 100644 index 00000000..3e5edcb7 --- /dev/null +++ b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808LocationAdditionalInfo.java @@ -0,0 +1,247 @@ +package com.lingniu.ingest.protocol.jt808.model; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public final class Jt808LocationAdditionalInfo { + + public static final String DEFAULT_PREFIX = "jt808.extra."; + + private static final String[] VEHICLE_SIGNAL_BITS = { + "low_beam", + "high_beam", + "right_turn", + "left_turn", + "brake", + "reverse", + "fog_lamp", + "position_lamp", + "horn", + "air_conditioner", + "neutral", + "retarder", + "abs", + "heater", + "clutch" + }; + + private static final String[] IO_STATUS_BITS = { + "deep_sleep", + "sleep" + }; + + private Jt808LocationAdditionalInfo() { + } + + public static Map decode(Map extensionItems) { + return decode(extensionItems, DEFAULT_PREFIX); + } + + public static Map decode(Map extensionItems, String prefix) { + if (extensionItems == null || extensionItems.isEmpty()) { + return Map.of(); + } + String p = prefix == null ? "" : prefix; + Map out = new LinkedHashMap<>(); + extensionItems.forEach((id, bytes) -> decodeItem(out, p, id == null ? -1 : id, bytes)); + return Map.copyOf(out); + } + + public static Double mileageKm(Map extensionItems) { + if (extensionItems == null) { + return null; + } + byte[] bytes = extensionItems.get(0x01); + if (bytes == null || bytes.length != 4) { + return null; + } + return u32(bytes, 0) * 0.1; + } + + private static void decodeItem(Map out, String prefix, int id, byte[] bytes) { + if (bytes == null) { + return; + } + switch (id) { + case 0x01 -> put(out, prefix + "mileage_km", scaled(bytes, 4, 1)); + case 0x02 -> put(out, prefix + "fuel_l", scaled(bytes, 2, 1)); + case 0x03 -> put(out, prefix + "recorder_speed_kmh", scaled(bytes, 2, 1)); + case 0x04 -> put(out, prefix + "manual_alarm_event_id", unsigned(bytes, 2)); + case 0x05 -> decodeTirePressure(out, prefix, bytes); + case 0x06 -> put(out, prefix + "compartment_temp_c", signedHighBitWord(bytes)); + case 0x11 -> decodeOverspeed(out, prefix, bytes); + case 0x12 -> decodeAreaRouteAlarm(out, prefix, bytes); + case 0x13 -> decodeRouteTime(out, prefix, bytes); + case 0x25 -> decodeVehicleSignal(out, prefix, bytes); + case 0x2A -> decodeIoStatus(out, prefix, bytes); + case 0x2B -> decodeAnalog(out, prefix, bytes); + case 0x30 -> put(out, prefix + "network_signal_strength", unsigned(bytes, 1)); + case 0x31 -> put(out, prefix + "gnss_satellite_count", unsigned(bytes, 1)); + default -> { + } + } + } + + private static void decodeTirePressure(Map out, String prefix, byte[] bytes) { + if (bytes.length != 30) { + return; + } + List values = new ArrayList<>(); + List validIndexes = new ArrayList<>(); + List invalidIndexes = new ArrayList<>(); + for (int i = 0; i < bytes.length; i++) { + int value = u8(bytes, i); + if (value == 0xFF) { + invalidIndexes.add(i); + } else { + validIndexes.add(i); + values.add(Integer.toString(value)); + } + } + put(out, prefix + "tire_pressure_pa_values", String.join(",", values)); + put(out, prefix + "tire_pressure_valid_indexes", ranges(validIndexes)); + put(out, prefix + "tire_pressure_invalid_indexes", ranges(invalidIndexes)); + } + + private static void decodeOverspeed(Map out, String prefix, byte[] bytes) { + if (bytes.length != 1 && bytes.length != 5) { + return; + } + put(out, prefix + "overspeed.position_type", Integer.toString(u8(bytes, 0))); + if (bytes.length == 5) { + put(out, prefix + "overspeed.area_route_id", Long.toString(u32(bytes, 1))); + } + } + + private static void decodeAreaRouteAlarm(Map out, String prefix, byte[] bytes) { + if (bytes.length != 6) { + return; + } + put(out, prefix + "area_route_alarm.position_type", Integer.toString(u8(bytes, 0))); + put(out, prefix + "area_route_alarm.area_route_id", Long.toString(u32(bytes, 1))); + put(out, prefix + "area_route_alarm.direction", Integer.toString(u8(bytes, 5))); + } + + private static void decodeRouteTime(Map out, String prefix, byte[] bytes) { + if (bytes.length != 7) { + return; + } + put(out, prefix + "route_time.segment_id", Long.toString(u32(bytes, 0))); + put(out, prefix + "route_time.duration_seconds", Integer.toString(u16(bytes, 4))); + put(out, prefix + "route_time.result", Integer.toString(u8(bytes, 6))); + } + + private static void decodeVehicleSignal(Map out, String prefix, byte[] bytes) { + if (bytes.length != 4) { + return; + } + long raw = u32(bytes, 0); + put(out, prefix + "vehicle_signal_raw", Long.toString(raw)); + for (int i = 0; i < VEHICLE_SIGNAL_BITS.length; i++) { + put(out, prefix + "vehicle_signal." + VEHICLE_SIGNAL_BITS[i], Boolean.toString(bit(raw, i))); + } + } + + private static void decodeIoStatus(Map out, String prefix, byte[] bytes) { + if (bytes.length != 2) { + return; + } + int raw = u16(bytes, 0); + put(out, prefix + "io_status_raw", Integer.toString(raw)); + for (int i = 0; i < IO_STATUS_BITS.length; i++) { + put(out, prefix + "io_status." + IO_STATUS_BITS[i], Boolean.toString(bit(raw, i))); + } + } + + private static void decodeAnalog(Map out, String prefix, byte[] bytes) { + if (bytes.length != 4) { + return; + } + long raw = u32(bytes, 0); + put(out, prefix + "analog_raw", Long.toString(raw)); + put(out, prefix + "analog_ad0", Integer.toString((int) (raw & 0xFFFF))); + put(out, prefix + "analog_ad1", Integer.toString((int) ((raw >>> 16) & 0xFFFF))); + } + + private static String scaled(byte[] bytes, int expectedLength, int scale) { + if (bytes.length != expectedLength) { + return null; + } + long raw = expectedLength == 4 ? u32(bytes, 0) : u16(bytes, 0); + return BigDecimal.valueOf(raw, scale).stripTrailingZeros().toPlainString(); + } + + private static String unsigned(byte[] bytes, int expectedLength) { + if (bytes.length != expectedLength) { + return null; + } + return switch (expectedLength) { + case 1 -> Integer.toString(u8(bytes, 0)); + case 2 -> Integer.toString(u16(bytes, 0)); + case 4 -> Long.toString(u32(bytes, 0)); + default -> null; + }; + } + + private static String signedHighBitWord(byte[] bytes) { + if (bytes.length != 2) { + return null; + } + int raw = u16(bytes, 0); + int value = (raw & 0x8000) == 0 ? raw : -(raw & 0x7FFF); + return Integer.toString(value); + } + + private static int u8(byte[] bytes, int offset) { + return bytes[offset] & 0xFF; + } + + private static int u16(byte[] bytes, int offset) { + return (u8(bytes, offset) << 8) | u8(bytes, offset + 1); + } + + private static long u32(byte[] bytes, int offset) { + return ((long) u8(bytes, offset) << 24) + | ((long) u8(bytes, offset + 1) << 16) + | ((long) u8(bytes, offset + 2) << 8) + | u8(bytes, offset + 3); + } + + private static boolean bit(long raw, int bit) { + return ((raw >>> bit) & 1L) == 1L; + } + + private static void put(Map out, String key, String value) { + if (value != null) { + out.put(key, value); + } + } + + private static String ranges(List indexes) { + if (indexes.isEmpty()) { + return ""; + } + List ranges = new ArrayList<>(); + int start = indexes.getFirst(); + int previous = start; + for (int i = 1; i < indexes.size(); i++) { + int current = indexes.get(i); + if (current == previous + 1) { + previous = current; + continue; + } + ranges.add(formatRange(start, previous)); + start = current; + previous = current; + } + ranges.add(formatRange(start, previous)); + return String.join(",", ranges); + } + + private static String formatRange(int start, int end) { + return start == end ? Integer.toString(start) : start + "-" + end; + } +} diff --git a/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808LocationFlagInfo.java b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808LocationFlagInfo.java new file mode 100644 index 00000000..ea964d78 --- /dev/null +++ b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808LocationFlagInfo.java @@ -0,0 +1,99 @@ +package com.lingniu.ingest.protocol.jt808.model; + +import java.util.LinkedHashMap; +import java.util.Map; + +public final class Jt808LocationFlagInfo { + + private static final String[] ALARM_BITS = { + "emergency", + "overspeed", + "fatigue_driving", + "warning", + "gnss_module_fault", + "gnss_antenna_disconnected", + "gnss_antenna_short_circuit", + "terminal_main_power_undervoltage", + "terminal_main_power_lost", + "display_fault", + "tts_fault", + "camera_fault", + "transport_ic_card_fault", + "overspeed_warning", + "fatigue_driving_warning", + "prohibited_driving", + null, + null, + "daily_driving_timeout", + "overtime_parking", + "area_in_out", + "route_in_out", + "route_time_abnormal", + "route_deviation", + "vss_fault", + "fuel_abnormal", + "stolen", + "illegal_ignition", + "illegal_displacement", + "collision_rollover", + "rollover_warning" + }; + + private Jt808LocationFlagInfo() { + } + + public static Map decodeAlarm(long alarmFlag, String prefix) { + Map out = new LinkedHashMap<>(); + String p = prefix == null ? "" : prefix; + for (int i = 0; i < ALARM_BITS.length; i++) { + String name = ALARM_BITS[i]; + if (name != null) { + out.put(p + name, Boolean.toString(bit(alarmFlag, i))); + } + } + return Map.copyOf(out); + } + + public static Map decodeStatus(long statusFlag, String prefix) { + Map out = new LinkedHashMap<>(); + String p = prefix == null ? "" : prefix; + out.put(p + "acc_on", Boolean.toString(bit(statusFlag, 0))); + out.put(p + "positioned", Boolean.toString(bit(statusFlag, 1))); + out.put(p + "south_latitude", Boolean.toString(bit(statusFlag, 2))); + out.put(p + "west_longitude", Boolean.toString(bit(statusFlag, 3))); + out.put(p + "out_of_service", Boolean.toString(bit(statusFlag, 4))); + out.put(p + "encrypted", Boolean.toString(bit(statusFlag, 5))); + out.put(p + "forward_collision_warning", Boolean.toString(bit(statusFlag, 6))); + out.put(p + "lane_departure_warning", Boolean.toString(bit(statusFlag, 7))); + out.put(p + "load_status", loadStatus(statusFlag)); + out.put(p + "fuel_circuit_cut", Boolean.toString(bit(statusFlag, 10))); + out.put(p + "electric_circuit_cut", Boolean.toString(bit(statusFlag, 11))); + out.put(p + "door_locked", Boolean.toString(bit(statusFlag, 12))); + out.put(p + "door1_open", Boolean.toString(bit(statusFlag, 13))); + out.put(p + "door2_open", Boolean.toString(bit(statusFlag, 14))); + out.put(p + "door3_open", Boolean.toString(bit(statusFlag, 15))); + out.put(p + "door4_open", Boolean.toString(bit(statusFlag, 16))); + out.put(p + "door5_open", Boolean.toString(bit(statusFlag, 17))); + out.put(p + "gps_used", Boolean.toString(bit(statusFlag, 18))); + out.put(p + "beidou_used", Boolean.toString(bit(statusFlag, 19))); + out.put(p + "glonass_used", Boolean.toString(bit(statusFlag, 20))); + out.put(p + "galileo_used", Boolean.toString(bit(statusFlag, 21))); + out.put(p + "moving", Boolean.toString(bit(statusFlag, 22))); + return Map.copyOf(out); + } + + private static String loadStatus(long statusFlag) { + int code = (int) ((statusFlag >>> 8) & 0x03); + return switch (code) { + case 0 -> "EMPTY"; + case 1 -> "HALF_LOADED"; + case 2 -> "RESERVED"; + case 3 -> "FULL"; + default -> "UNKNOWN"; + }; + } + + private static boolean bit(long raw, int bit) { + return ((raw >>> bit) & 1L) == 1L; + } +} diff --git a/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808MessageId.java b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808MessageId.java index edc9c3ef..358b020e 100644 --- a/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808MessageId.java +++ b/modules/protocols/protocol-jt808/src/main/java/com/lingniu/ingest/protocol/jt808/model/Jt808MessageId.java @@ -36,4 +36,56 @@ public final class Jt808MessageId { public static final int PLATFORM_TRACK_CONTROL = 0x8202; public static final int PLATFORM_ALARM_ACK = 0x8203; public static final int PLATFORM_DELETE_CIRCLE_AREA = 0x8601; + + public static String nameOf(int messageId) { + return switch (messageId) { + case TERMINAL_GENERAL_RESPONSE -> "TERMINAL_GENERAL_RESPONSE"; + case TERMINAL_HEARTBEAT -> "TERMINAL_HEARTBEAT"; + case TERMINAL_UNREGISTER -> "TERMINAL_UNREGISTER"; + case TERMINAL_REGISTER -> "TERMINAL_REGISTER"; + case TERMINAL_AUTH -> "TERMINAL_AUTH"; + case TERMINAL_PARAMS_REPORT -> "TERMINAL_PARAMS_REPORT"; + case TERMINAL_ATTRS_REPORT -> "TERMINAL_ATTRS_REPORT"; + case TERMINAL_LOCATION -> "TERMINAL_LOCATION"; + case TERMINAL_LOCATION_BATCH -> "TERMINAL_LOCATION_BATCH"; + case TERMINAL_EVENT_REPORT -> "TERMINAL_EVENT_REPORT"; + case TERMINAL_CAN_DATA_REPORT -> "TERMINAL_CAN_DATA_REPORT"; + case TERMINAL_MEDIA_EVENT -> "TERMINAL_MEDIA_EVENT"; + case TERMINAL_MEDIA_UPLOAD -> "TERMINAL_MEDIA_UPLOAD"; + case TERMINAL_PASSTHROUGH -> "TERMINAL_PASSTHROUGH"; + case PLATFORM_GENERAL_RESPONSE -> "PLATFORM_GENERAL_RESPONSE"; + case PLATFORM_REGISTER_ACK -> "PLATFORM_REGISTER_ACK"; + case PLATFORM_SET_PARAMS -> "PLATFORM_SET_PARAMS"; + case PLATFORM_QUERY_PARAMS -> "PLATFORM_QUERY_PARAMS"; + case PLATFORM_TERMINAL_CONTROL -> "PLATFORM_TERMINAL_CONTROL"; + case PLATFORM_QUERY_ATTRS -> "PLATFORM_QUERY_ATTRS"; + case PLATFORM_QUERY_LOCATION -> "PLATFORM_QUERY_LOCATION"; + case PLATFORM_TRACK_CONTROL -> "PLATFORM_TRACK_CONTROL"; + case PLATFORM_ALARM_ACK -> "PLATFORM_ALARM_ACK"; + case PLATFORM_DELETE_CIRCLE_AREA -> "PLATFORM_DELETE_CIRCLE_AREA"; + default -> "UNKNOWN_0x" + hex(messageId); + }; + } + + public static String reportTypeOf(int messageId) { + return switch (messageId) { + case TERMINAL_LOCATION -> "LOCATION"; + case TERMINAL_LOCATION_BATCH -> "LOCATION_BATCH"; + case TERMINAL_HEARTBEAT -> "HEARTBEAT"; + case TERMINAL_AUTH -> "AUTH"; + case TERMINAL_REGISTER -> "REGISTER"; + case TERMINAL_UNREGISTER -> "UNREGISTER"; + case TERMINAL_GENERAL_RESPONSE -> "TERMINAL_RESPONSE"; + case TERMINAL_PARAMS_REPORT -> "PARAMS_REPORT"; + case TERMINAL_ATTRS_REPORT -> "ATTRS_REPORT"; + case TERMINAL_MEDIA_EVENT -> "MEDIA_EVENT"; + case TERMINAL_MEDIA_UPLOAD -> "MEDIA_UPLOAD"; + case TERMINAL_PASSTHROUGH -> "PASSTHROUGH"; + default -> "RAW"; + }; + } + + public static String hex(int messageId) { + return String.format("%04X", messageId & 0xFFFF); + } } diff --git a/modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandlerTest.java b/modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandlerTest.java index d089c6fc..b1471dd9 100644 --- a/modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandlerTest.java +++ b/modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandlerTest.java @@ -41,6 +41,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; class Jt808ChannelHandlerTest { @@ -86,7 +87,7 @@ class Jt808ChannelHandlerTest { } @Test - void unresolvedRegisterDoesNotBindUnknownVinToExternalIdentifiers() { + void unresolvedRegisterBindsGeneratedInternalVinToExternalIdentifiers() { InMemorySessionStore sessions = new InMemorySessionStore(); InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService(); DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(new ImmediateKafkaSink())); @@ -115,9 +116,9 @@ class Jt808ChannelHandlerTest { assertThat(identity.resolve(new VehicleIdentityLookup( ProtocolId.JT808, "", "123456789012", "", ""))) .satisfies(resolved -> { - assertThat(resolved.vin()).isEqualTo("123456789012"); - assertThat(resolved.resolved()).isFalse(); - assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.FALLBACK_PHONE); + assertThat(resolved.vin()).startsWith("JT808"); + assertThat(resolved.resolved()).isTrue(); + assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE); }); batchExecutor.close(); @@ -269,14 +270,143 @@ class Jt808ChannelHandlerTest { assertThat(archive.metadata()) .containsEntry("vin", "LNVIN000000000808") .containsEntry("phone", "123456789012") + .containsEntry("messageId", "0x0200") + .containsEntry("messageName", "TERMINAL_LOCATION") + .containsEntry("reportType", "LOCATION") + .containsEntry("body.alarmFlag", "0") + .containsEntry("body.statusFlag", "196608") + .containsEntry("body.status.acc_on", "false") + .containsEntry("body.status.load_status", "EMPTY") + .containsEntry("body.status.door4_open", "true") + .containsEntry("body.status.door5_open", "true") + .containsEntry("body.alarm.overspeed", "false") + .containsEntry("body.longitude", "116.397128") + .containsEntry("body.latitude", "39.916527") + .containsEntry("body.altitudeM", "50") + .containsEntry("body.directionDeg", "90") + .containsEntry("body.deviceTime", "2024-01-01T19:04:05Z") + .containsEntry("body.extra.0x01", "00003039") + .containsEntry("body.extra.mileage_km", "1234.5") + .containsEntry("body.extra.0x30", "58") + .containsEntry("body.extra.network_signal_strength", "88") .containsEntry("identityResolved", "true") .containsEntry("identitySource", "BOUND_PHONE"); + assertThat(Double.parseDouble(archive.metadata().get("body.speedKmh"))).isCloseTo(52.3, within(0.001)); }); batchExecutor.close(); eventBus.close(); } + @Test + void registerArchiveIncludesParsedBodyFieldsForRawQuery() throws Exception { + RecordingSink sink = new RecordingSink(1); + DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink)); + AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish); + Dispatcher dispatcher = new Dispatcher( + new HandlerRegistry(), + new InterceptorChain(List.of()), + new HandlerInvoker(), + eventBus, + batchExecutor); + InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService(); + identity.bind(new VehicleIdentityBinding( + ProtocolId.JT808, "LNVIN000000000808", "123456789012", "DEV808", "B80808")); + Jt808ChannelHandler handler = new Jt808ChannelHandler( + new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser()))), + dispatcher, + new InMemorySessionStore(), + identity, + new Jt808ChannelRegistry(), + new Jt808PendingRequests()); + EmbeddedChannel channel = new EmbeddedChannel(handler); + byte[] frame = buildFrame( + Jt808MessageId.TERMINAL_REGISTER, + "123456789012", + 3, + buildRegisterBody("DEV808", "B80808")); + + channel.writeInbound(frame); + + assertThat(sink.await()).isTrue(); + assertThat(sink.events).singleElement().satisfies(event -> { + assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class); + VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event; + assertThat(archive.metadata()) + .containsEntry("messageId", "0x0100") + .containsEntry("messageName", "TERMINAL_REGISTER") + .containsEntry("reportType", "REGISTER") + .containsEntry("body.province", "44") + .containsEntry("body.city", "4401") + .containsEntry("body.maker", "MAKER") + .containsEntry("body.deviceType", "TYPE-A") + .containsEntry("body.deviceId", "DEV808") + .containsEntry("body.plateColor", "1") + .containsEntry("body.plate", "B80808"); + }); + + batchExecutor.close(); + eventBus.close(); + } + + @Test + void unboundRegisterCreatesVinBindingForRegisterAndFollowingRawFrames() throws Exception { + RecordingSink sink = new RecordingSink(2); + DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink)); + AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish); + Dispatcher dispatcher = new Dispatcher( + new HandlerRegistry(), + new InterceptorChain(List.of()), + new HandlerInvoker(), + eventBus, + batchExecutor); + InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService(); + Jt808ChannelHandler handler = new Jt808ChannelHandler( + new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser(), new LocationBodyParser()))), + dispatcher, + new InMemorySessionStore(), + identity, + new Jt808ChannelRegistry(), + new Jt808PendingRequests()); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + channel.writeInbound(buildFrame( + Jt808MessageId.TERMINAL_REGISTER, + "13079963289", + 3, + buildRegisterBody("9963289", "HUA00113F"))); + channel.writeInbound(buildFrame( + Jt808MessageId.TERMINAL_LOCATION, + "13079963289", + 4, + buildLocationBody())); + + assertThat(sink.await()).isTrue(); + VehicleEvent.RawArchive register = (VehicleEvent.RawArchive) sink.events.get(0); + VehicleEvent.RawArchive location = (VehicleEvent.RawArchive) sink.events.get(1); + assertThat(register.metadata()) + .containsEntry("reportType", "REGISTER") + .containsEntry("body.deviceId", "9963289") + .containsEntry("body.plate", "HUA00113F") + .containsEntry("identityResolved", "true") + .containsEntry("identitySource", "REGISTERED"); + assertThat(register.vin()) + .startsWith("JT808") + .hasSize(17); + assertThat(location.metadata()) + .containsEntry("reportType", "LOCATION") + .containsEntry("vin", register.vin()) + .containsEntry("identityResolved", "true") + .containsEntry("identitySource", "BOUND_PHONE"); + assertThat(location.vin()).isEqualTo(register.vin()); + assertThat(identity.resolve(new com.lingniu.ingest.identity.VehicleIdentityLookup( + ProtocolId.JT808, "", "13079963289", "", "")).vin()) + .isEqualTo(register.vin()); + + batchExecutor.close(); + eventBus.close(); + } + @Test void unboundLocationArchiveUsesUnknownVinInsteadOfPhoneFallback() throws Exception { RecordingSink sink = new RecordingSink(1); @@ -309,6 +439,9 @@ class Jt808ChannelHandlerTest { assertThat(archive.metadata()) .containsEntry("vin", "unknown") .containsEntry("phone", "123456789012") + .containsEntry("messageId", "0x0200") + .containsEntry("messageName", "TERMINAL_LOCATION") + .containsEntry("reportType", "LOCATION") .containsEntry("identityResolved", "false") .containsEntry("identitySource", "FALLBACK_PHONE"); }); @@ -351,13 +484,13 @@ class Jt808ChannelHandlerTest { VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event; assertThat(archive.command()).isEqualTo(Jt808MessageId.TERMINAL_REGISTER); assertThat(archive.metadata()) - .containsEntry("jt808.register.province", "44") - .containsEntry("jt808.register.city", "4401") - .containsEntry("jt808.register.maker", "MAKER") - .containsEntry("jt808.register.deviceType", "TYPE-A") - .containsEntry("jt808.register.deviceId", "DEV808") - .containsEntry("jt808.register.plateColor", "1") - .containsEntry("jt808.register.plate", "B80808"); + .containsEntry("body.province", "44") + .containsEntry("body.city", "4401") + .containsEntry("body.maker", "MAKER") + .containsEntry("body.deviceType", "TYPE-A") + .containsEntry("body.deviceId", "DEV808") + .containsEntry("body.plateColor", "1") + .containsEntry("body.plate", "B80808"); }); batchExecutor.close(); @@ -581,6 +714,12 @@ class Jt808ChannelHandlerTest { writeU16(os, 90); byte[] bcd = BcdCodec.encode("240102030405"); os.write(bcd, 0, 6); + os.write(0x01); + os.write(4); + writeU32(os, 12345); + os.write(0x30); + os.write(1); + os.write(0x58); return os.toByteArray(); } diff --git a/modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapperTest.java b/modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapperTest.java index 87f362db..2b4d6fbe 100644 --- a/modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapperTest.java +++ b/modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapperTest.java @@ -38,6 +38,10 @@ class Jt808EventMapperTest { assertThat(events.get(0).metadata()) .containsEntry("vin", "unknown") .containsEntry("phone", "123456789012") + .containsEntry("jt808.status.acc_on", "false") + .containsEntry("jt808.status.positioned", "false") + .containsEntry("jt808.status.load_status", "EMPTY") + .containsEntry("jt808.alarm.overspeed", "false") .containsEntry("identityResolved", "false") .containsEntry("identitySource", "FALLBACK_PHONE"); } @@ -58,7 +62,66 @@ class Jt808EventMapperTest { .extracting(VehicleEvent::metadata) .satisfies(meta -> assertThat(meta) .containsEntry("jt808.extra.0x01", "00003039") - .containsEntry("jt808.extra.0x30", "58")); + .containsEntry("jt808.extra.0x30", "58") + .containsEntry("jt808.extra.mileage_km", "1234.5") + .containsEntry("jt808.extra.network_signal_strength", "88")); + } + + @Test + void locationAdditionalItemsAreDecodedAsTable27Fields() { + var header = new Jt808Header( + Jt808MessageId.TERMINAL_LOCATION, 64, 0, false, + Jt808Header.ProtocolVersion.V2013, "123456789012", 1, 0, 0); + var body = new Jt808Body.Location( + 0, 0, 116.397128, 39.916527, 50, 52.3, 90, + Instant.parse("2024-01-02T03:04:05Z"), + java.util.Map.ofEntries( + java.util.Map.entry(0x01, new byte[]{0x00, 0x00, 0x30, 0x39}), + java.util.Map.entry(0x02, new byte[]{0x00, 0x64}), + java.util.Map.entry(0x03, new byte[]{0x01, (byte) 0xF4}), + java.util.Map.entry(0x04, new byte[]{0x00, 0x02}), + java.util.Map.entry(0x05, tirePressureBytes()), + java.util.Map.entry(0x06, new byte[]{(byte) 0x80, 0x05}), + java.util.Map.entry(0x11, new byte[]{0x01, 0x00, 0x00, 0x00, 0x2A}), + java.util.Map.entry(0x12, new byte[]{0x04, 0x00, 0x00, 0x00, 0x2B, 0x01}), + java.util.Map.entry(0x13, new byte[]{0x00, 0x00, 0x00, 0x2C, 0x00, 0x78, 0x01}), + java.util.Map.entry(0x25, new byte[]{0x00, 0x00, 0x00, 0x05}), + java.util.Map.entry(0x2A, new byte[]{0x00, 0x03}), + java.util.Map.entry(0x2B, new byte[]{0x12, 0x34, 0x56, 0x78}), + java.util.Map.entry(0x30, new byte[]{0x1F}), + java.util.Map.entry(0x31, new byte[]{0x0C}))); + + VehicleEvent.Location event = (VehicleEvent.Location) mapper.toEvents(new Jt808Message(header, body)).getFirst(); + + assertThat(event.payload().totalMileageKm()).isEqualTo(1234.5); + assertThat(event.metadata()) + .containsEntry("jt808.extra.mileage_km", "1234.5") + .containsEntry("jt808.extra.fuel_l", "10") + .containsEntry("jt808.extra.recorder_speed_kmh", "50") + .containsEntry("jt808.extra.manual_alarm_event_id", "2") + .containsEntry("jt808.extra.tire_pressure_pa_values", "32,33") + .containsEntry("jt808.extra.tire_pressure_invalid_indexes", "2-29") + .containsEntry("jt808.extra.compartment_temp_c", "-5") + .containsEntry("jt808.extra.overspeed.position_type", "1") + .containsEntry("jt808.extra.overspeed.area_route_id", "42") + .containsEntry("jt808.extra.area_route_alarm.position_type", "4") + .containsEntry("jt808.extra.area_route_alarm.area_route_id", "43") + .containsEntry("jt808.extra.area_route_alarm.direction", "1") + .containsEntry("jt808.extra.route_time.segment_id", "44") + .containsEntry("jt808.extra.route_time.duration_seconds", "120") + .containsEntry("jt808.extra.route_time.result", "1") + .containsEntry("jt808.extra.vehicle_signal_raw", "5") + .containsEntry("jt808.extra.vehicle_signal.low_beam", "true") + .containsEntry("jt808.extra.vehicle_signal.high_beam", "false") + .containsEntry("jt808.extra.vehicle_signal.right_turn", "true") + .containsEntry("jt808.extra.io_status_raw", "3") + .containsEntry("jt808.extra.io_status.deep_sleep", "true") + .containsEntry("jt808.extra.io_status.sleep", "true") + .containsEntry("jt808.extra.analog_raw", "305419896") + .containsEntry("jt808.extra.analog_ad0", "22136") + .containsEntry("jt808.extra.analog_ad1", "4660") + .containsEntry("jt808.extra.network_signal_strength", "31") + .containsEntry("jt808.extra.gnss_satellite_count", "12"); } @Test @@ -249,4 +312,12 @@ class Jt808EventMapperTest { .containsEntry("parseError", "true"); }); } + + private static byte[] tirePressureBytes() { + byte[] bytes = new byte[30]; + java.util.Arrays.fill(bytes, (byte) 0xFF); + bytes[0] = 32; + bytes[1] = 33; + return bytes; + } } diff --git a/modules/services/event-history-service/pom.xml b/modules/services/event-history-service/pom.xml index 10389f5c..cce05440 100644 --- a/modules/services/event-history-service/pom.xml +++ b/modules/services/event-history-service/pom.xml @@ -36,6 +36,10 @@ org.apache.kafka kafka-clients + + com.taosdata.jdbc + taos-jdbcdriver + org.springframework.boot spring-boot-starter-web diff --git a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java index 3ae8342a..25bd56a1 100644 --- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java +++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryController.java @@ -12,6 +12,10 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import java.io.IOException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.util.List; import java.util.Map; @@ -27,6 +31,8 @@ import java.util.Map; @RequestMapping("/api/event-history") public class EventHistoryController { + private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai"); + private final EventFileStore store; public EventHistoryController(EventFileStore store) { @@ -40,8 +46,13 @@ public class EventHistoryController { @RequestParam String dateTo, @RequestParam(defaultValue = "ASC") EventFileQuery.Order order, @RequestParam(defaultValue = "100") int limit, - @RequestParam(required = false) String vin) throws IOException { - return queryRecords(protocol, dateFrom, dateTo, order, limit, vin).stream() + @RequestParam(required = false) String vin, + @RequestParam(required = false) String eventType, + @RequestParam(required = false) String cursorEventTime, + @RequestParam(required = false) String cursorIngestTime, + @RequestParam(required = false) String cursorEventId) throws IOException { + return queryRecords(protocol, dateFrom, dateTo, order, limit, vin, eventType, + cursorEventTime, cursorIngestTime, cursorEventId).stream() .map(RecordResponse::from) .toList(); } @@ -51,7 +62,11 @@ public class EventHistoryController { String dateTo, EventFileQuery.Order order, int limit, - String vin) throws IOException { + String vin, + String eventType, + String cursorEventTime, + String cursorIngestTime, + String cursorEventId) throws IOException { QueryTimeRange range = QueryTimeRange.parse(dateFrom, dateTo); return store.query(new EventFileQuery( protocol, @@ -62,7 +77,22 @@ public class EventHistoryController { order, limit, vin, - null)); + eventType, + instantParam(cursorEventTime), + instantParam(cursorIngestTime), + cursorEventId)); + } + + private static Instant instantParam(String raw) { + if (raw == null || raw.isBlank()) { + return null; + } + String value = raw.trim(); + try { + return OffsetDateTime.parse(value).toInstant(); + } catch (RuntimeException ignored) { + return LocalDateTime.parse(value).atZone(DEFAULT_ZONE).toInstant(); + } } public record RecordResponse( diff --git a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java index afaad22c..7d89ea89 100644 --- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java +++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestor.java @@ -1,8 +1,8 @@ package com.lingniu.ingest.eventhistory; import com.google.protobuf.InvalidProtocolBufferException; +import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; -import com.lingniu.ingest.api.consumer.EnvelopeIngestor; import com.lingniu.ingest.eventfilestore.EventFileRecord; import com.lingniu.ingest.eventfilestore.EventFileStore; import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; @@ -20,7 +20,7 @@ import java.util.List; *

当前 32960 本机运行配置没有启用 Kafka consumer；这个类保留给“其他服务把 envelope * 写回历史库”的解耦部署方式。失败时返回结构化结果，由 Kafka worker 决定是否进 DLQ。 */ -public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor { +public final class EventHistoryEnvelopeIngestor implements EnvelopeBatchIngestor { private final EventFileStore store; private final TelemetryEnvelopeRecordMapper mapper; @@ -33,7 +33,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor { public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper, TdengineHistoryWriter tdengineWriter) { - this(mapper, tdengineWriter, true); + this(mapper, tdengineWriter, false); } public EventHistoryEnvelopeIngestor(TelemetryEnvelopeRecordMapper mapper, @@ -45,7 +45,7 @@ public final class EventHistoryEnvelopeIngestor implements EnvelopeIngestor { public EventHistoryEnvelopeIngestor(EventFileStore store, TelemetryEnvelopeRecordMapper mapper, TdengineHistoryWriter tdengineWriter) { - this(store, mapper, tdengineWriter, true, true); + this(store, mapper, tdengineWriter, true, false); } public EventHistoryEnvelopeIngestor(EventFileStore store, diff --git a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryController.java b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryController.java index 15bbb10e..613b91ce 100644 --- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryController.java +++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryController.java @@ -1,7 +1,5 @@ package com.lingniu.ingest.eventhistory; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader; import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery; import com.lingniu.ingest.tdenginehistory.TdengineLocationRow; @@ -32,8 +30,6 @@ import java.util.List; @Tag(name = "jt-808-location-controller", description = "JT808 位置历史分页查询接口。") public final class Jt808LocationHistoryController { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final TdengineHistoryReader reader; public Jt808LocationHistoryController(TdengineHistoryReader reader) { @@ -127,8 +123,7 @@ public final class Jt808LocationHistoryController { long alarmFlag, long statusFlag, Double totalMileageKm, - String rawUri, - String metadataJson) { + String rawUri) { private static LocationResponse from(TdengineLocationRow row) { return new LocationResponse( @@ -147,23 +142,7 @@ public final class Jt808LocationHistoryController { row.alarmFlag(), row.statusFlag(), row.totalMileageKm(), - rawUri(row), - row.metadataJson()); - } - - private static String rawUri(TdengineLocationRow row) { - if (row.rawUri() != null && !row.rawUri().isBlank()) { - return row.rawUri(); - } - String metadataJson = row.metadataJson(); - if (metadataJson == null || metadataJson.isBlank()) { - return row.rawUri(); - } - try { - return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri()); - } catch (JsonProcessingException ignored) { - return row.rawUri(); - } + row.rawUri()); } } } diff --git a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/LocationHistoryController.java b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/LocationHistoryController.java index f39ac1a6..7ac4c90c 100644 --- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/LocationHistoryController.java +++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/LocationHistoryController.java @@ -1,7 +1,5 @@ package com.lingniu.ingest.eventhistory; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader; import com.lingniu.ingest.tdenginehistory.TdengineLocationQuery; import com.lingniu.ingest.tdenginehistory.TdengineLocationRow; @@ -32,8 +30,6 @@ import java.util.Locale; @Tag(name = "location-history-controller", description = "通用位置历史分页查询接口。") public final class LocationHistoryController { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final TdengineHistoryReader reader; public LocationHistoryController(TdengineHistoryReader reader) { @@ -161,7 +157,6 @@ public final class LocationHistoryController { long statusFlag, Double totalMileageKm, String rawUri, - String metadataJson, String protocol) { private static LocationResponse from(TdengineLocationRow row) { @@ -181,24 +176,8 @@ public final class LocationHistoryController { row.alarmFlag(), row.statusFlag(), row.totalMileageKm(), - rawUri(row), - row.metadataJson(), + row.rawUri(), row.protocol()); } - - private static String rawUri(TdengineLocationRow row) { - if (row.rawUri() != null && !row.rawUri().isBlank()) { - return row.rawUri(); - } - String metadataJson = row.metadataJson(); - if (metadataJson == null || metadataJson.isBlank()) { - return row.rawUri(); - } - try { - return OBJECT_MAPPER.readTree(metadataJson).path("rawArchiveUri").asText(row.rawUri()); - } catch (JsonProcessingException ignored) { - return row.rawUri(); - } - } } } diff --git a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java index 7f65e0c8..1bf68c50 100644 --- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java +++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/RawFrameHistoryController.java @@ -184,6 +184,7 @@ public final class RawFrameHistoryController { String parseError, String peer, String metadataJson, + String parsedJson, Map metadata, Map parsedFields, String protocol, @@ -209,8 +210,9 @@ public final class RawFrameHistoryController { row.parseError(), row.peer(), row.metadataJson(), + row.parsedJson(), metadata, - parsedFields(reader, row, metadata, includeParsedFields), + parsedFields(reader, row, metadata, parsed(row.parsedJson()), includeParsedFields), row.protocol(), row.vehicleKey(), row.vin(), @@ -231,18 +233,23 @@ public final class RawFrameHistoryController { private static Map parsedFields(TdengineHistoryReader reader, TdengineRawFrameRow row, Map metadata, + Map parsed, boolean includeParsedFields) throws IOException { Map out = new LinkedHashMap<>(); - for (Map.Entry entry : metadata.entrySet()) { - String key = entry.getKey(); - if (key == null || key.isBlank()) { - continue; - } - if (key.startsWith("raw.") || key.startsWith("jt808.")) { - out.put(key, entry.getValue()); - } else { - out.put("raw." + key, entry.getValue()); + if (parsed.isEmpty()) { + for (Map.Entry entry : metadata.entrySet()) { + String key = entry.getKey(); + if (key == null || key.isBlank()) { + continue; + } + if (key.startsWith("raw.") || key.startsWith("jt808.")) { + out.put(key, entry.getValue()); + } else { + out.put("raw." + key, entry.getValue()); + } } + } else { + out.putAll(parsed); } if (!includeParsedFields) { return Map.copyOf(out); @@ -262,6 +269,17 @@ public final class RawFrameHistoryController { return Map.copyOf(out); } + private static Map parsed(String parsedJson) { + if (parsedJson == null || parsedJson.isBlank()) { + return Map.of(); + } + try { + return OBJECT_MAPPER.readValue(parsedJson, MAP_TYPE); + } catch (Exception ignored) { + return Map.of("_raw", parsedJson); + } + } + private static void addLocation(Map out, TdengineLocationRow row) { out.put("location.longitude", row.longitude()); out.put("location.latitude", row.latitude()); diff --git a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java index 905d0842..d8f13f8d 100644 --- a/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java +++ b/modules/services/event-history-service/src/main/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapper.java @@ -150,6 +150,7 @@ public final class TelemetryEnvelopeRecordMapper { payload.put("infoType", metadata.get("infoType")); } payload.put("rawSizeBytes", rawArchive.getSizeBytes()); + putParsedJson(payload, rawArchive.getParsedJson()); payload.put("metadata", metadata); try { return OBJECT_MAPPER.writeValueAsString(payload); @@ -157,4 +158,16 @@ public final class TelemetryEnvelopeRecordMapper { throw new IllegalArgumentException("failed to serialize raw_archive", e); } } + + private static void putParsedJson(Map payload, String parsedJson) { + if (parsedJson == null || parsedJson.isBlank()) { + return; + } + try { + payload.put("parsed", OBJECT_MAPPER.readTree(parsedJson)); + } catch (JsonProcessingException ex) { + payload.put("parsed", parsedJson); + payload.put("parsedJsonError", ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage()); + } + } } diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryControllerTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryControllerTest.java index c46f990b..7e11b7ae 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryControllerTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryControllerTest.java @@ -26,7 +26,11 @@ class EventHistoryControllerTest { "2026-06-22", EventFileQuery.Order.DESC, 50, - "VIN001"); + "VIN001", + "RAW_ARCHIVE", + "2026-06-22T08:00:00Z", + "2026-06-22T08:00:01Z", + "event-0"); assertThat(records).extracting(EventHistoryController.RecordResponse::eventId).containsExactly("event-1"); assertThat(records.getFirst().eventTime()).isEqualTo("2026-06-22T08:00:00Z"); @@ -37,6 +41,10 @@ class EventHistoryControllerTest { assertThat(store.lastQuery.order()).isEqualTo(EventFileQuery.Order.DESC); assertThat(store.lastQuery.limit()).isEqualTo(50); assertThat(store.lastQuery.vin()).isEqualTo("VIN001"); + assertThat(store.lastQuery.eventType()).isEqualTo("RAW_ARCHIVE"); + assertThat(store.lastQuery.cursorEventTime()).isEqualTo(Instant.parse("2026-06-22T08:00:00Z")); + assertThat(store.lastQuery.cursorIngestTime()).isEqualTo(Instant.parse("2026-06-22T08:00:01Z")); + assertThat(store.lastQuery.cursorEventId()).isEqualTo("event-0"); } private static EventFileRecord record(String eventId) { diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestorTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestorTest.java index 13acf47b..a04016cb 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestorTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/EventHistoryEnvelopeIngestorTest.java @@ -126,7 +126,7 @@ class EventHistoryEnvelopeIngestorTest { } @Test - void tryIngestWritesRawLocationAndTelemetryFactsToTdengineWhenWriterExists() throws Exception { + void tryIngestWritesRawAndLocationFactsToTdengineWhenWriterExists() throws Exception { CapturingStore store = new CapturingStore(); CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter(); EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor( @@ -180,10 +180,7 @@ class EventHistoryEnvelopeIngestorTest { .extracting(TdengineLocationRow::factId) .containsExactly("jt808-location-1"); assertThat(tdengineWriter.locations.getFirst().longitude()).isEqualTo(113.12); - assertThat(tdengineWriter.telemetryFields) - .extracting(TdengineTelemetryFieldRow::fieldKey) - .containsExactly("location.speedKmh"); - assertThat(tdengineWriter.telemetryFields.getFirst().valueDouble()).isEqualTo(42.5); + assertThat(tdengineWriter.telemetryFields).isEmpty(); } @Test @@ -200,16 +197,14 @@ class EventHistoryEnvelopeIngestorTest { .containsExactly("frame-jt808-1"); assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId) .containsExactly("jt808-location-1"); - assertThat(tdengineWriter.telemetryFields) - .extracting(TdengineTelemetryFieldRow::fieldKey) - .containsExactly("location.speedKmh"); + assertThat(tdengineWriter.telemetryFields).isEmpty(); } @Test - void tryIngestCanSkipTelemetryFieldFactsForHighThroughputRuntime() { + void tryIngestCanWriteTelemetryFieldFactsWhenExplicitlyEnabled() { CapturingTdengineWriter tdengineWriter = new CapturingTdengineWriter(); EventHistoryEnvelopeIngestor ingestor = new EventHistoryEnvelopeIngestor( - new TelemetryEnvelopeRecordMapper(), tdengineWriter, false); + new TelemetryEnvelopeRecordMapper(), tdengineWriter, true); EnvelopeIngestResult result = ingestor.tryIngest( jt808LocationEnvelope("jt808-location-1", "frame-jt808-1", "013800000001").toByteArray()); @@ -219,7 +214,9 @@ class EventHistoryEnvelopeIngestorTest { .containsExactly("frame-jt808-1"); assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId) .containsExactly("jt808-location-1"); - assertThat(tdengineWriter.telemetryFields).isEmpty(); + assertThat(tdengineWriter.telemetryFields) + .extracting(TdengineTelemetryFieldRow::fieldKey) + .containsExactly("location.speedKmh"); } @Test @@ -249,9 +246,7 @@ class EventHistoryEnvelopeIngestorTest { .containsExactly("frame-jt808-1", "frame-jt808-2"); assertThat(tdengineWriter.locations).extracting(TdengineLocationRow::factId) .containsExactly("jt808-location-1", "jt808-location-2"); - assertThat(tdengineWriter.telemetryFields) - .extracting(TdengineTelemetryFieldRow::fieldKey) - .containsExactly("location.speedKmh", "location.speedKmh"); + assertThat(tdengineWriter.telemetryFields).isEmpty(); } private static VehicleEnvelope envelope(String eventId) { diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Gb32960DecodedFrameServiceTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Gb32960DecodedFrameServiceTest.java index 4abd12e4..d5db9f1f 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Gb32960DecodedFrameServiceTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Gb32960DecodedFrameServiceTest.java @@ -314,6 +314,7 @@ class Gb32960DecodedFrameServiceTest { "", "127.0.0.1:32960", "{\"platformAccount\":\"Hyundai\"}", + "", "GB32960", vin, vin, @@ -360,6 +361,7 @@ class Gb32960DecodedFrameServiceTest { "", "8.134.95.166:37302", "{\"platformAccount\":\"Hyundai\"}", + "", "GB32960", "unknown:GB32960:platform-login", "", @@ -409,6 +411,7 @@ class Gb32960DecodedFrameServiceTest { "", "127.0.0.1:32960", "{\"platformAccount\":\"Hyundai\"}", + "", "GB32960", vin, vin, diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryControllerTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryControllerTest.java index 4b621cf3..bb1fa2f0 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryControllerTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Jt808LocationHistoryControllerTest.java @@ -51,13 +51,12 @@ class Jt808LocationHistoryControllerTest { } @Test - void fallsBackToRawArchiveUriInMetadataWhenLocationRawUriIsBlank() throws Exception { + void returnsLocationRawUriFromCoreColumn() throws Exception { CapturingReader reader = new CapturingReader(new TdenginePage<>( List.of(row( "fact-blank-raw", "2026-06-29T05:00:01Z", - "", - "{\"rawArchiveUri\":\"archive://jt808/metadata-frame.bin\"}")), + "archive://jt808/core-frame.bin")), Optional.empty())); Jt808LocationHistoryController controller = new Jt808LocationHistoryController(reader); @@ -71,7 +70,7 @@ class Jt808LocationHistoryControllerTest { null); assertThat(response.items().getFirst().rawUri()) - .isEqualTo("archive://jt808/metadata-frame.bin"); + .isEqualTo("archive://jt808/core-frame.bin"); } @Test @@ -92,10 +91,10 @@ class Jt808LocationHistoryControllerTest { } private static TdengineLocationRow row(String factId, String ts) { - return row(factId, ts, "archive://jt808/frame-1.bin", "{\"messageId\":\"512\"}"); + return row(factId, ts, "archive://jt808/frame-1.bin"); } - private static TdengineLocationRow row(String factId, String ts, String rawUri, String metadataJson) { + private static TdengineLocationRow row(String factId, String ts, String rawUri) { Instant instant = Instant.parse(ts); return new TdengineLocationRow( instant, @@ -111,7 +110,6 @@ class Jt808LocationHistoryControllerTest { 3L, null, rawUri, - metadataJson, "JT808", "jt808:g7gps", "", diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Jt808RawFrameHistoryControllerTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Jt808RawFrameHistoryControllerTest.java index 8103a5f0..ea323ed6 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Jt808RawFrameHistoryControllerTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/Jt808RawFrameHistoryControllerTest.java @@ -96,6 +96,7 @@ class Jt808RawFrameHistoryControllerTest { "", "222.66.200.68:41000", "{\"jt808.register.deviceId\":\"9963320\",\"jt808.register.plate\":\"沪A61559F\"}", + "", "JT808", "jt808:13079963320", "unknown", diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/LocationHistoryControllerTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/LocationHistoryControllerTest.java index 0004ea2c..050ba86b 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/LocationHistoryControllerTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/LocationHistoryControllerTest.java @@ -82,7 +82,6 @@ class LocationHistoryControllerTest { 1, null, "", - "{\"rawArchiveUri\":\"archive://raw.bin\"}", "MQTT_YUTONG", "LMRKH9AC2R1004087", "LMRKH9AC2R1004087", diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java index eea46a31..efb2b5fe 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/RawFrameHistoryControllerTest.java @@ -91,7 +91,6 @@ class RawFrameHistoryControllerTest { 524290, 14504.446, raw.rawUri(), - "{\"rawArchiveUri\":\"" + raw.rawUri() + "\"}", "JT808", "VIN-JT808-1", "VIN-JT808-1", @@ -183,6 +182,7 @@ class RawFrameHistoryControllerTest { "xinda", "{\"rawArchiveUri\":\"archive://2026/06/29/XINDA_PUSH/unknown/" + frameId + ".bin\",\"plateNo\":\"粤BD12345\",\"rawJson\":\"{\\\"carName\\\":\\\"粤BD12345\\\"}\"}", + "", "XINDA_PUSH", "LB9A32A24P0LS1270", "LB9A32A24P0LS1270", diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapperTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapperTest.java index b1bce6bf..875a7415 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapperTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/TelemetryEnvelopeRecordMapperTest.java @@ -2,9 +2,11 @@ package com.lingniu.ingest.eventhistory; import com.lingniu.ingest.api.ProtocolId; import com.lingniu.ingest.eventfilestore.EventFileRecord; +import com.lingniu.ingest.sink.mq.proto.RawArchiveRef; import com.lingniu.ingest.sink.mq.proto.TelemetryField; import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot; import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -12,6 +14,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; class TelemetryEnvelopeRecordMapperTest { + private final ObjectMapper objectMapper = new ObjectMapper(); + @Test void mapsFullFieldEnvelopeToEventFileRecord() { VehicleEnvelope envelope = VehicleEnvelope.newBuilder() @@ -91,4 +95,33 @@ class TelemetryEnvelopeRecordMapperTest { .containsEntry("tenant", "ln") .containsEntry("originalSource", "FUTURE_OEM"); } + + @Test + void rawArchiveRecordStoresParsedJsonInPayloadOnly() throws Exception { + VehicleEnvelope envelope = VehicleEnvelope.newBuilder() + .setSchemaVersion("1.0") + .setEventId("raw-808-1") + .setTraceId("trace-raw-1") + .setVin("VIN808") + .setSource("JT808") + .setEventTimeMs(1_782_112_400_000L) + .setIngestTimeMs(1_782_112_401_000L) + .putMetadata("phone", "13307795425") + .setRawArchive(RawArchiveRef.newBuilder() + .setUri("archive://2026/06/30/JT808/VIN808/raw-808-1.bin") + .setSizeBytes(64) + .setParsedJson("{\"messageId\":\"0x0200\",\"body\":{\"speedKmh\":60.5}}") + .build()) + .build(); + + EventFileRecord record = new TelemetryEnvelopeRecordMapper().toRecord(envelope); + + assertThat(record.eventType()).isEqualTo("RAW_ARCHIVE"); + assertThat(record.metadata()) + .containsEntry("phone", "13307795425") + .doesNotContainKey("rawArchiveParsedJson"); + var payload = objectMapper.readTree(record.payloadJson()); + assertThat(payload.path("parsed").path("messageId").asText()).isEqualTo("0x0200"); + assertThat(payload.path("parsed").path("body").path("speedKmh").asDouble()).isEqualTo(60.5); + } } diff --git a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/config/EventHistoryAutoConfigurationTest.java b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/config/EventHistoryAutoConfigurationTest.java index 9bfbe7de..02d1d4b4 100644 --- a/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/config/EventHistoryAutoConfigurationTest.java +++ b/modules/services/event-history-service/src/test/java/com/lingniu/ingest/eventhistory/config/EventHistoryAutoConfigurationTest.java @@ -35,11 +35,14 @@ class EventHistoryAutoConfigurationTest { @Test void createsHistoryBeansWhenEnabled() { contextRunner - .withPropertyValues("lingniu.ingest.event-history.enabled=true") + .withPropertyValues( + "lingniu.ingest.event-history.enabled=true", + "lingniu.ingest.sink.mq.consumer.enabled=true") .run(context -> { assertThat(context).hasSingleBean(TelemetryEnvelopeRecordMapper.class); assertThat(context).hasSingleBean(EventHistoryEnvelopeIngestor.class); - assertThat(context).hasSingleBean(EnvelopeConsumerProcessor.class); + assertThat(context.getBeansOfType(EnvelopeConsumerProcessor.class)) + .containsOnlyKeys("eventHistoryEnvelopeConsumerProcessor"); assertThat(context).hasSingleBean(EventHistoryController.class); assertThat(context).doesNotHaveBean(Gb32960DecodedFrameService.class); assertThat(context).doesNotHaveBean(Gb32960FrameController.class); diff --git a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/config/VehicleStatAutoConfiguration.java b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/config/VehicleStatAutoConfiguration.java index 12ae756a..e4bd434f 100644 --- a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/config/VehicleStatAutoConfiguration.java +++ b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/config/VehicleStatAutoConfiguration.java @@ -23,6 +23,9 @@ import com.lingniu.ingest.vehiclestat.jt808.Jt808MileageStreamProcessor; import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -36,7 +39,11 @@ import java.nio.file.Path; import java.time.Duration; import java.time.ZoneId; -@AutoConfiguration +@AutoConfiguration(after = { + DataSourceAutoConfiguration.class, + JdbcTemplateAutoConfiguration.class, + RedisAutoConfiguration.class +}) @EnableConfigurationProperties(VehicleStatProperties.class) @ConditionalOnProperty(prefix = "lingniu.ingest.vehicle-stat", name = "enabled", havingValue = "true") public class VehicleStatAutoConfiguration { diff --git a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/JdbcJt808DailyMileageRepository.java b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/JdbcJt808DailyMileageRepository.java index dccc3149..bdccc5ed 100644 --- a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/JdbcJt808DailyMileageRepository.java +++ b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/JdbcJt808DailyMileageRepository.java @@ -4,13 +4,16 @@ import org.springframework.jdbc.core.JdbcTemplate; import java.sql.Timestamp; import java.time.Instant; +import java.util.Locale; +import java.util.Objects; public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageRepository { private final JdbcTemplate jdbc; public JdbcJt808DailyMileageRepository(JdbcTemplate jdbc) { - this.jdbc = jdbc; + this.jdbc = Objects.requireNonNull(jdbc, "jdbc must not be null"); + initializeSchema(); } @Override @@ -18,18 +21,23 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR jdbc.update(""" insert into vehicle_daily_mileage_jt808 ( stat_date, vehicle_key, vin, phone, first_event_time, last_event_time, - gps_mileage_km, speed_integral_km, odometer_mileage_km, + daily_mileage_km, mileage_source, gps_mileage_km, speed_integral_km, + odometer_mileage_km, first_total_mileage_km, last_total_mileage_km, accepted_points, bad_jump_segments, long_gap_segments, out_of_order_points, odometer_anomalies, data_quality - ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on duplicate key update vin = values(vin), phone = values(phone), first_event_time = values(first_event_time), last_event_time = values(last_event_time), + daily_mileage_km = values(daily_mileage_km), + mileage_source = values(mileage_source), gps_mileage_km = values(gps_mileage_km), speed_integral_km = values(speed_integral_km), odometer_mileage_km = values(odometer_mileage_km), + first_total_mileage_km = values(first_total_mileage_km), + last_total_mileage_km = values(last_total_mileage_km), accepted_points = values(accepted_points), bad_jump_segments = values(bad_jump_segments), long_gap_segments = values(long_gap_segments), @@ -44,9 +52,13 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR result.phone(), timestamp(result.firstEventTime()), timestamp(result.lastEventTime()), + result.dailyMileageKm(), + result.mileageSource(), result.gpsMileageKm(), result.speedIntegralKm(), result.odometerMileageKm(), + result.firstTotalMileageKm(), + result.lastTotalMileageKm(), result.acceptedPoints(), result.badJumpSegments(), result.longGapSegments(), @@ -55,6 +67,59 @@ public final class JdbcJt808DailyMileageRepository implements Jt808DailyMileageR result.dataQuality()); } + private void initializeSchema() { + jdbc.execute(""" + create table if not exists vehicle_daily_mileage_jt808 ( + stat_date date not null, + vehicle_key varchar(64) not null, + vin varchar(32) null, + phone varchar(32) null, + first_event_time timestamp null, + last_event_time timestamp null, + daily_mileage_km decimal(12,3) not null default 0, + mileage_source varchar(32) not null default 'INSUFFICIENT', + gps_mileage_km decimal(12,3) not null, + speed_integral_km decimal(12,3) not null, + odometer_mileage_km decimal(12,3) null, + first_total_mileage_km decimal(12,3) null, + last_total_mileage_km decimal(12,3) null, + accepted_points int not null, + bad_jump_segments int not null, + long_gap_segments int not null, + out_of_order_points int not null, + odometer_anomalies int not null, + data_quality varchar(16) not null, + updated_at timestamp not null default current_timestamp on update current_timestamp, + primary key (stat_date, vehicle_key), + key idx_vehicle_daily_mileage_jt808_vehicle_date (vehicle_key, stat_date), + key idx_vehicle_daily_mileage_jt808_vin_date (vin, stat_date), + key idx_vehicle_daily_mileage_jt808_phone_date (phone, stat_date) + ) + """); + ensureColumn("daily_mileage_km", "daily_mileage_km decimal(12,3) not null default 0"); + ensureColumn("mileage_source", "mileage_source varchar(32) not null default 'INSUFFICIENT'"); + ensureColumn("first_total_mileage_km", "first_total_mileage_km decimal(12,3) null"); + ensureColumn("last_total_mileage_km", "last_total_mileage_km decimal(12,3) null"); + } + + private void ensureColumn(String columnName, String definition) { + if (!columnExists(columnName)) { + jdbc.execute("alter table vehicle_daily_mileage_jt808 add column " + definition); + } + } + + private boolean columnExists(String columnName) { + Integer count = jdbc.queryForObject(""" + select count(*) + from information_schema.columns + where lower(table_name) = ? + and lower(column_name) = ? + """, Integer.class, + "vehicle_daily_mileage_jt808", + columnName.toLowerCase(Locale.ROOT)); + return count != null && count > 0; + } + private static Timestamp timestamp(Instant instant) { return instant == null ? null : Timestamp.from(instant); } diff --git a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageResult.java b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageResult.java index 43b4b4b1..dd07026e 100644 --- a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageResult.java +++ b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageResult.java @@ -10,9 +10,13 @@ public record Jt808DailyMileageResult( String phone, Instant firstEventTime, Instant lastEventTime, + double dailyMileageKm, + String mileageSource, double gpsMileageKm, double speedIntegralKm, Double odometerMileageKm, + Double firstTotalMileageKm, + Double lastTotalMileageKm, int acceptedPoints, int badJumpSegments, int longGapSegments, diff --git a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageState.java b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageState.java index 83d9259f..bc848a21 100644 --- a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageState.java +++ b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageState.java @@ -12,6 +12,7 @@ public final class Jt808DailyMileageState { private Instant firstEventTime; private Instant lastEventTime; private Jt808LocationPoint lastPoint; + private double firstTotalMileageKm = Double.NaN; private double lastTotalMileageKm = Double.NaN; private double gpsMileageKm; private double speedIntegralKm; @@ -47,6 +48,7 @@ public final class Jt808DailyMileageState { Instant firstEventTime, Instant lastEventTime, Jt808LocationPoint lastPoint, + double firstTotalMileageKm, double lastTotalMileageKm, double gpsMileageKm, double speedIntegralKm, @@ -60,6 +62,7 @@ public final class Jt808DailyMileageState { state.firstEventTime = firstEventTime; state.lastEventTime = lastEventTime; state.lastPoint = lastPoint; + state.firstTotalMileageKm = firstTotalMileageKm; state.lastTotalMileageKm = lastTotalMileageKm; state.gpsMileageKm = gpsMileageKm; state.speedIntegralKm = speedIntegralKm; @@ -82,7 +85,7 @@ public final class Jt808DailyMileageState { } if (lastPoint == null) { firstEventTime = point.eventTime(); - updateOdometer(point); + updateOdometer(point, calculator); accept(point); return; } @@ -100,7 +103,7 @@ public final class Jt808DailyMileageState { gpsMileageKm += segment.gpsKm(); speedIntegralKm += segment.speedIntegralKm(); } - updateOdometer(point); + updateOdometer(point, calculator); accept(point); } @@ -112,9 +115,13 @@ public final class Jt808DailyMileageState { phone, firstEventTime, lastEventTime, + dailyMileageKm(), + mileageSource(), gpsMileageKm, speedIntegralKm, odometerMileageKm > 0.0 ? odometerMileageKm : null, + nullable(firstTotalMileageKm), + nullable(lastTotalMileageKm), acceptedPoints, badJumpSegments, longGapSegments, @@ -129,11 +136,14 @@ public final class Jt808DailyMileageState { acceptedPoints++; } - private void updateOdometer(Jt808LocationPoint point) { + private void updateOdometer(Jt808LocationPoint point, Jt808GpsMileageCalculator calculator) { if (point.totalMileageKm() == null || !Double.isFinite(point.totalMileageKm())) { return; } double current = point.totalMileageKm(); + if (!Double.isFinite(firstTotalMileageKm)) { + firstTotalMileageKm = current; + } if (!Double.isFinite(lastTotalMileageKm)) { lastTotalMileageKm = current; return; @@ -143,15 +153,56 @@ public final class Jt808DailyMileageState { lastTotalMileageKm = current; return; } - odometerMileageKm += current - lastTotalMileageKm; + double delta = current - lastTotalMileageKm; + if (lastPoint != null && !calculator.plausibleOdometerDelta(lastPoint, point, delta)) { + odometerAnomalies++; + lastTotalMileageKm = current; + return; + } + odometerMileageKm += delta; lastTotalMileageKm = current; } + private double dailyMileageKm() { + if (hasUsableTotalMileage()) { + return odometerMileageKm; + } + if (gpsMileageKm > 0.0) { + return gpsMileageKm; + } + return 0.0; + } + + private String mileageSource() { + if (hasUsableTotalMileage()) { + return odometerAnomalies > 0 ? "JT808_TOTAL_MILEAGE_PARTIAL" : "JT808_TOTAL_MILEAGE"; + } + if (gpsMileageKm > 0.0) { + return "GPS_DISTANCE"; + } + return "INSUFFICIENT"; + } + + private boolean hasUsableTotalMileage() { + if (odometerMileageKm > 0.0) { + return true; + } + return acceptedPoints >= 2 + && odometerAnomalies == 0 + && Double.isFinite(firstTotalMileageKm) + && Double.isFinite(lastTotalMileageKm) + && lastTotalMileageKm >= firstTotalMileageKm; + } + + private static Double nullable(double value) { + return Double.isFinite(value) ? value : null; + } + private String dataQuality() { if (acceptedPoints < 2) { return "PARTIAL"; } - if (badJumpSegments > 0 || longGapSegments > 0 || vehicleKey.startsWith("jt808:")) { + if (badJumpSegments > 0 || longGapSegments > 0 || odometerAnomalies > 0 || vehicleKey.startsWith("jt808:")) { return "PARTIAL"; } return "GOOD"; @@ -201,6 +252,10 @@ public final class Jt808DailyMileageState { return lastTotalMileageKm; } + public double firstTotalMileageKm() { + return firstTotalMileageKm; + } + public int acceptedPoints() { return acceptedPoints; } diff --git a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808GpsMileageCalculator.java b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808GpsMileageCalculator.java index d25c9739..a53bf28f 100644 --- a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808GpsMileageCalculator.java +++ b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808GpsMileageCalculator.java @@ -5,6 +5,7 @@ import java.time.Duration; public final class Jt808GpsMileageCalculator { private static final double EARTH_RADIUS_KM = 6371.0088; + private static final double ODOMETER_DELTA_TOLERANCE_KM = 0.1; private final Duration maxSegmentGap; private final double maxImpliedSpeedKmh; @@ -61,6 +62,20 @@ public final class Jt808GpsMileageCalculator { return new SegmentResult(gpsKm, speedIntegralKm, true, false, false, false); } + public boolean plausibleOdometerDelta(Jt808LocationPoint previous, + Jt808LocationPoint current, + double deltaKm) { + if (previous == null || current == null || !Double.isFinite(deltaKm) || deltaKm < 0.0) { + return false; + } + Duration elapsed = Duration.between(previous.eventTime(), current.eventTime()); + if (elapsed.isZero() || elapsed.isNegative()) { + return false; + } + double hours = elapsed.toMillis() / 3_600_000.0; + return deltaKm <= maxImpliedSpeedKmh * hours + ODOMETER_DELTA_TOLERANCE_KM; + } + private static double haversineKm(double lon1, double lat1, double lon2, double lat2) { double latRad1 = Math.toRadians(lat1); double latRad2 = Math.toRadians(lat2); diff --git a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808LocationPointExtractor.java b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808LocationPointExtractor.java index afbf57e0..cd553d68 100644 --- a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808LocationPointExtractor.java +++ b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/Jt808LocationPointExtractor.java @@ -13,7 +13,8 @@ public final class Jt808LocationPointExtractor { public Optional extract(VehicleEnvelope envelope) { if (envelope == null || !"JT808".equalsIgnoreCase(envelope.getSource()) - || !envelope.hasTelemetrySnapshot()) { + || !envelope.hasTelemetrySnapshot() + || !"LOCATION".equalsIgnoreCase(envelope.getTelemetrySnapshot().getEventType())) { return Optional.empty(); } Map fields = fields(envelope); diff --git a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/RedisJt808MileageStateStore.java b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/RedisJt808MileageStateStore.java index 0c6c6002..bf19c19b 100644 --- a/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/RedisJt808MileageStateStore.java +++ b/modules/services/vehicle-stat-service/src/main/java/com/lingniu/ingest/vehiclestat/jt808/RedisJt808MileageStateStore.java @@ -64,7 +64,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore String firstEventTime, String lastEventTime, PointSnapshot lastPoint, - double lastTotalMileageKm, + Double firstTotalMileageKm, + Double lastTotalMileageKm, double gpsMileageKm, double speedIntegralKm, double odometerMileageKm, @@ -83,7 +84,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore format(state.firstEventTime()), format(state.lastEventTime()), PointSnapshot.from(state.lastPoint()), - state.lastTotalMileageKm(), + finiteOrNull(state.firstTotalMileageKm()), + finiteOrNull(state.lastTotalMileageKm()), state.gpsMileageKm(), state.speedIntegralKm(), state.odometerMileageKm(), @@ -103,7 +105,8 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore parseInstant(firstEventTime), parseInstant(lastEventTime), lastPoint == null ? null : lastPoint.toPoint(), - lastTotalMileageKm, + nanIfNull(firstTotalMileageKm), + nanIfNull(lastTotalMileageKm), gpsMileageKm, speedIntegralKm, odometerMileageKm, @@ -164,4 +167,12 @@ public final class RedisJt808MileageStateStore implements Jt808MileageStateStore private static Instant parseInstant(String value) { return value == null || value.isBlank() ? null : Instant.parse(value); } + + private static Double finiteOrNull(double value) { + return Double.isFinite(value) ? value : null; + } + + private static double nanIfNull(Double value) { + return value == null ? Double.NaN : value; + } } diff --git a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/config/VehicleStatAutoConfigurationTest.java b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/config/VehicleStatAutoConfigurationTest.java index a94029d0..ff82104a 100644 --- a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/config/VehicleStatAutoConfigurationTest.java +++ b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/config/VehicleStatAutoConfigurationTest.java @@ -21,10 +21,13 @@ import com.lingniu.ingest.vehiclestat.jt808.RedisJt808MileageStateStore; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import javax.sql.DataSource; import java.time.LocalDate; import java.util.List; import java.util.Optional; @@ -117,6 +120,29 @@ class VehicleStatAutoConfigurationTest { }); } + @Test + void createsJt808MileageBeansAfterJdbcAutoConfiguration() { + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of( + JdbcTemplateAutoConfiguration.class, + VehicleStatAutoConfiguration.class)) + .withBean(EnvelopeDeadLetterSink.class, () -> record -> {}) + .withBean(DataSource.class, () -> new DriverManagerDataSource( + "jdbc:h2:mem:vehicle_stat_auto;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1", + "sa", + "")) + .withPropertyValues( + "lingniu.ingest.vehicle-stat.enabled=true", + "lingniu.ingest.vehicle-stat.jt808.enabled=true", + "lingniu.ingest.vehicle-stat.jt808.state-store=memory") + .run(context -> { + assertThat(context).hasSingleBean(JdbcTemplate.class); + assertThat(context).hasSingleBean(Jt808DailyMileageRepository.class); + assertThat(context).hasSingleBean(Jt808MileageStreamProcessor.class); + assertThat(context).hasSingleBean(VehicleStatEnvelopeIngestor.class); + }); + } + private static final class InMemoryVehicleStatRepository implements VehicleStatRepository { @Override public void appendMileagePoint(String vin, MileagePoint point) {} @Override public List mileagePoints(String vin, LocalDate statDate) { return List.of(); } diff --git a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/JdbcJt808DailyMileageRepositoryTest.java b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/JdbcJt808DailyMileageRepositoryTest.java index 62ec0dcd..4142bbe1 100644 --- a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/JdbcJt808DailyMileageRepositoryTest.java +++ b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/JdbcJt808DailyMileageRepositoryTest.java @@ -24,30 +24,20 @@ class JdbcJt808DailyMileageRepositoryTest { ""); jdbc = new JdbcTemplate(dataSource); jdbc.execute("drop table if exists vehicle_daily_mileage_jt808"); - jdbc.execute(""" - create table vehicle_daily_mileage_jt808 ( - stat_date date not null, - vehicle_key varchar(64) not null, - vin varchar(32) null, - phone varchar(32) null, - first_event_time timestamp null, - last_event_time timestamp null, - gps_mileage_km decimal(12,3) not null, - speed_integral_km decimal(12,3) not null, - odometer_mileage_km decimal(12,3) null, - accepted_points int not null, - bad_jump_segments int not null, - long_gap_segments int not null, - out_of_order_points int not null, - odometer_anomalies int not null, - data_quality varchar(16) not null, - updated_at timestamp not null default current_timestamp, - primary key (stat_date, vehicle_key) - ) - """); repository = new JdbcJt808DailyMileageRepository(jdbc); } + @Test + void initializesTableWhenMissing() { + Integer count = jdbc.queryForObject(""" + select count(*) + from information_schema.tables + where table_name = 'vehicle_daily_mileage_jt808' + """, Integer.class); + + assertThat(count).isEqualTo(1); + } + @Test void upsertsDailyMileageResult() { Jt808DailyMileageResult first = new Jt808DailyMileageResult( @@ -58,8 +48,12 @@ class JdbcJt808DailyMileageRepositoryTest { Instant.parse("2026-06-30T00:00:00Z"), Instant.parse("2026-06-30T00:01:00Z"), 1.23456, + "GPS_DISTANCE", + 1.23456, 1.2, null, + null, + null, 2, 0, 0, @@ -73,9 +67,13 @@ class JdbcJt808DailyMileageRepositoryTest { "13900000001", Instant.parse("2026-06-30T00:00:00Z"), Instant.parse("2026-06-30T00:02:00Z"), + 2.0, + "JT808_TOTAL_MILEAGE", 2.34567, 2.3, 2.0, + 100.0, + 102.0, 3, 1, 0, @@ -87,14 +85,19 @@ class JdbcJt808DailyMileageRepositoryTest { repository.upsert(updated); Map row = jdbc.queryForMap(""" - select vehicle_key, gps_mileage_km, odometer_mileage_km, accepted_points, - bad_jump_segments, data_quality + select vehicle_key, daily_mileage_km, mileage_source, gps_mileage_km, + odometer_mileage_km, first_total_mileage_km, last_total_mileage_km, + accepted_points, bad_jump_segments, data_quality from vehicle_daily_mileage_jt808 where stat_date = '2026-06-30' and vehicle_key = 'VIN001' """); assertThat(row.get("vehicle_key")).isEqualTo("VIN001"); + assertThat(row.get("daily_mileage_km").toString()).isEqualTo("2.000"); + assertThat(row.get("mileage_source")).isEqualTo("JT808_TOTAL_MILEAGE"); assertThat(row.get("gps_mileage_km").toString()).isEqualTo("2.346"); assertThat(row.get("odometer_mileage_km").toString()).isEqualTo("2.000"); + assertThat(row.get("first_total_mileage_km").toString()).isEqualTo("100.000"); + assertThat(row.get("last_total_mileage_km").toString()).isEqualTo("102.000"); assertThat(row.get("accepted_points")).isEqualTo(3); assertThat(row.get("bad_jump_segments")).isEqualTo(1); assertThat(row.get("data_quality")).isEqualTo("PARTIAL"); diff --git a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageStateTest.java b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageStateTest.java index 033afe10..6d5882bc 100644 --- a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageStateTest.java +++ b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808DailyMileageStateTest.java @@ -25,6 +25,10 @@ class Jt808DailyMileageStateTest { assertThat(state.gpsMileageKm()).isZero(); assertThat(state.speedIntegralKm()).isZero(); assertThat(state.odometerMileageKm()).isZero(); + assertThat(state.firstTotalMileageKm()).isEqualTo(100.0); + assertThat(state.lastTotalMileageKm()).isEqualTo(100.0); + assertThat(state.toResult().dailyMileageKm()).isZero(); + assertThat(state.toResult().mileageSource()).isEqualTo("INSUFFICIENT"); assertThat(state.firstEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z")); assertThat(state.lastEventTime()).isEqualTo(Instant.parse("2026-06-30T00:00:00Z")); } @@ -41,6 +45,12 @@ class Jt808DailyMileageStateTest { assertThat(state.gpsMileageKm()).isBetween(0.99, 1.01); assertThat(state.speedIntegralKm()).isBetween(0.99, 1.01); assertThat(state.odometerMileageKm()).isCloseTo(1.2, offset(0.000001)); + assertThat(state.firstTotalMileageKm()).isEqualTo(100.0); + assertThat(state.lastTotalMileageKm()).isEqualTo(101.2); + assertThat(state.toResult().dailyMileageKm()).isCloseTo(1.2, offset(0.000001)); + assertThat(state.toResult().mileageSource()).isEqualTo("JT808_TOTAL_MILEAGE"); + assertThat(state.toResult().firstTotalMileageKm()).isEqualTo(100.0); + assertThat(state.toResult().lastTotalMileageKm()).isEqualTo(101.2); assertThat(state.badJumpSegments()).isZero(); assertThat(state.longGapSegments()).isZero(); } @@ -68,6 +78,36 @@ class Jt808DailyMileageStateTest { assertThat(state.odometerMileageKm()).isZero(); assertThat(state.odometerAnomalies()).isEqualTo(1); + assertThat(state.toResult().dailyMileageKm()).isGreaterThan(0.9); + assertThat(state.toResult().mileageSource()).isEqualTo("GPS_DISTANCE"); + } + + @Test + void odometerJumpAboveConfiguredSpeedIsIgnoredAndCounted() { + Jt808DailyMileageState state = Jt808DailyMileageState.empty("VIN001", "VIN001", "13900000001", + LocalDate.of(2026, 6, 30)); + state.apply(point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0, 100.0), calculator); + + state.apply(point("2026-06-30T00:01:00Z", 120.000100, 30.000000, 36.0, 200.0), calculator); + + assertThat(state.odometerMileageKm()).isZero(); + assertThat(state.odometerAnomalies()).isEqualTo(1); + assertThat(state.toResult().dataQuality()).isEqualTo("PARTIAL"); + } + + @Test + void gpsDistanceIsFallbackWhenTotalMileageIsMissing() { + Jt808DailyMileageState state = Jt808DailyMileageState.empty("VIN001", "VIN001", "13900000001", + LocalDate.of(2026, 6, 30)); + state.apply(point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0, null), calculator); + + state.apply(point("2026-06-30T00:01:40Z", 120.010370, 30.000000, 36.0, null), calculator); + + assertThat(state.toResult().dailyMileageKm()).isBetween(0.99, 1.01); + assertThat(state.toResult().mileageSource()).isEqualTo("GPS_DISTANCE"); + assertThat(state.toResult().odometerMileageKm()).isNull(); + assertThat(state.toResult().firstTotalMileageKm()).isNull(); + assertThat(state.toResult().lastTotalMileageKm()).isNull(); } private static Jt808LocationPoint point(String time, diff --git a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808GpsMileageCalculatorTest.java b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808GpsMileageCalculatorTest.java index 868c739e..4e81aac4 100644 --- a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808GpsMileageCalculatorTest.java +++ b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808GpsMileageCalculatorTest.java @@ -71,6 +71,18 @@ class Jt808GpsMileageCalculatorTest { assertThat(result.gpsKm()).isZero(); } + @Test + void validatesPlausibleOdometerDeltaByConfiguredSpeed() { + assertThat(calculator.plausibleOdometerDelta( + point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0), + point("2026-06-30T00:01:00Z", 120.001000, 30.000000, 36.0), + 2.0)).isTrue(); + assertThat(calculator.plausibleOdometerDelta( + point("2026-06-30T00:00:00Z", 120.000000, 30.000000, 36.0), + point("2026-06-30T00:01:00Z", 120.001000, 30.000000, 36.0), + 20.0)).isFalse(); + } + private static Jt808LocationPoint point(String time, double longitude, double latitude, Double speedKmh) { return new Jt808LocationPoint( "VIN001", diff --git a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808LocationPointExtractorTest.java b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808LocationPointExtractorTest.java index de3c5a77..a6d555d7 100644 --- a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808LocationPointExtractorTest.java +++ b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808LocationPointExtractorTest.java @@ -57,6 +57,21 @@ class Jt808LocationPointExtractorTest { field("longitude", "118.901034")))).isEmpty(); } + @Test + void ignoresJt808NonLocationSnapshot() { + VehicleEnvelope envelope = envelope("JT808", "VIN001", "13900000001", + field("longitude", "118.901034"), + field("latitude", "31.946325")) + .toBuilder() + .setTelemetrySnapshot(TelemetrySnapshot.newBuilder() + .setEventType("REGISTER") + .addFields(field("longitude", "118.901034")) + .addFields(field("latitude", "31.946325"))) + .build(); + + assertThat(extractor.extract(envelope)).isEmpty(); + } + private static VehicleEnvelope envelope(String source, String vin, String phone, TelemetryField... fields) { TelemetrySnapshot.Builder snapshot = TelemetrySnapshot.newBuilder().setEventType("LOCATION"); for (TelemetryField field : fields) { diff --git a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808MileageStreamProcessorTest.java b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808MileageStreamProcessorTest.java index 8b5c5ae0..9bd9a7b4 100644 --- a/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808MileageStreamProcessorTest.java +++ b/modules/services/vehicle-stat-service/src/test/java/com/lingniu/ingest/vehiclestat/jt808/Jt808MileageStreamProcessorTest.java @@ -30,6 +30,8 @@ class Jt808MileageStreamProcessorTest { assertThat(state.gpsMileageKm()).isGreaterThan(0.9); assertThat(repository.results).hasSize(2); assertThat(repository.results.getLast().vehicleKey()).isEqualTo("VIN001"); + assertThat(repository.results.getLast().dailyMileageKm()).isEqualTo(1.0); + assertThat(repository.results.getLast().mileageSource()).isEqualTo("JT808_TOTAL_MILEAGE"); assertThat(repository.results.getLast().odometerMileageKm()).isEqualTo(1.0); } diff --git a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStore.java b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStore.java index f583916e..029b0d46 100644 --- a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStore.java +++ b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStore.java @@ -36,11 +36,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { private static final TypeReference> STRING_MAP = new TypeReference<>() {}; private static final String HEADER_VERSION = "event-records-v1"; + private static final int VEHICLE_BUCKETS = 64; private final Path root; private final Path indexPath; private final ZoneId partitionZone; private final ObjectMapper objectMapper; + private final boolean writeParquetEnabled; private volatile boolean indexInitialized; public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) { @@ -48,6 +50,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { } public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) { + this(root, partitionZone, objectMapper, true); + } + + public DuckDbParquetEventFileStore(Path root, + ZoneId partitionZone, + ObjectMapper objectMapper, + boolean writeParquetEnabled) { if (root == null) { throw new IllegalArgumentException("root must not be null"); } @@ -55,6 +64,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { this.indexPath = this.root.resolve("events.duckdb"); this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone; this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper; + this.writeParquetEnabled = writeParquetEnabled; } @Override @@ -67,16 +77,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { Partition partition = partition(record); byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record); } - for (Map.Entry> entry : byPartition.entrySet()) { - writePartition(entry.getKey(), entry.getValue()); - appendIndex(entry.getKey(), entry.getValue()); + if (writeParquetEnabled) { + writePartitions(byPartition); } + appendIndex(byPartition); } @Override public List query(EventFileQuery query) throws IOException { - // 有 VIN 时直接扫该车分区 parquet，避免先查总索引再和车辆文件 join。 - if (query.vin() != null) { + // 有 VIN 且启用 Parquet 副本时，可直接扫分区文件；索引-only 模式走 DuckDB 主索引。 + if (query.vin() != null && writeParquetEnabled) { return queryParquet(query); } // 无 VIN 的管理类查询走 DuckDB 索引，代价更高但不影响高频单车查询路径。 @@ -130,6 +140,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { if (query.eventTimeTo() != null) { predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli()); } + if (query.cursorEventTime() != null) { + predicates.add(cursorPredicateSql(query)); + } // 这里的 SQL 只拼接经过 escape 的内部值；外部 rawArchiveUri 查询使用 PreparedStatement。 String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n"; String sql = """ @@ -182,6 +195,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { if (query.eventTimeTo() != null) { where.append(" AND event_time_ms <= ?\n"); } + if (query.cursorEventTime() != null) { + where.append(cursorPredicatePrepared(query.order())); + } String sql = """ SELECT event_id, protocol, event_type, vin, event_time_ms, ingest_time_ms, @@ -209,6 +225,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { if (query.eventTimeTo() != null) { ps.setLong(index++, query.eventTimeTo().toEpochMilli()); } + if (query.cursorEventTime() != null) { + long eventTimeMs = query.cursorEventTime().toEpochMilli(); + long ingestTimeMs = query.cursorIngestTime().toEpochMilli(); + ps.setLong(index++, eventTimeMs); + ps.setLong(index++, eventTimeMs); + ps.setLong(index++, ingestTimeMs); + ps.setLong(index++, eventTimeMs); + ps.setLong(index++, ingestTimeMs); + ps.setString(index++, query.cursorEventId()); + } ps.setInt(index, query.limit()); try (ResultSet rs = ps.executeQuery()) { List result = new ArrayList<>(); @@ -222,16 +248,43 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { } } - private void writePartition(Partition partition, List records) throws IOException { - Path dir = partition.dir(root); - Files.createDirectories(dir); - Path partFile = dir.resolve("events.parquet"); - Path tempFile = dir.resolve("events-" + UUID.randomUUID() + ".tmp.parquet"); + private static String cursorPredicateSql(EventFileQuery query) { + long eventTimeMs = query.cursorEventTime().toEpochMilli(); + long ingestTimeMs = query.cursorIngestTime().toEpochMilli(); + String eventId = escapeSql(query.cursorEventId()); + String cmp = query.order() == EventFileQuery.Order.DESC ? "<" : ">"; + return """ + ( + event_time_ms %s %d + OR (event_time_ms = %d AND ingest_time_ms %s %d) + OR (event_time_ms = %d AND ingest_time_ms = %d AND event_id %s '%s') + ) + """.formatted(cmp, eventTimeMs, eventTimeMs, cmp, ingestTimeMs, + eventTimeMs, ingestTimeMs, cmp, eventId); + } + + private static String cursorPredicatePrepared(EventFileQuery.Order order) { + String cmp = order == EventFileQuery.Order.DESC ? "<" : ">"; + return """ + AND ( + event_time_ms %s ? + OR (event_time_ms = ? AND ingest_time_ms %s ?) + OR (event_time_ms = ? AND ingest_time_ms = ? AND event_id %s ?) + ) + """.formatted(cmp, cmp, cmp); + } + + private void writePartitions(Map> recordsByPartition) throws IOException { + if (recordsByPartition.isEmpty()) { + return; + } + List tempFiles = new ArrayList<>(); try (Connection connection = DriverManager.getConnection("jdbc:duckdb:"); Statement statement = connection.createStatement()) { statement.execute(""" CREATE TEMPORARY TABLE event_records ( + partition_id INTEGER, event_id VARCHAR, protocol VARCHAR, event_type VARCHAR, @@ -246,54 +299,62 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { ) """); try (PreparedStatement ps = connection.prepareStatement(""" - INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """)) { - for (EventFileRecord record : records) { - ps.setString(1, record.eventId()); - ps.setString(2, record.protocol().name()); - ps.setString(3, record.eventType()); - ps.setString(4, record.vin()); - ps.setLong(5, record.eventTime().toEpochMilli()); - ps.setString(6, record.eventTime().toString()); - ps.setLong(7, record.ingestTime().toEpochMilli()); - ps.setString(8, record.ingestTime().toString()); - ps.setString(9, record.rawArchiveUri()); - ps.setString(10, objectMapper.writeValueAsString(record.metadata())); - ps.setString(11, record.payloadJson()); - ps.addBatch(); + int partitionId = 0; + for (List records : recordsByPartition.values()) { + for (EventFileRecord record : records) { + ps.setInt(1, partitionId); + ps.setString(2, record.eventId()); + ps.setString(3, record.protocol().name()); + ps.setString(4, record.eventType()); + ps.setString(5, record.vin()); + ps.setLong(6, record.eventTime().toEpochMilli()); + ps.setString(7, record.eventTime().toString()); + ps.setLong(8, record.ingestTime().toEpochMilli()); + ps.setString(9, record.ingestTime().toString()); + ps.setString(10, record.rawArchiveUri()); + ps.setString(11, objectMapper.writeValueAsString(record.metadata())); + ps.setString(12, record.payloadJson()); + ps.addBatch(); + } + partitionId++; } ps.executeBatch(); } - if (Files.exists(partFile)) { - // DuckDB 不能原地追加 parquet；先 UNION 旧文件和本批数据，再原子替换分区文件。 + int partitionId = 0; + for (Partition partition : recordsByPartition.keySet()) { + Path dir = partition.dir(root); + Files.createDirectories(dir); + String fragmentId = UUID.randomUUID().toString(); + Path partFile = dir.resolve("events-" + fragmentId + ".parquet"); + Path tempFile = dir.resolve("events-" + fragmentId + ".tmp.parquet"); + tempFiles.add(tempFile); statement.execute(""" - CREATE TEMPORARY TABLE combined_records AS - SELECT event_id, protocol, event_type, vin, - event_time_ms, event_time, ingest_time_ms, ingest_time, - raw_archive_uri, metadata_json, payload_json - FROM read_parquet('%s') - UNION ALL - SELECT event_id, protocol, event_type, vin, - event_time_ms, event_time, ingest_time_ms, ingest_time, - raw_archive_uri, metadata_json, payload_json - FROM event_records - """.formatted(escapeSql(partFile))); - statement.execute("COPY combined_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)"); - } else { - statement.execute("COPY event_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)"); + COPY ( + SELECT event_id, protocol, event_type, vin, + event_time_ms, event_time, ingest_time_ms, ingest_time, + raw_archive_uri, metadata_json, payload_json + FROM event_records + WHERE partition_id = %d + ) TO '%s' (FORMAT parquet) + """.formatted(partitionId, escapeSql(tempFile))); + Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + partitionId++; } - Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); } catch (SQLException e) { - throw new IOException("write parquet event store failed: " + partFile, e); + throw new IOException("write parquet event store failed", e); } finally { - Files.deleteIfExists(tempFile); + for (Path tempFile : tempFiles) { + Files.deleteIfExists(tempFile); + } } } - private void appendIndex(Partition partition, List records) throws IOException { + private void appendIndex(Map> recordsByPartition) throws IOException { ensureIndexInitialized(); try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) { - insertIndex(connection, partition, records); + insertIndex(connection, recordsByPartition); } catch (SQLException e) { throw new IOException("write duckdb event index failed", e); } @@ -364,23 +425,26 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { """.formatted(fileList)); } - private void insertIndex(Connection connection, Partition partition, List records) + private void insertIndex(Connection connection, Map> recordsByPartition) throws SQLException, IOException { try (PreparedStatement ps = connection.prepareStatement(""" INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """)) { - for (EventFileRecord record : records) { - ps.setString(1, record.eventId()); - ps.setString(2, record.protocol().name()); - ps.setString(3, record.eventType()); - ps.setString(4, record.vin()); - ps.setLong(5, record.eventTime().toEpochMilli()); - ps.setLong(6, record.ingestTime().toEpochMilli()); - ps.setString(7, partition.date().toString()); - ps.setString(8, record.rawArchiveUri()); - ps.setString(9, objectMapper.writeValueAsString(record.metadata())); - ps.setString(10, record.payloadJson()); - ps.addBatch(); + for (Map.Entry> entry : recordsByPartition.entrySet()) { + Partition partition = entry.getKey(); + for (EventFileRecord record : entry.getValue()) { + ps.setString(1, record.eventId()); + ps.setString(2, record.protocol().name()); + ps.setString(3, record.eventType()); + ps.setString(4, record.vin()); + ps.setLong(5, record.eventTime().toEpochMilli()); + ps.setLong(6, record.ingestTime().toEpochMilli()); + ps.setString(7, partition.date().toString()); + ps.setString(8, record.rawArchiveUri()); + ps.setString(9, objectMapper.writeValueAsString(record.metadata())); + ps.setString(10, record.payloadJson()); + ps.addBatch(); + } } ps.executeBatch(); } @@ -390,15 +454,22 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { List files = new ArrayList<>(); LocalDate date = query.dateFrom(); while (!date.isAfter(query.dateTo())) { - Path dir = query.vin() == null - ? partitionDir(query.protocol(), date) - : partitionDir(query.protocol(), date).resolve("vehicle=" + storageName(query.vin())); - // 目录结构把 VIN 放在 date 下，单车多日查询只需要遍历目标日期内的目标车辆目录。 - if (Files.isDirectory(dir)) { - try (Stream stream = Files.walk(dir)) { - stream.filter(path -> path.getFileName().toString().endsWith(".parquet")) - .sorted() - .forEach(files::add); + Path dateDir = partitionDir(query.protocol(), date); + List candidateDirs = query.vin() == null + ? List.of(dateDir) + : List.of( + // 新布局：高写入路径按协议/日期写一个批次文件，VIN 由 Parquet/索引字段过滤。 + dateDir.resolve("header=" + HEADER_VERSION), + // 兼容已落盘的旧布局。 + dateDir.resolve("bucket=" + bucketName(query.vin())), + dateDir.resolve("vehicle=" + storageName(query.vin()))); + for (Path dir : candidateDirs) { + if (Files.isDirectory(dir)) { + try (Stream stream = Files.walk(dir)) { + stream.filter(path -> path.getFileName().toString().endsWith(".parquet")) + .sorted() + .forEach(files::add); + } } } date = date.plusDays(1); @@ -434,8 +505,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { private Partition partition(EventFileRecord record) { return new Partition( record.protocol(), - LocalDate.ofInstant(record.eventTime(), partitionZone), - storageName(record.vin())); + LocalDate.ofInstant(record.eventTime(), partitionZone)); } private Path partitionDir(ProtocolId protocol, LocalDate date) { @@ -488,11 +558,18 @@ public final class DuckDbParquetEventFileStore implements EventFileStore { return vin.replaceAll("[^A-Za-z0-9._-]", "_"); } - private record Partition(ProtocolId protocol, LocalDate date, String vehicle) { + private static String bucketName(String vin) { + String storageName = storageName(vin); + if ("_unknown".equals(storageName)) { + return "_unknown"; + } + return "%02d".formatted(Math.floorMod(storageName.hashCode(), VEHICLE_BUCKETS)); + } + + private record Partition(ProtocolId protocol, LocalDate date) { private Path dir(Path root) { return root.resolve("protocol=" + protocol.name()) .resolve("date=" + date) - .resolve("vehicle=" + vehicle) .resolve("header=" + HEADER_VERSION); } } diff --git a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileQuery.java b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileQuery.java index f0e52f6b..488ee7bb 100644 --- a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileQuery.java +++ b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileQuery.java @@ -20,7 +20,10 @@ public record EventFileQuery( Order order, int limit, String vin, - String eventType + String eventType, + Instant cursorEventTime, + Instant cursorIngestTime, + String cursorEventId ) { public enum Order { ASC, @@ -45,6 +48,24 @@ public record EventFileQuery( limit = limit <= 0 ? 100 : limit; vin = vin == null || vin.isBlank() ? null : vin.trim(); eventType = eventType == null || eventType.isBlank() ? null : eventType.trim(); + cursorEventId = cursorEventId == null || cursorEventId.isBlank() ? null : cursorEventId.trim(); + if ((cursorEventTime == null || cursorIngestTime == null || cursorEventId == null) + && !(cursorEventTime == null && cursorIngestTime == null && cursorEventId == null)) { + throw new IllegalArgumentException("cursor eventTime, ingestTime and eventId must be provided together"); + } + } + + public EventFileQuery(ProtocolId protocol, + LocalDate dateFrom, + LocalDate dateTo, + Instant eventTimeFrom, + Instant eventTimeTo, + Order order, + int limit, + String vin, + String eventType) { + this(protocol, dateFrom, dateTo, eventTimeFrom, eventTimeTo, order, limit, + vin, eventType, null, null, null); } public EventFileQuery(ProtocolId protocol, diff --git a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileStoreSink.java b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileStoreSink.java index 10d79ae4..0b8153c9 100644 --- a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileStoreSink.java +++ b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileStoreSink.java @@ -226,9 +226,9 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable { return payload; } - private static Map rawArchivePayload(VehicleEvent.RawArchive raw, - String rawArchiveKey, - String rawArchiveUri) { + private Map rawArchivePayload(VehicleEvent.RawArchive raw, + String rawArchiveKey, + String rawArchiveUri) throws IOException { Map payload = new LinkedHashMap<>(); payload.put("eventId", raw.eventId()); payload.put("vin", raw.vin()); @@ -241,10 +241,18 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable { payload.put("command", hex(raw.command())); payload.put("infoType", hex(raw.infoType())); payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length); + putParsedJson(payload, raw.parsedJson()); payload.put("metadata", raw.metadata()); return payload; } + private void putParsedJson(Map payload, String parsedJson) throws IOException { + if (parsedJson == null || parsedJson.isBlank()) { + return; + } + payload.put("parsed", objectMapper.readTree(parsedJson)); + } + private static String hex(int value) { return "0x" + String.format("%04X", value); } diff --git a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfiguration.java b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfiguration.java index de3863d6..305d7ba1 100644 --- a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfiguration.java +++ b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfiguration.java @@ -44,7 +44,8 @@ public class EventFileStoreAutoConfiguration { return new DuckDbParquetEventFileStore( Path.of(properties.getPath()), ZoneId.of(properties.getZoneId()), - mapper); + mapper, + properties.isWriteParquetEnabled()); } @Bean diff --git a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreProperties.java b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreProperties.java index b95043a0..4db78bca 100644 --- a/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreProperties.java +++ b/modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreProperties.java @@ -11,10 +11,10 @@ public class EventFileStoreProperties { private boolean enabled = false; /** - * Parquet 文件库根路径。 + * 历史库根路径。 * - *

实际结构为 {@code protocol=GB32960/date=yyyy-MM-dd/vehicle=VIN/header=event-records-v1/events.parquet}， - * 另有 {@code events.duckdb} sidecar index 放在根路径下。 + *

{@code events.duckdb} 是可查询主索引；开启 Parquet 副本时，还会在该根目录下按 + * {@code protocol/date/header} 保存追加文件。 */ private String path = "./event-store"; @@ -39,6 +39,14 @@ public class EventFileStoreProperties { */ private long flushIntervalMillis = 1000; + /** + * 是否同步写 Parquet 副本。 + * + *

高写入生产链路可以关闭，只保留 DuckDB JSON 主索引和 raw archive 引用；需要离线副本或 + * 索引重建能力时再开启。 + */ + private boolean writeParquetEnabled = true; + public boolean isEnabled() { return enabled; } @@ -78,4 +86,12 @@ public class EventFileStoreProperties { public void setFlushIntervalMillis(long flushIntervalMillis) { this.flushIntervalMillis = flushIntervalMillis; } + + public boolean isWriteParquetEnabled() { + return writeParquetEnabled; + } + + public void setWriteParquetEnabled(boolean writeParquetEnabled) { + this.writeParquetEnabled = writeParquetEnabled; + } } diff --git a/modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStoreTest.java b/modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStoreTest.java index 94e3cf79..bdb2688e 100644 --- a/modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStoreTest.java +++ b/modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStoreTest.java @@ -1,6 +1,7 @@ package com.lingniu.ingest.eventfilestore; import com.lingniu.ingest.api.ProtocolId; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -11,6 +12,7 @@ import java.time.LocalDate; import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; @@ -28,7 +30,7 @@ class DuckDbParquetEventFileStoreTest { store.appendAll(List.of(first, second)); Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22"); - assertThat(parquetFiles(dateDir)).hasSize(2); + assertThat(parquetFiles(dateDir)).hasSize(1); EventFileQuery ascending = new EventFileQuery( ProtocolId.GB32960, @@ -135,16 +137,13 @@ class DuckDbParquetEventFileStoreTest { } @Test - void appendsSameVehicleDayHeaderToOneParquetFile() throws Exception { + void appendsSameVehicleDayHeaderAsAppendOnlyParquetFragments() throws Exception { EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai")); store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME")); store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME")); - Path vehicleDayDir = tempDir.resolve("protocol=GB32960") - .resolve("date=2026-06-22") - .resolve("vehicle=VIN-SAME") - .resolve("header=event-records-v1"); - assertThat(parquetFiles(vehicleDayDir)).containsExactly(vehicleDayDir.resolve("events.parquet")); + Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22"); + assertThat(parquetFiles(dateDir)).hasSize(2); EventFileQuery query = new EventFileQuery( ProtocolId.GB32960, @@ -159,19 +158,54 @@ class DuckDbParquetEventFileStoreTest { .containsExactly("first", "second"); } + @Test + void writesManyVehiclesIntoBoundedDailyLayoutInsteadOfVehicleDirectories() throws Exception { + EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai")); + List records = IntStream.range(0, 80) + .mapToObj(index -> rawRecord( + "raw-" + index, + Instant.parse("2026-06-21T16:00:00Z").plusMillis(index), + "VIN-" + index)) + .toList(); + + store.appendAll(records); + + Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22"); + try (var files = Files.list(dateDir)) { + List partitionDirs = files + .filter(Files::isDirectory) + .map(path -> path.getFileName().toString()) + .sorted() + .toList(); + assertThat(partitionDirs).noneMatch(name -> name.startsWith("vehicle=")); + assertThat(partitionDirs).hasSizeLessThan(records.size()); + } + } + + @Test + void writesOneDailyParquetFragmentForOneBatchAcrossManyVehicles() throws Exception { + EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai")); + List records = IntStream.range(0, 80) + .mapToObj(index -> rawRecord( + "daily-raw-" + index, + Instant.parse("2026-06-21T16:00:00Z").plusMillis(index), + "VIN-DAILY-" + index)) + .toList(); + + store.appendAll(records); + + Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22"); + assertThat(parquetFiles(dateDir)).hasSize(1); + } + @Test void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception { EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai")); store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED")); - Path vehicleDayDir = tempDir.resolve("protocol=GB32960") - .resolve("date=2026-06-22") - .resolve("vehicle=VIN-INDEXED") - .resolve("header=event-records-v1"); - try (var files = Files.list(vehicleDayDir)) { - for (Path file : files.filter(p -> p.getFileName().toString().endsWith(".parquet")).toList()) { - Files.delete(file); - } + Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22"); + for (Path file : parquetFiles(dateDir)) { + Files.delete(file); } EventFileQuery query = new EventFileQuery( @@ -219,6 +253,31 @@ class DuckDbParquetEventFileStoreTest { assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri()); } + @Test + void canQueryVinRecordsFromIndexWhenParquetWritingIsDisabled() throws Exception { + EventFileStore store = new DuckDbParquetEventFileStore( + tempDir, + ZoneId.of("Asia/Shanghai"), + new ObjectMapper(), + false); + EventFileRecord record = rawRecord("index-only", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEX-ONLY"); + + store.append(record); + + assertThat(parquetFiles(tempDir)).isEmpty(); + EventFileQuery query = new EventFileQuery( + ProtocolId.GB32960, + LocalDate.parse("2026-06-22"), + LocalDate.parse("2026-06-22"), + EventFileQuery.Order.DESC, + 10, + "VIN-INDEX-ONLY", + "RAW_ARCHIVE"); + assertThat(store.query(query)) + .extracting(EventFileRecord::eventId) + .containsExactly("index-only"); + } + private static List parquetFiles(Path root) throws Exception { try (var files = Files.walk(root)) { return files.filter(p -> p.getFileName().toString().endsWith(".parquet")) diff --git a/modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfigurationTest.java b/modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfigurationTest.java index 57546aee..55c5b645 100644 --- a/modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfigurationTest.java +++ b/modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfigurationTest.java @@ -20,12 +20,15 @@ class EventFileStoreAutoConfigurationTest { contextRunner .withPropertyValues( "lingniu.ingest.event-file-store.enabled=true", - "lingniu.ingest.event-file-store.path=target/test-event-store") + "lingniu.ingest.event-file-store.path=target/test-event-store", + "lingniu.ingest.event-file-store.write-parquet-enabled=false") .run(context -> { assertThat(context).hasSingleBean(EventFileStore.class); assertThat(context).hasSingleBean(EventFileStoreSink.class); assertThat(context.getBean(EventFileStoreProperties.class).getPath()) .isEqualTo(Path.of("target/test-event-store").toString()); + assertThat(context.getBean(EventFileStoreProperties.class).isWriteParquetEnabled()) + .isFalse(); }); } diff --git a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/EnvelopeMapper.java b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/EnvelopeMapper.java index 05a36ad0..2398d5e4 100644 --- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/EnvelopeMapper.java +++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/EnvelopeMapper.java @@ -92,6 +92,7 @@ public final class EnvelopeMapper { .setUri(uri) .setChecksum(checksum) .setSizeBytes(size) + .setParsedJson(nullToEmpty(ra.parsedJson())) .build()); b.setRawFrameFact(com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload.newBuilder() .setFrameId(frameId) diff --git a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerRunner.java b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerRunner.java index a9833c89..dadd2679 100644 --- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerRunner.java +++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerRunner.java @@ -10,17 +10,19 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCloseable { private static final Logger log = LoggerFactory.getLogger(KafkaEnvelopeConsumerRunner.class); - private final List workers; + private final Supplier> workersSupplier; private final Duration pollTimeout; private final Duration loopBackoff; private final boolean autoStartup; private final AtomicBoolean running = new AtomicBoolean(false); private final AtomicBoolean closed = new AtomicBoolean(false); + private volatile List workers; private ExecutorService executor; public KafkaEnvelopeConsumerRunner(List workers, @@ -30,14 +32,28 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl if (workers == null || workers.isEmpty()) { throw new IllegalArgumentException("workers must not be empty"); } + this.workersSupplier = () -> List.copyOf(workers); this.workers = List.copyOf(workers); this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout; this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff; this.autoStartup = autoStartup; } + public KafkaEnvelopeConsumerRunner(Supplier> workersSupplier, + Duration pollTimeout, + Duration loopBackoff, + boolean autoStartup) { + if (workersSupplier == null) { + throw new IllegalArgumentException("workersSupplier must not be null"); + } + this.workersSupplier = workersSupplier; + this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout; + this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff; + this.autoStartup = autoStartup; + } + public List workers() { - return workers; + return workersOrCreate(); } @Override @@ -45,17 +61,35 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl if (!running.compareAndSet(false, true)) { return; } + List activeWorkers = workersOrCreate(); // 每个 worker 一条后台线程，避免某个处理器阻塞时拖慢其他消费组。 - executor = Executors.newFixedThreadPool(workers.size(), r -> { + executor = Executors.newFixedThreadPool(activeWorkers.size(), r -> { Thread thread = new Thread(r, "kafka-envelope-consumer"); thread.setDaemon(true); return thread; }); - for (KafkaEnvelopeConsumerWorker worker : workers) { + for (KafkaEnvelopeConsumerWorker worker : activeWorkers) { executor.submit(() -> pollLoop(worker)); } } + private List workersOrCreate() { + List current = workers; + if (current != null) { + return current; + } + synchronized (this) { + if (workers == null) { + List created = workersSupplier.get(); + if (created == null || created.isEmpty()) { + throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings"); + } + workers = List.copyOf(created); + } + return workers; + } + } + private void pollLoop(KafkaEnvelopeConsumerWorker worker) { while (running.get()) { try { @@ -121,7 +155,11 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl if (!closed.compareAndSet(false, true)) { return; } - for (KafkaEnvelopeConsumerWorker worker : workers) { + List current = workers; + if (current == null) { + return; + } + for (KafkaEnvelopeConsumerWorker worker : current) { worker.close(); } } diff --git a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorker.java b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorker.java index 9e8fabe0..9e928eb9 100644 --- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorker.java +++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorker.java @@ -36,26 +36,26 @@ public final class KafkaEnvelopeConsumerWorker implements AutoCloseable { public int pollOnce(Duration timeout) { ConsumerRecords records = consumer.poll(timeout == null ? Duration.ZERO : timeout); - Map> batches = new LinkedHashMap<>(); + Map> byProcessor = new LinkedHashMap<>(); + int processed = 0; for (ConsumerRecord record : records) { EnvelopeConsumerProcessor processor = processorsByTopic.get(record.topic()); if (processor == null) { // worker 可能订阅多个 topic；没有显式绑定处理器的 topic 不参与提交语义。 continue; } - batches.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord( + byProcessor.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord( record.topic(), record.partition(), record.offset(), record.key(), record.value())); + processed++; } - int processed = 0; - for (Map.Entry> entry : batches.entrySet()) { + for (Map.Entry> entry : byProcessor.entrySet()) { // EnvelopeConsumerProcessor 内部会把解析或业务错误转成 DLQ 记录， - // 这里保持 Kafka worker 的职责单一：轮询、批量分发、成功后提交 offset。 - entry.getKey().processAll(entry.getValue()); - processed += entry.getValue().size(); + // 这里保持 Kafka worker 的职责单一：轮询、分发、成功后提交 offset。 + entry.getKey().processBatch(entry.getValue()); } if (processed > 0) { // commitSync 放在批次末尾，保证同一个 poll 批次内的消息按 Kafka offset 一起确认。 diff --git a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqAutoConfiguration.java b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqAutoConfiguration.java index 53185392..b5a6c61c 100644 --- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqAutoConfiguration.java +++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqAutoConfiguration.java @@ -1,6 +1,5 @@ package com.lingniu.ingest.sink.mq; -import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import io.github.resilience4j.circuitbreaker.CircuitBreaker; import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig; import org.apache.kafka.clients.producer.KafkaProducer; @@ -8,15 +7,12 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import java.time.Duration; -import java.util.List; -import java.util.Map; import java.util.Properties; /** @@ -100,21 +96,4 @@ public class SinkMqAutoConfiguration { return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq()); } - @Bean - @ConditionalOnMissingBean - @ConditionalOnBean(EnvelopeConsumerProcessor.class) - @ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true") - public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(Map processors, - SinkMqProperties props) { - // Consumer runner 根据 processor bean 名和配置 binding 生成 worker；没有 binding 时直接失败，避免静默不消费。 - List workers = new KafkaEnvelopeConsumerFactory().createWorkers(processors, props); - if (workers.isEmpty()) { - throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings"); - } - return new KafkaEnvelopeConsumerRunner( - workers, - Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()), - Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()), - props.getConsumer().isAutoStartup()); - } } diff --git a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfiguration.java b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfiguration.java new file mode 100644 index 00000000..2302d080 --- /dev/null +++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfiguration.java @@ -0,0 +1,42 @@ +package com.lingniu.ingest.sink.mq; + +import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; +import org.springframework.beans.factory.ListableBeanFactory; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +@AutoConfiguration(after = SinkMqAutoConfiguration.class) +@AutoConfigureAfter(name = { + "com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration", + "com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration", + "com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration" +}) +@EnableConfigurationProperties(SinkMqProperties.class) +@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true) +public class SinkMqConsumerAutoConfiguration { + + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true") + public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory, + SinkMqProperties props) { + return new KafkaEnvelopeConsumerRunner( + () -> createWorkers(beanFactory, props), + Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()), + Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()), + props.getConsumer().isAutoStartup()); + } + + private List createWorkers(ListableBeanFactory beanFactory, SinkMqProperties props) { + Map processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class); + return new KafkaEnvelopeConsumerFactory().createWorkers(processors, props); + } +} diff --git a/modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto b/modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto index 67dcadb8..910ed28b 100644 --- a/modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto +++ b/modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto @@ -45,6 +45,7 @@ message RawArchiveRef { string uri = 1; string checksum = 2; int64 size_bytes = 3; + string parsed_json = 4; } message TelemetrySnapshot { diff --git a/modules/sinks/sink-mq/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/modules/sinks/sink-mq/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 94fdb2e7..6e83b36d 100644 --- a/modules/sinks/sink-mq/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/modules/sinks/sink-mq/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1 +1,2 @@ com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration +com.lingniu.ingest.sink.mq.SinkMqConsumerAutoConfiguration diff --git a/modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorkerTest.java b/modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorkerTest.java index 323fdcdf..589686b2 100644 --- a/modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorkerTest.java +++ b/modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorkerTest.java @@ -1,6 +1,7 @@ package com.lingniu.ingest.sink.mq; import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord; +import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor; import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor; import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord; import com.lingniu.ingest.api.consumer.EnvelopeIngestResult; @@ -14,6 +15,7 @@ import java.time.Duration; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.assertThat; @@ -44,4 +46,39 @@ class KafkaEnvelopeConsumerWorkerTest { assertThat(record.payload()).containsExactly(0x01, 0x02); }); } + + @Test + void pollsKafkaRecordsAndDispatchesBatchWhenProcessorSupportsIt() { + MockConsumer consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST); + TopicPartition partition = new TopicPartition("vehicle.raw", 0); + consumer.assign(List.of(partition)); + consumer.updateBeginningOffsets(Map.of(partition, 0L)); + consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 12L, "VIN001", new byte[]{0x01})); + consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 13L, "VIN002", new byte[]{0x02})); + AtomicInteger batchCalls = new AtomicInteger(); + EnvelopeConsumerProcessor processor = new EnvelopeConsumerProcessor( + "event-history", + new EnvelopeBatchIngestor() { + @Override + public EnvelopeIngestResult tryIngest(byte[] kafkaValue) { + throw new AssertionError("single-record ingest should not be used for a batch-capable processor"); + } + + @Override + public List tryIngestAll(List kafkaValues) { + batchCalls.incrementAndGet(); + assertThat(kafkaValues).hasSize(2); + return List.of( + EnvelopeIngestResult.stored("event-1", "VIN001"), + EnvelopeIngestResult.invalid("bad envelope")); + } + }, + record -> {}); + + int processed = new KafkaEnvelopeConsumerWorker( + consumer, Map.of("vehicle.raw", processor)).pollOnce(Duration.ZERO); + + assertThat(processed).isEqualTo(2); + assertThat(batchCalls).hasValue(1); + } } diff --git a/modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfigurationTest.java b/modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfigurationTest.java index fe7f4379..88b196a0 100644 --- a/modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfigurationTest.java +++ b/modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfigurationTest.java @@ -10,7 +10,7 @@ import static org.assertj.core.api.Assertions.assertThat; class SinkMqConsumerAutoConfigurationTest { private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() - .withUserConfiguration(SinkMqAutoConfiguration.class) + .withUserConfiguration(SinkMqAutoConfiguration.class, SinkMqConsumerAutoConfiguration.class) .withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class, () -> new EnvelopeConsumerProcessor( "vehicle-state", diff --git a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineEnvelopeRows.java b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineEnvelopeRows.java index 9322d071..3119a079 100644 --- a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineEnvelopeRows.java +++ b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineEnvelopeRows.java @@ -43,6 +43,7 @@ public final class TdengineEnvelopeRows { raw.getParseError(), raw.getPeer(), json(metadata), + parsedJson(envelope, metadata), protocol(envelope), vehicleKey(envelope, raw.getVehicleKey(), raw.getPhone()), firstNonBlank(raw.getVin(), envelope.getVin()), @@ -73,7 +74,6 @@ public final class TdengineEnvelopeRows { location.getStatusFlag(), totalMileage(envelope), rawUri, - json(envelope.getMetadataMap()), protocol(envelope), vehicleKey(envelope, envelope.getMetadataOrDefault("vehicle_key", ""), envelope.getMetadataOrDefault("phone", "")), @@ -188,6 +188,13 @@ public final class TdengineEnvelopeRows { return Double.parseDouble(value); } + private static String parsedJson(VehicleEnvelope envelope, Map metadata) { + if (envelope.hasRawArchive() && !envelope.getRawArchive().getParsedJson().isBlank()) { + return envelope.getRawArchive().getParsedJson(); + } + return firstNonBlank(metadata.get("parsedJson"), metadata.get("parsed_json")); + } + private static Double valueDouble(TelemetryField field) { String valueType = field.getValueType(); if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) { diff --git a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java index f2a4d756..3bea7022 100644 --- a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java +++ b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java @@ -6,11 +6,11 @@ import java.util.List; public final class TdengineHistoryQueries { private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, " - + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, " + + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json, " + "protocol, vehicle_key, vin, phone"; private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, " + "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, " - + "metadata_json, protocol, vehicle_key, vin, phone"; + + "protocol, vehicle_key, vin, phone"; private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, " + "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, " + "metadata_json, protocol, vehicle_key, vin, phone"; diff --git a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistorySchema.java b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistorySchema.java index 66714ed0..a8eca827 100644 --- a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistorySchema.java +++ b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistorySchema.java @@ -64,7 +64,8 @@ public final class TdengineHistorySchema { parse_status NCHAR(16), parse_error NCHAR(512), peer NCHAR(128), - metadata_json NCHAR(4096) + metadata_json NCHAR(4096), + parsed_json NCHAR(16374) ) TAGS ( protocol NCHAR(16), vehicle_key NCHAR(128), @@ -88,8 +89,7 @@ public final class TdengineHistorySchema { alarm_flag BIGINT, status_flag BIGINT, total_mileage_km DOUBLE, - raw_uri NCHAR(512), - metadata_json NCHAR(4096) + raw_uri NCHAR(512) ) TAGS ( protocol NCHAR(16), vehicle_key NCHAR(128), diff --git a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatements.java b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatements.java index cc4eea41..14eb1744 100644 --- a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatements.java +++ b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatements.java @@ -6,12 +6,12 @@ import java.util.Arrays; public final class TdengineHistoryStatements { private static final String RAW_FRAME_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, " - + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json"; - private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?"; + + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json"; + private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?"; private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, " - + "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, metadata_json"; - private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?"; + + "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri"; + private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?"; private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, " + "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json"; @@ -35,7 +35,7 @@ public final class TdengineHistoryStatements { values( row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(), row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(), - row.parseError(), row.peer(), row.metadataJson() + row.parseError(), row.peer(), row.metadataJson(), row.parsedJson() ) ); } @@ -49,7 +49,7 @@ public final class TdengineHistoryStatements { values( row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(), row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(), - row.totalMileageKm(), row.rawUri(), row.metadataJson() + row.totalMileageKm(), row.rawUri() ) ); } diff --git a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java index 8779971a..50efd057 100644 --- a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java +++ b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java @@ -157,6 +157,7 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader { rs.getString("parse_error"), rs.getString("peer"), rs.getString("metadata_json"), + rs.getString("parsed_json"), rs.getString("protocol"), rs.getString("vehicle_key"), rs.getString("vin"), @@ -179,7 +180,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader { rs.getLong("status_flag"), nullableDouble(rs, "total_mileage_km"), rs.getString("raw_uri"), - rs.getString("metadata_json"), rs.getString("protocol"), rs.getString("vehicle_key"), rs.getString("vin"), diff --git a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineLocationRow.java b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineLocationRow.java index 8a3d80cb..0b7f4567 100644 --- a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineLocationRow.java +++ b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineLocationRow.java @@ -16,7 +16,6 @@ public record TdengineLocationRow( long statusFlag, Double totalMileageKm, String rawUri, - String metadataJson, String protocol, String vehicleKey, String vin, diff --git a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineRawFrameRow.java b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineRawFrameRow.java index a9a473bd..a965bcc0 100644 --- a/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineRawFrameRow.java +++ b/modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineRawFrameRow.java @@ -16,6 +16,7 @@ public record TdengineRawFrameRow( String parseError, String peer, String metadataJson, + String parsedJson, String protocol, String vehicleKey, String vin, diff --git a/modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatementsTest.java b/modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatementsTest.java index 0381ec07..ccb22a66 100644 --- a/modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatementsTest.java +++ b/modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatementsTest.java @@ -27,6 +27,7 @@ class TdengineHistoryStatementsTest { "", "10.0.0.1:808", "{\"auth\":\"passed\"}", + "{\"body\":{\"speedKmh\":42.0}}", "JT808", "jt808:g7gps", "VIN123", @@ -39,13 +40,13 @@ class TdengineHistoryStatementsTest { .contains(" USING raw_frames TAGS ('JT808', 'jt808:g7gps', 'VIN123', '013800000000')"); assertThat(batch.insertSql()) .startsWith("INSERT INTO raw_jt808_") - .contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json)") - .endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + .contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json)") + .endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); assertThat(batch.values()) .containsExactly( row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(), row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(), - row.parseError(), row.peer(), row.metadataJson()); + row.parseError(), row.peer(), row.metadataJson(), row.parsedJson()); } @Test @@ -64,7 +65,6 @@ class TdengineHistoryStatementsTest { 3, null, "archive://gb32960/frame-1.bin", - "{}", "GB32960", "vin:O'HARE", "O'HARE", @@ -79,7 +79,7 @@ class TdengineHistoryStatementsTest { .containsExactly( row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(), row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(), - row.totalMileageKm(), row.rawUri(), row.metadataJson()); + row.totalMileageKm(), row.rawUri()); } @Test diff --git a/modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriterTest.java b/modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriterTest.java index 10f18269..1d20685f 100644 --- a/modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriterTest.java +++ b/modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriterTest.java @@ -74,7 +74,6 @@ class TdengineJdbcHistoryWriterTest { 3, null, "archive://gb32960/frame-1.bin", - "{}", "GB32960", "vin:VIN123", "VIN123", @@ -217,6 +216,7 @@ class TdengineJdbcHistoryWriterTest { parseError, "10.0.0.1:808", "{}", + "{\"frame\":\"" + frameId + "\"}", "JT808", "jt808:g7gps", "VIN123", diff --git a/pom.xml b/pom.xml index 2b6417af..c00e8abd 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ 2.2.0 1.13.6 1.42.1 + 1.2.5 1.12 1.1.3 3.8.4 @@ -306,6 +307,11 @@ kafka-clients ${kafka.version} + + org.eclipse.paho + org.eclipse.paho.client.mqttv3 + ${paho-mqtt.version} + org.fusesource.mqtt-client mqtt-client