shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: productionize raw history ingestion

Browse Source
This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
modules/inbound/inbound-mqtt/pom.xml
View File

@@ -9,7 +9,7 @@
</parent>
<artifactId>inbound-mqtt</artifactId>
<name>inbound-mqtt</name>
<description>MQTT 接入模块（支持多 endpoint，宇通等），Fusesource MQTT Client + 双向 TLS。</description>
<description>MQTT 接入模块（支持多 endpoint，宇通等），Eclipse Paho MQTT Client + 双向 TLS。</description>
<dependencies>
<dependency>
@@ -34,8 +34,8 @@
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.fusesource.mqtt-client</groupId>
<artifactId>mqtt-client</artifactId>
<groupId>org.eclipse.paho</groupId>
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>

262
modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttEndpointManager.java
View File

@@ -11,30 +11,33 @@ import com.lingniu.ingest.identity.VehicleIdentitySource;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import com.lingniu.ingest.inbound.mqtt.model.MqttPayload;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import org.fusesource.mqtt.client.Future;
import org.fusesource.mqtt.client.FutureConnection;
import org.fusesource.mqtt.client.MQTT;
import org.fusesource.mqtt.client.Message;
import org.fusesource.mqtt.client.QoS;
import org.fusesource.mqtt.client.Topic;
import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
import org.eclipse.paho.client.mqttv3.MqttAsyncClient;
import org.eclipse.paho.client.mqttv3.MqttCallbackExtended;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.MqttException;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
/**
* MQTT 多 endpoint 生命周期管理。
*
* <p>为每个 endpoint 创建一个 fusesource MQTT FutureConnection → 连接 → 订阅 → 接收循环 →
* <p>为每个 endpoint 创建一个 Paho 异步客户端 → 连接 → 订阅 → 注册回调 →
* 回调里解析 payload → {@link Dispatcher#dispatch(RawFrame)}。
*
* <p>每个 endpoint 在独立虚拟线程中阻塞接收，业务处理由 Dispatcher 侧的 Disruptor 承接。
* <p>MQTT 回调不做业务处理，虚拟线程由 Dispatcher 侧的 Disruptor 承接。
*/
public final class MqttEndpointManager implements AutoCloseable {
@@ -44,9 +47,8 @@ public final class MqttEndpointManager implements AutoCloseable {
private final MqttProfileRegistry profileRegistry;
private final VehicleIdentityResolver identityResolver;
private final Dispatcher dispatcher;
private final List<FutureConnection> connections = new ArrayList<>();
private final List<Thread> workers = new ArrayList<>();
private volatile boolean running;
private final List<MqttAsyncClient> clients = new ArrayList<>();
private boolean started;
public MqttEndpointManager(MqttInboundProperties props,
MqttProfileRegistry profileRegistry,
@@ -64,111 +66,78 @@ public final class MqttEndpointManager implements AutoCloseable {
this.dispatcher = dispatcher;
}
public void start() {
running = true;
log.info("mqtt endpoint manager starting, endpoints={}", props.getEndpoints().size());
public synchronized void start() {
if (started) {
return;
}
started = true;
if (props.getEndpoints().isEmpty()) {
log.warn("mqtt inbound enabled but no endpoints configured");
return;
}
log.info("mqtt inbound starting endpoints={}", props.getEndpoints().size());
for (MqttInboundProperties.Endpoint ep : props.getEndpoints()) {
Thread worker = Thread.ofVirtual()
.name("mqtt-endpoint-" + safeName(ep.getName()))
.start(() -> runEndpoint(ep));
workers.add(worker);
try {
// 多 endpoint 互不影响：单个连接初始化失败会派发 operationalError，但不会阻止其他 endpoint。
log.info("mqtt endpoint [{}] initializing uri={} topic={} profile={} qos={} cleanSession={} tlsHostnameVerification={}",
ep.getName(), ep.getUri(), ep.getTopic(), ep.getProfile(), ep.getQos(), ep.isCleanSession(),
ep.getTls() == null || ep.getTls().isHostnameVerificationEnabled());
MqttAsyncClient client = buildClient(ep);
clients.add(client);
connectAndSubscribe(client, ep);
} catch (Exception e) {
log.error("mqtt endpoint [{}] init failed, skipped", ep.getName(), e);
publishOperationalError(ep, "init", e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
}
}
}
private void runEndpoint(MqttInboundProperties.Endpoint ep) {
while (running && !Thread.currentThread().isInterrupted()) {
FutureConnection connection = null;
try {
log.info("mqtt endpoint [{}] initializing uri={} topic={} qos={} tls={}",
ep.getName(), ep.getUri(), ep.getTopic(), ep.getQos(), hasCustomTls(ep.getTls()));
MQTT mqtt = buildClient(ep);
connection = mqtt.futureConnection();
connections.add(connection);
log.info("mqtt endpoint [{}] connect sending", ep.getName());
connection.connect().await();
log.info("mqtt endpoint [{}] connected", ep.getName());
connection.subscribe(new Topic[]{new Topic(ep.getTopic(), mapQos(ep.getQos()))}).await();
log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
receiveLoop(ep, connection);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
} catch (Exception e) {
if (isPeerDisconnect(e)) {
log.warn("mqtt endpoint [{}] peer disconnected, reconnecting: {}", ep.getName(), e.getMessage());
} else {
log.error("mqtt endpoint [{}] connect/receive failed", ep.getName(), e);
publishOperationalError(ep, "connect",
private MqttAsyncClient buildClient(MqttInboundProperties.Endpoint ep) throws MqttException {
String clientId = ep.getClientId() == null || ep.getClientId().isBlank()
? "lingniu-" + UUID.randomUUID() : ep.getClientId();
return new MqttAsyncClient(pahoServerUri(ep.getUri()), clientId, new MemoryPersistence());
}
private void connectAndSubscribe(MqttAsyncClient client, MqttInboundProperties.Endpoint ep) throws MqttException {
client.setCallback(new MqttCallbackExtended() {
@Override
public void connectComplete(boolean reconnect, String serverURI) {
if (!reconnect) {
return;
}
try {
client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep));
log.info("mqtt endpoint [{}] re-subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
} catch (Exception e) {
log.error("mqtt endpoint [{}] re-subscribe failed topic={}", ep.getName(), ep.getTopic(), e);
publishOperationalError(ep, "subscribe",
e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
}
sleepBeforeReconnect(ep);
} finally {
if (connection != null) {
connections.remove(connection);
try {
connection.disconnect().await();
} catch (Exception ignored) {
// best-effort close
}
}
}
}
}
private MQTT buildClient(MqttInboundProperties.Endpoint ep) throws Exception {
MQTT mqtt = new MQTT();
mqtt.setHost(ep.getUri());
if (ep.getClientId() != null && !ep.getClientId().isBlank()) {
mqtt.setClientId(ep.getClientId());
}
mqtt.setCleanSession(ep.isCleanSession());
if (ep.getUsername() != null && !ep.getUsername().isBlank()) {
mqtt.setUserName(ep.getUsername());
mqtt.setPassword(ep.getPassword() == null ? "" : ep.getPassword());
}
mqtt.setConnectAttemptsMax(1);
mqtt.setReconnectAttemptsMax(0);
if (ep.getKeepAliveSeconds() > 0) {
mqtt.setKeepAlive((short) ep.getKeepAliveSeconds());
}
mqtt.setVersion("3.1.1");
if (hasCustomTls(ep.getTls())) {
mqtt.setSslContext(MqttTlsSupport.buildSslContext(ep.getTls()));
}
return mqtt;
}
private void receiveLoop(MqttInboundProperties.Endpoint ep, FutureConnection connection) throws Exception {
while (running && !Thread.currentThread().isInterrupted()) {
Future<Message> futureMessage = connection.receive();
Message message = futureMessage.await();
try {
message.ack();
onMessage(ep, message.getTopic(), message.getPayload());
} catch (Exception e) {
log.warn("mqtt endpoint [{}] message ack/dispatch failed topic={} len={}",
ep.getName(), message.getTopic(), message.getPayload() == null ? 0 : message.getPayload().length, e);
@Override
public void connectionLost(Throwable cause) {
log.warn("mqtt endpoint [{}] connection lost", ep.getName(), cause);
}
}
}
private boolean isPeerDisconnect(Exception e) {
Throwable current = e;
while (current != null) {
if (current instanceof EOFException) {
return true;
@Override
public void messageArrived(String topic, MqttMessage message) {
log.info("mqtt endpoint [{}] received topic={} bytes={}", ep.getName(), topic, message.getPayload().length);
onMessage(ep, topic, message.getPayload());
}
current = current.getCause();
}
return false;
}
private void sleepBeforeReconnect(MqttInboundProperties.Endpoint ep) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@Override
public void deliveryComplete(IMqttDeliveryToken token) {
// inbound only
}
});
MqttConnectOptions options = connectOptions(ep);
client.connect(options).waitForCompletion(timeoutMs(ep));
log.info("mqtt endpoint [{}] connected serverUri={} clientId={}",
ep.getName(), client.getServerURI(), client.getClientId());
client.subscribe(ep.getTopic(), mapQos(ep.getQos())).waitForCompletion(timeoutMs(ep));
log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
}
private void onMessage(MqttInboundProperties.Endpoint ep, String topic, byte[] payload) {
@@ -319,11 +288,58 @@ public final class MqttEndpointManager implements AutoCloseable {
return value.replace("\\", "\\\\").replace("\"", "\\\"");
}
private static QoS mapQos(int level) {
private static MqttConnectOptions connectOptions(MqttInboundProperties.Endpoint ep) {
MqttConnectOptions options = new MqttConnectOptions();
options.setAutomaticReconnect(true);
options.setCleanSession(ep.isCleanSession());
if (ep.getKeepAliveSeconds() > 0) {
options.setKeepAliveInterval(ep.getKeepAliveSeconds());
}
if (ep.getConnectionTimeoutSeconds() > 0) {
options.setConnectionTimeout(ep.getConnectionTimeoutSeconds());
}
if (hasText(ep.getUsername())) {
options.setUserName(ep.getUsername());
options.setPassword(ep.getPassword() == null ? new char[0] : ep.getPassword().toCharArray());
}
if (isSslUri(ep.getUri()) && hasCustomTls(ep.getTls())) {
options.setSocketFactory(MqttTlsSupport.socketFactory(ep.getTls()));
if (ep.getTls() != null && !ep.getTls().isHostnameVerificationEnabled()) {
options.setSSLHostnameVerifier((hostname, session) -> true);
}
}
return options;
}
private static String pahoServerUri(String value) {
URI uri = URI.create(value);
String scheme = uri.getScheme();
String pahoScheme = switch (scheme == null ? "" : scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> "ssl";
default -> "tcp";
};
int port = uri.getPort() > 0 ? uri.getPort() : defaultPort(scheme);
return pahoScheme + "://" + uri.getHost() + ":" + port;
}
private static int mapQos(int level) {
return switch (level) {
case 0 -> QoS.AT_MOST_ONCE;
case 2 -> QoS.EXACTLY_ONCE;
default -> QoS.AT_LEAST_ONCE;
case 0 -> 0;
case 2 -> 2;
default -> 1;
};
}
private static long timeoutMs(MqttInboundProperties.Endpoint ep) {
int seconds = ep.getConnectionTimeoutSeconds() > 0 ? ep.getConnectionTimeoutSeconds() : 10;
return TimeUnit.SECONDS.toMillis(seconds);
}
private static int defaultPort(String scheme) {
if (scheme == null) return 1883;
return switch (scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> 8883;
default -> 1883;
};
}
@@ -331,13 +347,6 @@ public final class MqttEndpointManager implements AutoCloseable {
return value != null && value.trim().length() == 17;
}
private static String safeName(String value) {
if (value == null || value.isBlank()) {
return "unnamed";
}
return value.replaceAll("[^a-zA-Z0-9_.-]", "_");
}
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}
private static boolean hasCustomTls(MqttInboundProperties.Tls tls) {
@@ -345,23 +354,30 @@ public final class MqttEndpointManager implements AutoCloseable {
|| (hasText(tls.getClientPem()) && hasText(tls.getClientKey())));
}
private static boolean isSslUri(String value) {
String scheme = URI.create(value).getScheme();
return scheme != null && switch (scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> true;
default -> false;
};
}
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
@Override
public void close() {
running = false;
for (FutureConnection connection : List.copyOf(connections)) {
for (MqttAsyncClient c : clients) {
try {
connection.disconnect().await();
if (c.isConnected()) {
c.disconnectForcibly(1000, 1000, false);
}
c.close(true);
} catch (Exception ignored) {
// best-effort
}
}
for (Thread worker : workers) {
worker.interrupt();
}
log.info("mqtt endpoint manager stopped, connections={}", connections.size());
log.info("mqtt endpoint manager stopped, clients={}", clients.size());
}
}

223
modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/client/MqttTlsSupport.java
View File

@@ -2,19 +2,31 @@ package com.lingniu.ingest.inbound.mqtt.client;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import java.io.InputStream;
import javax.net.ssl.X509ExtendedTrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyFactory;
import java.security.KeyStore;
import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.spec.PKCS8EncodedKeySpec;
import java.util.ArrayList;
import java.util.Base64;
@@ -25,93 +37,81 @@ public final class MqttTlsSupport {
private MqttTlsSupport() {}
public static SSLContext buildSslContext(MqttInboundProperties.Tls tls) {
public static SSLSocketFactory socketFactory(MqttInboundProperties.Tls tls) {
try {
TrustManagerFactory tmf = null;
KeyManagerFactory kmf = null;
KeyManager[] keyManagers = null;
TrustManager[] trustManagers = null;
if (tls != null && hasText(tls.getCaPem())) {
tmf = trustManagerFactory(Path.of(tls.getCaPem()));
trustManagers = trustManagerFactory(Path.of(tls.getCaPem())).getTrustManagers();
if (!tls.isHostnameVerificationEnabled()) {
trustManagers = withoutHostnameVerification(trustManagers);
}
}
if (tls != null && hasText(tls.getClientPem()) && hasText(tls.getClientKey())) {
kmf = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey()));
keyManagers = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey())).getKeyManagers();
}
SSLContext context = SSLContext.getInstance("TLSv1.2");
context.init(
kmf == null ? null : kmf.getKeyManagers(),
tmf == null ? null : tmf.getTrustManagers(),
new SecureRandom());
return context;
SSLContext context = SSLContext.getInstance("TLS");
context.init(keyManagers, trustManagers, null);
SSLSocketFactory factory = context.getSocketFactory();
if (tls != null && !tls.isHostnameVerificationEnabled()) {
return new HostnameVerificationDisablingSocketFactory(factory);
}
return factory;
} catch (Exception e) {
throw new IllegalStateException("mqtt tls config build failed", e);
}
}
private static TrustManagerFactory trustManagerFactory(Path caPath) throws Exception {
private static TrustManagerFactory trustManagerFactory(Path caPem) throws Exception {
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(null, null);
int i = 0;
for (Certificate cert : readCertificates(caPath)) {
for (Certificate cert : readCertificates(caPem)) {
trustStore.setCertificateEntry("ca-" + i++, cert);
}
TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore);
return tmf;
}
private static KeyManagerFactory keyManagerFactory(Path clientCertPath, Path clientKeyPath) throws Exception {
List<Certificate> chain = readCertificates(clientCertPath);
PrivateKey key = readPrivateKey(clientKeyPath);
private static KeyManagerFactory keyManagerFactory(Path clientPem, Path clientKey) throws Exception {
List<Certificate> chain = readCertificates(clientPem);
PrivateKey key = readPrivateKey(clientKey);
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
keyStore.load(null, null);
char[] password = new char[0];
keyStore.setCertificateEntry("client", chain.getFirst());
keyStore.setKeyEntry("clientKey", key, password, chain.toArray(Certificate[]::new));
KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX");
keyStore.setKeyEntry("mqtt-client", key, password, chain.toArray(Certificate[]::new));
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
kmf.init(keyStore, password);
return kmf;
}
private static List<Certificate> readCertificates(Path path) throws Exception {
private static List<Certificate> readCertificates(Path pem) throws Exception {
String text = Files.readString(pem, StandardCharsets.UTF_8);
List<Certificate> certificates = new ArrayList<>();
CertificateFactory factory = CertificateFactory.getInstance("X.509");
String text = readUtf8IfPossible(path);
if (text != null && text.contains("-----BEGIN CERTIFICATE-----")) {
List<Certificate> certificates = new ArrayList<>();
for (String block : pemBlocks(text, "CERTIFICATE")) {
certificates.add(factory.generateCertificate(
new java.io.ByteArrayInputStream(Base64.getMimeDecoder().decode(block))));
}
if (!certificates.isEmpty()) {
return certificates;
for (String block : pemBlocks(text, "CERTIFICATE")) {
try (ByteArrayInputStream in = new ByteArrayInputStream(Base64.getMimeDecoder().decode(block))) {
certificates.add(factory.generateCertificate(in));
}
}
try (InputStream in = Files.newInputStream(path)) {
Collection<? extends Certificate> certificates = factory.generateCertificates(in);
if (certificates.isEmpty()) {
throw new IllegalArgumentException("no certificate found in " + path);
}
return List.copyOf(certificates);
if (certificates.isEmpty()) {
throw new IllegalArgumentException("no certificate found in " + pem);
}
return certificates;
}
private static PrivateKey readPrivateKey(Path path) throws Exception {
String text = Files.readString(path, StandardCharsets.UTF_8);
private static PrivateKey readPrivateKey(Path pem) throws Exception {
String text = Files.readString(pem, StandardCharsets.UTF_8);
List<String> blocks = pemBlocks(text, "PRIVATE KEY");
if (blocks.isEmpty()) {
throw new IllegalArgumentException("no PKCS#8 private key found in " + path);
throw new IllegalArgumentException("no PKCS#8 private key found in " + pem);
}
byte[] der = Base64.getMimeDecoder().decode(blocks.getFirst());
PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der);
return KeyFactory.getInstance("RSA").generatePrivate(spec);
}
private static String readUtf8IfPossible(Path path) {
try {
return Files.readString(path, StandardCharsets.UTF_8);
} catch (Exception ignored) {
return null;
}
}
private static List<String> pemBlocks(String text, String type) {
String begin = "-----BEGIN " + type + "-----";
String end = "-----END " + type + "-----";
@@ -132,4 +132,129 @@ public final class MqttTlsSupport {
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
private static TrustManager[] withoutHostnameVerification(TrustManager[] managers) {
TrustManager[] wrapped = new TrustManager[managers.length];
for (int i = 0; i < managers.length; i++) {
TrustManager manager = managers[i];
if (manager instanceof X509TrustManager x509) {
wrapped[i] = new ChainOnlyTrustManager(x509);
} else {
wrapped[i] = manager;
}
}
return wrapped;
}
private static final class ChainOnlyTrustManager extends X509ExtendedTrustManager {
private final X509TrustManager delegate;
private ChainOnlyTrustManager(X509TrustManager delegate) {
this.delegate = delegate;
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket)
throws CertificateException {
if (delegate instanceof X509ExtendedTrustManager extended) {
extended.checkClientTrusted(chain, authType, socket);
} else {
delegate.checkClientTrusted(chain, authType);
}
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket)
throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
throws CertificateException {
if (delegate instanceof X509ExtendedTrustManager extended) {
extended.checkClientTrusted(chain, authType, engine);
} else {
delegate.checkClientTrusted(chain, authType);
}
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
delegate.checkClientTrusted(chain, authType);
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
delegate.checkServerTrusted(chain, authType);
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return delegate.getAcceptedIssuers();
}
}
private static final class HostnameVerificationDisablingSocketFactory extends SSLSocketFactory {
private final SSLSocketFactory delegate;
private HostnameVerificationDisablingSocketFactory(SSLSocketFactory delegate) {
this.delegate = delegate;
}
@Override
public String[] getDefaultCipherSuites() {
return delegate.getDefaultCipherSuites();
}
@Override
public String[] getSupportedCipherSuites() {
return delegate.getSupportedCipherSuites();
}
@Override
public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException {
return prepare(delegate.createSocket(socket, host, port, autoClose));
}
@Override
public Socket createSocket() throws IOException {
return prepare(delegate.createSocket());
}
@Override
public Socket createSocket(String host, int port) throws IOException {
return prepare(delegate.createSocket(host, port));
}
@Override
public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException {
return prepare(delegate.createSocket(host, port, localHost, localPort));
}
@Override
public Socket createSocket(InetAddress host, int port) throws IOException {
return prepare(delegate.createSocket(host, port));
}
@Override
public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException {
return prepare(delegate.createSocket(address, port, localAddress, localPort));
}
private static Socket prepare(Socket socket) {
if (socket instanceof SSLSocket sslSocket) {
SSLParameters parameters = sslSocket.getSSLParameters();
parameters.setEndpointIdentificationAlgorithm(null);
sslSocket.setSSLParameters(parameters);
}
return socket;
}
}
}

152
modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundAutoConfiguration.java
View File

@@ -10,19 +10,20 @@ import com.lingniu.ingest.inbound.mqtt.parser.MqttPayloadParser;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfile;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import com.lingniu.ingest.inbound.mqtt.profile.YutongMqttProfile;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import java.util.ArrayList;
import java.util.List;
@AutoConfiguration
@EnableConfigurationProperties(MqttInboundProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.mqtt", name = "enabled", havingValue = "true")
@Conditional(MqttInboundEnabledCondition.class)
public class MqttInboundAutoConfiguration {
@Bean
@@ -61,25 +62,144 @@ public class MqttInboundAutoConfiguration {
return new MqttRealtimeHandler(profileRegistry);
}
@Bean(destroyMethod = "close")
@Bean(initMethod = "start", destroyMethod = "close")
@Lazy(false)
@ConditionalOnMissingBean
public MqttEndpointManager mqttEndpointManager(MqttInboundProperties props,
MqttProfileRegistry profileRegistry,
VehicleIdentityResolver identityResolver,
Dispatcher dispatcher) {
Dispatcher dispatcher,
Environment environment) {
applyLegacySingleEndpointFallback(props, environment);
return new MqttEndpointManager(props, profileRegistry, identityResolver, dispatcher);
}
/**
* Spring Boot 启动完成后触发 MQTT 连接，避免与 Netty / Kafka 启动竞争。
*/
@Bean
public ApplicationRunner mqttStartupRunner(MqttEndpointManager manager) {
return new ApplicationRunner() {
@Override
public void run(ApplicationArguments args) {
manager.start();
private static void applyLegacySingleEndpointFallback(MqttInboundProperties props, Environment env) {
if (!props.getEndpoints().isEmpty()) {
return;
}
String uri = firstText(env,
"lingniu.ingest.mqtt.endpoints[0].uri",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_URI",
"MQTT_URI",
"YUTONG_MQTT_URI");
if (uri.isBlank() || uri.contains("${")) {
return;
}
MqttInboundProperties.Endpoint endpoint = new MqttInboundProperties.Endpoint();
endpoint.setName(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].name",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_NAME",
"MQTT_ENDPOINT_NAME",
"YUTONG_MQTT_ENDPOINT_NAME"), "yutong"));
endpoint.setUri(uri);
endpoint.setTopic(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].topic",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TOPIC",
"MQTT_TOPIC",
"YUTONG_MQTT_TOPIC"), "#"));
endpoint.setQos(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].qos",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_QOS",
"MQTT_QOS",
"YUTONG_MQTT_QOS"), 1));
endpoint.setClientId(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].client-id",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLIENT_ID",
"MQTT_CLIENT_ID",
"YUTONG_MQTT_CLIENT_ID"));
endpoint.setUsername(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].username",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_USERNAME",
"MQTT_USERNAME",
"YUTONG_MQTT_USERNAME"));
endpoint.setPassword(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].password",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_PASSWORD",
"MQTT_PASSWORD",
"YUTONG_MQTT_PASSWORD"));
endpoint.setCleanSession(booleanValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].clean-session",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CLEAN_SESSION",
"MQTT_CLEAN_SESSION",
"YUTONG_MQTT_CLEAN_SESSION"), false));
endpoint.setKeepAliveSeconds(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].keep-alive-seconds",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_KEEP_ALIVE_SECONDS",
"MQTT_KEEP_ALIVE_SECONDS",
"YUTONG_MQTT_KEEP_ALIVE_SECONDS"), 20));
endpoint.setConnectionTimeoutSeconds(intValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].connection-timeout-seconds",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_CONNECTION_TIMEOUT_SECONDS",
"MQTT_CONNECTION_TIMEOUT_SECONDS",
"YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS"), 10));
endpoint.setProfile(defaultText(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].profile",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_PROFILE",
"MQTT_PROFILE",
"YUTONG_MQTT_PROFILE"), "yutong"));
MqttInboundProperties.Tls tls = endpoint.getTls();
tls.setCaPem(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.ca-pem",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CA_PEM",
"MQTT_TLS_CA_PEM",
"YUTONG_MQTT_TLS_CA_PEM"));
tls.setClientPem(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.client-pem",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_PEM",
"MQTT_TLS_CLIENT_PEM",
"YUTONG_MQTT_TLS_CLIENT_PEM"));
tls.setClientKey(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.client-key",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_CLIENT_KEY",
"MQTT_TLS_CLIENT_KEY",
"YUTONG_MQTT_TLS_CLIENT_KEY"));
tls.setHostnameVerificationEnabled(booleanValue(firstText(env,
"lingniu.ingest.mqtt.endpoints[0].tls.hostname-verification-enabled",
"LINGNIU_INGEST_MQTT_ENDPOINTS_0_TLS_HOSTNAME_VERIFICATION_ENABLED",
"MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED",
"YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED"), true));
props.setEndpoints(new ArrayList<>(List.of(endpoint)));
}
private static String firstText(Environment env, String... keys) {
for (String key : keys) {
String value = env.getProperty(key);
if (value != null && !value.isBlank() && !value.contains("${")) {
return value;
}
};
}
return "";
}
private static String defaultText(String value, String fallback) {
return value == null || value.isBlank() ? fallback : value;
}
private static int intValue(String value, int fallback) {
if (value == null || value.isBlank()) {
return fallback;
}
try {
return Integer.parseInt(value.trim());
} catch (NumberFormatException ex) {
return fallback;
}
}
private static boolean booleanValue(String value, boolean fallback) {
if (value == null || value.isBlank()) {
return fallback;
}
String normalized = value.trim().toLowerCase();
if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized)
|| "on".equals(normalized)) {
return true;
}
if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized)
|| "off".equals(normalized)) {
return false;
}
return fallback;
}
}

41
modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundEnabledCondition.java Normal file
View File

@@ -0,0 +1,41 @@
package com.lingniu.ingest.inbound.mqtt.config;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.env.Environment;
import org.springframework.core.type.AnnotatedTypeMetadata;
final class MqttInboundEnabledCondition implements Condition {
@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
Environment env = context.getEnvironment();
Boolean direct = parseBoolean(env.getProperty("lingniu.ingest.mqtt.enabled"));
if (direct != null) {
return direct;
}
return isTrue(env.getProperty("MQTT_ENABLED"))
|| isTrue(env.getProperty("YUTONG_MQTT_ENABLED"));
}
private static boolean isTrue(String value) {
Boolean parsed = parseBoolean(value);
return parsed != null && parsed;
}
private static Boolean parseBoolean(String value) {
if (value == null || value.isBlank() || value.contains("${")) {
return null;
}
String normalized = value.trim().toLowerCase();
if ("true".equals(normalized) || "1".equals(normalized) || "yes".equals(normalized)
|| "on".equals(normalized)) {
return true;
}
if ("false".equals(normalized) || "0".equals(normalized) || "no".equals(normalized)
|| "off".equals(normalized)) {
return false;
}
return null;
}
}

4
modules/inbound/inbound-mqtt/src/main/java/com/lingniu/ingest/inbound/mqtt/config/MqttInboundProperties.java
View File

@@ -82,6 +82,8 @@ public class MqttInboundProperties {
public String getClientKey() { return clientKey; }
public void setClientKey(String clientKey) { this.clientKey = clientKey; }
public boolean isHostnameVerificationEnabled() { return hostnameVerificationEnabled; }
public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) { this.hostnameVerificationEnabled = hostnameVerificationEnabled; }
public void setHostnameVerificationEnabled(boolean hostnameVerificationEnabled) {
this.hostnameVerificationEnabled = hostnameVerificationEnabled;
}
}
}

4
modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttEndpointManagerTest.java
View File

@@ -172,7 +172,7 @@ class MqttEndpointManagerTest {
.containsEntry("endpoint", "bad-endpoint")
.containsEntry("profile", "yutong")
.containsEntry("operationalError", "true")
.containsEntry("phase", "connect")
.containsEntry("phase", "init")
.containsEntry("identityResolved", "false")
.containsEntry("identitySource", "UNKNOWN");
});
@@ -184,7 +184,7 @@ class MqttEndpointManagerTest {
.containsEntry("operationalError", "true")
.containsEntry("endpoint", "bad-endpoint")
.containsEntry("profile", "yutong")
.containsEntry("phase", "connect");
.containsEntry("phase", "init");
});
batchExecutor.close();

24
modules/inbound/inbound-mqtt/src/test/java/com/lingniu/ingest/inbound/mqtt/MqttTlsSupportTest.java
View File

@@ -5,11 +5,10 @@ import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import javax.net.ssl.SSLSocketFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.net.ssl.SSLContext;
import static org.assertj.core.api.Assertions.assertThat;
class MqttTlsSupportTest {
@@ -24,9 +23,9 @@ class MqttTlsSupportTest {
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
tls.setCaPem(ca.toString());
SSLContext config = MqttTlsSupport.buildSslContext(tls);
SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
assertThat(factory).isNotNull();
}
@Test
@@ -42,22 +41,9 @@ class MqttTlsSupportTest {
tls.setClientPem(cert.toString());
tls.setClientKey(key.toString());
SSLContext config = MqttTlsSupport.buildSslContext(tls);
SSLSocketFactory factory = MqttTlsSupport.socketFactory(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
}
@Test
void hostnameVerificationFlagDoesNotBlockSslContextCreation() throws Exception {
Path ca = tempDir.resolve("ca.pem");
Files.writeString(ca, CERT);
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
tls.setCaPem(ca.toString());
tls.setHostnameVerificationEnabled(false);
SSLContext config = MqttTlsSupport.buildSslContext(tls);
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
assertThat(factory).isNotNull();
}
private static final String CERT = """

24
modules/inbound/inbound-xinda-push/pom.xml
View File

@@ -41,6 +41,30 @@
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>

19
modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushClient.java
View File

@@ -6,6 +6,7 @@ import com.gps31.push.netty.PushClient;
import com.gps31.push.netty.PushMsg;
import com.gps31.push.netty.client.TcpClient;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.pipeline.RawFrame;
import com.lingniu.ingest.core.dispatcher.Dispatcher;
import com.lingniu.ingest.identity.InMemoryVehicleIdentityService;
@@ -220,6 +221,7 @@ public class XindaPushClient extends PushClient {
meta.put("parseError", "true");
meta.put("parseErrorMessage", parsed.errorMessage());
}
meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(payload.command(), json, parsed));
dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH,
payload.commandCode(),
@@ -247,6 +249,8 @@ public class XindaPushClient extends PushClient {
meta.put("operationalError", "true");
meta.put("phase", phase == null ? "" : phase);
meta.put("reason", reason == null ? "" : reason);
meta.put(RawArchiveKeys.META_PARSED_JSON,
parsedJson(payload.command(), payload.json(), new JsonParseResult(json, false, "")));
dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH,
payload.commandCode(),
@@ -271,6 +275,8 @@ public class XindaPushClient extends PushClient {
meta.put("operationalError", "true");
meta.put("phase", "message");
meta.put("reason", reason == null ? "" : reason);
JsonParseResult parsed = parseForMetadata(json);
meta.put(RawArchiveKeys.META_PARSED_JSON, parsedJson(cmd, json, parsed));
dispatcher.dispatch(new RawFrame(
ProtocolId.XINDA_PUSH,
0,
@@ -388,6 +394,19 @@ public class XindaPushClient extends PushClient {
}
}
private static String parsedJson(String command, String rawJson, JsonParseResult parsed) {
JSONObject root = new JSONObject(true);
root.put("command", command == null ? "" : command);
root.put("parseError", parsed != null && parsed.parseError());
if (parsed != null && parsed.parseError()) {
root.put("parseErrorMessage", parsed.errorMessage() == null ? "" : parsed.errorMessage());
root.put("rawJson", rawJson == null ? "" : rawJson);
} else {
root.put("body", parsed == null || parsed.json() == null ? new JSONObject() : parsed.json());
}
return root.toJSONString();
}
private record JsonParseResult(JSONObject json, boolean parseError, String errorMessage) {}
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}

1
modules/inbound/inbound-xinda-push/src/main/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapper.java
View File

@@ -185,7 +185,6 @@ public final class XindaPushEventMapper implements EventMapper<XindaPushMessage>
Map<String, String> out = new java.util.LinkedHashMap<>();
out.put("source", "xinda-push");
out.put("cmd", message.command());
out.put("rawJson", message.json());
out.put("vin", normalizedVin(identity));
out.put("externalVin", firstNonBlank(json, "vin"));
out.put("phone", phone(json));

3
modules/inbound/inbound-xinda-push/src/test/java/com/lingniu/ingest/inbound/xinda/XindaPushEventMapperTest.java
View File

@@ -37,7 +37,8 @@ class XindaPushEventMapperTest {
.containsEntry("externalVin", "LNVIN000000000101")
.containsEntry("phone", "13800138000")
.containsEntry("deviceId", "XD-101")
.containsEntry("plateNo", "粤B10101");
.containsEntry("plateNo", "粤B10101")
.doesNotContainKey("rawJson");
}
@Test