feat: productionize mqtt ingress and trim history APIs
Some checks failed
ci/woodpecker/push/woodpecker Pipeline was canceled

This commit is contained in:
lingniu
2026-06-30 01:28:51 +08:00
parent 1e2e59bddc
commit bf6728041b
15 changed files with 554 additions and 158 deletions

View File

@@ -9,7 +9,7 @@
</parent>
<artifactId>inbound-mqtt</artifactId>
<name>inbound-mqtt</name>
<description>MQTT 接入模块(支持多 endpoint宇通等HiveMQ MQTT Client + 双向 TLS。</description>
<description>MQTT 接入模块(支持多 endpoint宇通等Fusesource MQTT Client + 双向 TLS。</description>
<dependencies>
<dependency>
@@ -34,8 +34,8 @@
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.hivemq</groupId>
<artifactId>hivemq-mqtt-client</artifactId>
<groupId>org.fusesource.mqtt-client</groupId>
<artifactId>mqtt-client</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>

View File

@@ -1,9 +1,5 @@
package com.lingniu.ingest.inbound.mqtt.client;
import com.hivemq.client.mqtt.datatypes.MqttQos;
import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient;
import com.hivemq.client.mqtt.mqtt3.Mqtt3Client;
import com.hivemq.client.mqtt.mqtt3.message.connect.connack.Mqtt3ConnAck;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.pipeline.RawFrame;
import com.lingniu.ingest.core.dispatcher.Dispatcher;
@@ -15,27 +11,30 @@ import com.lingniu.ingest.identity.VehicleIdentitySource;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import com.lingniu.ingest.inbound.mqtt.model.MqttPayload;
import com.lingniu.ingest.inbound.mqtt.profile.MqttProfileRegistry;
import org.fusesource.mqtt.client.Future;
import org.fusesource.mqtt.client.FutureConnection;
import org.fusesource.mqtt.client.MQTT;
import org.fusesource.mqtt.client.Message;
import org.fusesource.mqtt.client.QoS;
import org.fusesource.mqtt.client.Topic;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URI;
import java.io.EOFException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
/**
* MQTT 多 endpoint 生命周期管理。
*
* <p>为每个 endpoint 创建一个 HiveMQ 异步客户端 → 连接 → 订阅 → 注册回调
* <p>为每个 endpoint 创建一个 fusesource MQTT FutureConnection → 连接 → 订阅 → 接收循环
* 回调里解析 payload → {@link Dispatcher#dispatch(RawFrame)}。
*
* <p>Netty EventLoop 回调不做业务处理,虚拟线程由 Dispatcher 侧的 Disruptor 承接。
* <p>每个 endpoint 在独立虚拟线程中阻塞接收,业务处理由 Dispatcher 侧的 Disruptor 承接。
*/
public final class MqttEndpointManager implements AutoCloseable {
@@ -45,7 +44,9 @@ public final class MqttEndpointManager implements AutoCloseable {
private final MqttProfileRegistry profileRegistry;
private final VehicleIdentityResolver identityResolver;
private final Dispatcher dispatcher;
private final List<Mqtt3AsyncClient> clients = new ArrayList<>();
private final List<FutureConnection> connections = new ArrayList<>();
private final List<Thread> workers = new ArrayList<>();
private volatile boolean running;
public MqttEndpointManager(MqttInboundProperties props,
MqttProfileRegistry profileRegistry,
@@ -64,90 +65,110 @@ public final class MqttEndpointManager implements AutoCloseable {
}
public void start() {
running = true;
log.info("mqtt endpoint manager starting, endpoints={}", props.getEndpoints().size());
for (MqttInboundProperties.Endpoint ep : props.getEndpoints()) {
Thread worker = Thread.ofVirtual()
.name("mqtt-endpoint-" + safeName(ep.getName()))
.start(() -> runEndpoint(ep));
workers.add(worker);
}
}
private void runEndpoint(MqttInboundProperties.Endpoint ep) {
while (running && !Thread.currentThread().isInterrupted()) {
FutureConnection connection = null;
try {
// 多 endpoint 互不影响:单个连接初始化失败会派发 operationalError但不会阻止其他 endpoint。
log.info("mqtt endpoint [{}] initializing uri={} topic={} qos={} tls={}",
ep.getName(), ep.getUri(), ep.getTopic(), ep.getQos(), hasCustomTls(ep.getTls()));
Mqtt3AsyncClient client = buildClient(ep);
log.info("mqtt endpoint [{}] client built", ep.getName());
clients.add(client);
connectAndSubscribe(client, ep);
} catch (Exception e) {
log.error("mqtt endpoint [{}] init failed, skipped", ep.getName(), e);
publishOperationalError(ep, "init", e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
}
}
}
private Mqtt3AsyncClient buildClient(MqttInboundProperties.Endpoint ep) {
URI uri = URI.create(ep.getUri());
int port = uri.getPort() > 0 ? uri.getPort() : defaultPort(uri.getScheme());
var builder = Mqtt3Client.builder()
.identifier(ep.getClientId() == null || ep.getClientId().isBlank()
? "lingniu-" + UUID.randomUUID() : ep.getClientId())
.serverHost(uri.getHost())
.serverPort(port)
.automaticReconnectWithDefaultConfig();
if (ep.getConnectionTimeoutSeconds() > 0) {
builder = builder.transportConfig()
.serverHost(uri.getHost())
.serverPort(port)
.socketConnectTimeout(ep.getConnectionTimeoutSeconds(), TimeUnit.SECONDS)
.mqttConnectTimeout(ep.getConnectionTimeoutSeconds(), TimeUnit.SECONDS)
.applyTransportConfig();
}
if ("ssl".equalsIgnoreCase(uri.getScheme()) || "tls".equalsIgnoreCase(uri.getScheme())
|| "mqtts".equalsIgnoreCase(uri.getScheme())) {
if (hasCustomTls(ep.getTls())) {
builder = builder.sslConfig(MqttTlsSupport.build(ep.getTls()));
} else {
builder = builder.sslWithDefaultConfig();
}
}
return builder.buildAsync();
}
private void connectAndSubscribe(Mqtt3AsyncClient client, MqttInboundProperties.Endpoint ep) {
var connectBuilder = client.connectWith()
.cleanSession(ep.isCleanSession());
if (ep.getKeepAliveSeconds() > 0) {
connectBuilder = connectBuilder.keepAlive(ep.getKeepAliveSeconds());
}
if (ep.getUsername() != null && !ep.getUsername().isBlank()) {
connectBuilder = connectBuilder.simpleAuth()
.username(ep.getUsername())
.password(ep.getPassword() == null ? new byte[0] : ep.getPassword().getBytes())
.applySimpleAuth();
}
log.info("mqtt endpoint [{}] connect sending", ep.getName());
CompletableFuture<Mqtt3ConnAck> timeoutProbe = connectBuilder.send()
.orTimeout(Math.max(15, ep.getConnectionTimeoutSeconds() + 5L), TimeUnit.SECONDS);
timeoutProbe.whenComplete((ack, err) -> {
if (err != null) {
log.error("mqtt endpoint [{}] connect failed", ep.getName(), err);
publishOperationalError(ep, "connect", err.getMessage() == null ? err.getClass().getSimpleName() : err.getMessage());
MQTT mqtt = buildClient(ep);
connection = mqtt.futureConnection();
connections.add(connection);
log.info("mqtt endpoint [{}] connect sending", ep.getName());
connection.connect().await();
log.info("mqtt endpoint [{}] connected", ep.getName());
connection.subscribe(new Topic[]{new Topic(ep.getTopic(), mapQos(ep.getQos()))}).await();
log.info("mqtt endpoint [{}] subscribed topic={} qos={}", ep.getName(), ep.getTopic(), ep.getQos());
receiveLoop(ep, connection);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
} catch (Exception e) {
if (isPeerDisconnect(e)) {
log.warn("mqtt endpoint [{}] peer disconnected, reconnecting: {}", ep.getName(), e.getMessage());
} else {
log.error("mqtt endpoint [{}] connect/receive failed", ep.getName(), e);
publishOperationalError(ep, "connect",
e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage());
}
sleepBeforeReconnect(ep);
} finally {
if (connection != null) {
connections.remove(connection);
try {
connection.disconnect().await();
} catch (Exception ignored) {
// best-effort close
}
}
}
log.info("mqtt endpoint [{}] connected code={}", ep.getName(), ack.getReturnCode());
client.subscribeWith()
.topicFilter(ep.getTopic())
.qos(mapQos(ep.getQos()))
.callback(publish -> onMessage(ep, publish.getTopic().toString(), publish.getPayloadAsBytes()))
.send()
.whenComplete((subAck, subErr) -> {
if (subErr != null) {
log.error("mqtt endpoint [{}] subscribe failed topic={}", ep.getName(), ep.getTopic(), subErr);
publishOperationalError(ep, "subscribe",
subErr.getMessage() == null ? subErr.getClass().getSimpleName() : subErr.getMessage());
} else {
log.info("mqtt endpoint [{}] subscribed topic={} qos={}",
ep.getName(), ep.getTopic(), ep.getQos());
}
});
});
}
}
private MQTT buildClient(MqttInboundProperties.Endpoint ep) throws Exception {
MQTT mqtt = new MQTT();
mqtt.setHost(ep.getUri());
if (ep.getClientId() != null && !ep.getClientId().isBlank()) {
mqtt.setClientId(ep.getClientId());
}
mqtt.setCleanSession(ep.isCleanSession());
if (ep.getUsername() != null && !ep.getUsername().isBlank()) {
mqtt.setUserName(ep.getUsername());
mqtt.setPassword(ep.getPassword() == null ? "" : ep.getPassword());
}
mqtt.setConnectAttemptsMax(1);
mqtt.setReconnectAttemptsMax(0);
if (ep.getKeepAliveSeconds() > 0) {
mqtt.setKeepAlive((short) ep.getKeepAliveSeconds());
}
mqtt.setVersion("3.1.1");
if (hasCustomTls(ep.getTls())) {
mqtt.setSslContext(MqttTlsSupport.buildSslContext(ep.getTls()));
}
return mqtt;
}
private void receiveLoop(MqttInboundProperties.Endpoint ep, FutureConnection connection) throws Exception {
while (running && !Thread.currentThread().isInterrupted()) {
Future<Message> futureMessage = connection.receive();
Message message = futureMessage.await();
try {
message.ack();
onMessage(ep, message.getTopic(), message.getPayload());
} catch (Exception e) {
log.warn("mqtt endpoint [{}] message ack/dispatch failed topic={} len={}",
ep.getName(), message.getTopic(), message.getPayload() == null ? 0 : message.getPayload().length, e);
}
}
}
private boolean isPeerDisconnect(Exception e) {
Throwable current = e;
while (current != null) {
if (current instanceof EOFException) {
return true;
}
current = current.getCause();
}
return false;
}
private void sleepBeforeReconnect(MqttInboundProperties.Endpoint ep) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
private void onMessage(MqttInboundProperties.Endpoint ep, String topic, byte[] payload) {
@@ -298,19 +319,11 @@ public final class MqttEndpointManager implements AutoCloseable {
return value.replace("\\", "\\\\").replace("\"", "\\\"");
}
private static MqttQos mapQos(int level) {
private static QoS mapQos(int level) {
return switch (level) {
case 0 -> MqttQos.AT_MOST_ONCE;
case 2 -> MqttQos.EXACTLY_ONCE;
default -> MqttQos.AT_LEAST_ONCE;
};
}
private static int defaultPort(String scheme) {
if (scheme == null) return 1883;
return switch (scheme.toLowerCase()) {
case "ssl", "tls", "mqtts" -> 8883;
default -> 1883;
case 0 -> QoS.AT_MOST_ONCE;
case 2 -> QoS.EXACTLY_ONCE;
default -> QoS.AT_LEAST_ONCE;
};
}
@@ -318,6 +331,13 @@ public final class MqttEndpointManager implements AutoCloseable {
return value != null && value.trim().length() == 17;
}
private static String safeName(String value) {
if (value == null || value.isBlank()) {
return "unnamed";
}
return value.replaceAll("[^a-zA-Z0-9_.-]", "_");
}
private record IdentityResolution(VehicleIdentity identity, String errorMessage) {}
private static boolean hasCustomTls(MqttInboundProperties.Tls tls) {
@@ -331,13 +351,17 @@ public final class MqttEndpointManager implements AutoCloseable {
@Override
public void close() {
for (Mqtt3AsyncClient c : clients) {
running = false;
for (FutureConnection connection : List.copyOf(connections)) {
try {
c.disconnect();
connection.disconnect().await();
} catch (Exception ignored) {
// best-effort
}
}
log.info("mqtt endpoint manager stopped, clients={}", clients.size());
for (Thread worker : workers) {
worker.interrupt();
}
log.info("mqtt endpoint manager stopped, connections={}", connections.size());
}
}

View File

@@ -1,18 +1,18 @@
package com.lingniu.ingest.inbound.mqtt.client;
import com.hivemq.client.mqtt.MqttClientSslConfig;
import com.hivemq.client.mqtt.MqttClientSslConfigBuilder;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyFactory;
import java.security.KeyStore;
import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import java.security.cert.CertificateFactory;
import java.security.spec.PKCS8EncodedKeySpec;
@@ -25,74 +25,93 @@ public final class MqttTlsSupport {
private MqttTlsSupport() {}
public static MqttClientSslConfig build(MqttInboundProperties.Tls tls) {
public static SSLContext buildSslContext(MqttInboundProperties.Tls tls) {
try {
MqttClientSslConfigBuilder builder = MqttClientSslConfig.builder();
TrustManagerFactory tmf = null;
KeyManagerFactory kmf = null;
if (tls != null && hasText(tls.getCaPem())) {
builder.trustManagerFactory(trustManagerFactory(Path.of(tls.getCaPem())));
tmf = trustManagerFactory(Path.of(tls.getCaPem()));
}
if (tls != null && hasText(tls.getClientPem()) && hasText(tls.getClientKey())) {
builder.keyManagerFactory(keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey())));
kmf = keyManagerFactory(Path.of(tls.getClientPem()), Path.of(tls.getClientKey()));
}
if (tls != null && !tls.isHostnameVerificationEnabled()) {
builder.hostnameVerifier((hostname, session) -> true);
}
return builder.build();
SSLContext context = SSLContext.getInstance("TLSv1.2");
context.init(
kmf == null ? null : kmf.getKeyManagers(),
tmf == null ? null : tmf.getTrustManagers(),
new SecureRandom());
return context;
} catch (Exception e) {
throw new IllegalStateException("mqtt tls config build failed", e);
}
}
private static TrustManagerFactory trustManagerFactory(Path caPem) throws Exception {
private static TrustManagerFactory trustManagerFactory(Path caPath) throws Exception {
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(null, null);
int i = 0;
for (Certificate cert : readCertificates(caPem)) {
for (Certificate cert : readCertificates(caPath)) {
trustStore.setCertificateEntry("ca-" + i++, cert);
}
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
tmf.init(trustStore);
return tmf;
}
private static KeyManagerFactory keyManagerFactory(Path clientPem, Path clientKey) throws Exception {
List<Certificate> chain = readCertificates(clientPem);
PrivateKey key = readPrivateKey(clientKey);
private static KeyManagerFactory keyManagerFactory(Path clientCertPath, Path clientKeyPath) throws Exception {
List<Certificate> chain = readCertificates(clientCertPath);
PrivateKey key = readPrivateKey(clientKeyPath);
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
keyStore.load(null, null);
char[] password = new char[0];
keyStore.setKeyEntry("mqtt-client", key, password, chain.toArray(Certificate[]::new));
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
keyStore.setCertificateEntry("client", chain.getFirst());
keyStore.setKeyEntry("clientKey", key, password, chain.toArray(Certificate[]::new));
KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX");
kmf.init(keyStore, password);
return kmf;
}
private static List<Certificate> readCertificates(Path pem) throws Exception {
String text = Files.readString(pem, StandardCharsets.UTF_8);
List<Certificate> certificates = new ArrayList<>();
private static List<Certificate> readCertificates(Path path) throws Exception {
CertificateFactory factory = CertificateFactory.getInstance("X.509");
for (String block : pemBlocks(text, "CERTIFICATE")) {
try (ByteArrayInputStream in = new ByteArrayInputStream(Base64.getMimeDecoder().decode(block))) {
certificates.add(factory.generateCertificate(in));
String text = readUtf8IfPossible(path);
if (text != null && text.contains("-----BEGIN CERTIFICATE-----")) {
List<Certificate> certificates = new ArrayList<>();
for (String block : pemBlocks(text, "CERTIFICATE")) {
certificates.add(factory.generateCertificate(
new java.io.ByteArrayInputStream(Base64.getMimeDecoder().decode(block))));
}
if (!certificates.isEmpty()) {
return certificates;
}
}
if (certificates.isEmpty()) {
throw new IllegalArgumentException("no certificate found in " + pem);
try (InputStream in = Files.newInputStream(path)) {
Collection<? extends Certificate> certificates = factory.generateCertificates(in);
if (certificates.isEmpty()) {
throw new IllegalArgumentException("no certificate found in " + path);
}
return List.copyOf(certificates);
}
return certificates;
}
private static PrivateKey readPrivateKey(Path pem) throws Exception {
String text = Files.readString(pem, StandardCharsets.UTF_8);
private static PrivateKey readPrivateKey(Path path) throws Exception {
String text = Files.readString(path, StandardCharsets.UTF_8);
List<String> blocks = pemBlocks(text, "PRIVATE KEY");
if (blocks.isEmpty()) {
throw new IllegalArgumentException("no PKCS#8 private key found in " + pem);
throw new IllegalArgumentException("no PKCS#8 private key found in " + path);
}
byte[] der = Base64.getMimeDecoder().decode(blocks.getFirst());
PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der);
return KeyFactory.getInstance("RSA").generatePrivate(spec);
}
private static String readUtf8IfPossible(Path path) {
try {
return Files.readString(path, StandardCharsets.UTF_8);
} catch (Exception ignored) {
return null;
}
}
private static List<String> pemBlocks(String text, String type) {
String begin = "-----BEGIN " + type + "-----";
String end = "-----END " + type + "-----";

View File

@@ -172,7 +172,7 @@ class MqttEndpointManagerTest {
.containsEntry("endpoint", "bad-endpoint")
.containsEntry("profile", "yutong")
.containsEntry("operationalError", "true")
.containsEntry("phase", "init")
.containsEntry("phase", "connect")
.containsEntry("identityResolved", "false")
.containsEntry("identitySource", "UNKNOWN");
});
@@ -184,7 +184,7 @@ class MqttEndpointManagerTest {
.containsEntry("operationalError", "true")
.containsEntry("endpoint", "bad-endpoint")
.containsEntry("profile", "yutong")
.containsEntry("phase", "init");
.containsEntry("phase", "connect");
});
batchExecutor.close();

View File

@@ -1,6 +1,5 @@
package com.lingniu.ingest.inbound.mqtt;
import com.hivemq.client.mqtt.MqttClientSslConfig;
import com.lingniu.ingest.inbound.mqtt.client.MqttTlsSupport;
import com.lingniu.ingest.inbound.mqtt.config.MqttInboundProperties;
import org.junit.jupiter.api.Test;
@@ -9,6 +8,8 @@ import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.net.ssl.SSLContext;
import static org.assertj.core.api.Assertions.assertThat;
class MqttTlsSupportTest {
@@ -23,10 +24,9 @@ class MqttTlsSupportTest {
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
tls.setCaPem(ca.toString());
MqttClientSslConfig config = MqttTlsSupport.build(tls);
SSLContext config = MqttTlsSupport.buildSslContext(tls);
assertThat(config.getTrustManagerFactory()).isPresent();
assertThat(config.getKeyManagerFactory()).isEmpty();
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
}
@Test
@@ -42,24 +42,22 @@ class MqttTlsSupportTest {
tls.setClientPem(cert.toString());
tls.setClientKey(key.toString());
MqttClientSslConfig config = MqttTlsSupport.build(tls);
SSLContext config = MqttTlsSupport.buildSslContext(tls);
assertThat(config.getTrustManagerFactory()).isPresent();
assertThat(config.getKeyManagerFactory()).isPresent();
assertThat(config.getHostnameVerifier()).isEmpty();
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
}
@Test
void canDisableHostnameVerificationForLegacyMutualTlsBrokers() throws Exception {
void hostnameVerificationFlagDoesNotBlockSslContextCreation() throws Exception {
Path ca = tempDir.resolve("ca.pem");
Files.writeString(ca, CERT);
MqttInboundProperties.Tls tls = new MqttInboundProperties.Tls();
tls.setCaPem(ca.toString());
tls.setHostnameVerificationEnabled(false);
MqttClientSslConfig config = MqttTlsSupport.build(tls);
SSLContext config = MqttTlsSupport.buildSslContext(tls);
assertThat(config.getHostnameVerifier()).isPresent();
assertThat(config.getProtocol()).isEqualTo("TLSv1.2");
}
private static final String CERT = """