388 lines
12 KiB
Go
388 lines
12 KiB
Go
package gateway
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
mqtt "github.com/eclipse/paho.mqtt.golang"
|
|
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
|
)
|
|
|
|
type MQTTClientConfig struct {
|
|
EndpointName string
|
|
Broker string
|
|
ClientID string
|
|
Username string
|
|
Password string
|
|
Topics []string
|
|
QoS byte
|
|
CleanSession bool
|
|
KeepAlive time.Duration
|
|
ConnectTimeout time.Duration
|
|
TLSCACertPath string
|
|
TLSClientCertPath string
|
|
TLSClientKeyPath string
|
|
TLSHostnameVerification bool
|
|
Sink eventbus.Sink
|
|
Resolver identity.Resolver
|
|
Logger *slog.Logger
|
|
Metrics *metrics.Registry
|
|
PublishUnified bool
|
|
DelegateFields bool
|
|
}
|
|
|
|
type MQTTClient struct {
|
|
cfg MQTTClientConfig
|
|
client mqtt.Client
|
|
}
|
|
|
|
func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
|
|
if strings.TrimSpace(cfg.Broker) == "" {
|
|
return nil, errors.New("mqtt broker is required")
|
|
}
|
|
if strings.TrimSpace(cfg.ClientID) == "" {
|
|
return nil, errors.New("mqtt client id is required")
|
|
}
|
|
if len(cfg.Topics) == 0 {
|
|
return nil, errors.New("mqtt topics are required")
|
|
}
|
|
if cfg.Sink == nil {
|
|
return nil, errors.New("sink is required")
|
|
}
|
|
if cfg.Logger == nil {
|
|
cfg.Logger = slog.Default()
|
|
}
|
|
if cfg.Resolver == nil {
|
|
cfg.Resolver = identity.NoopResolver{}
|
|
}
|
|
if cfg.EndpointName == "" {
|
|
cfg.EndpointName = "yutong"
|
|
}
|
|
return &MQTTClient{cfg: cfg}, nil
|
|
}
|
|
|
|
func (c *MQTTClient) Start(ctx context.Context) error {
|
|
opts, err := c.buildOptions(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
c.client = mqtt.NewClient(opts)
|
|
token := c.client.Connect()
|
|
if token.Wait() && token.Error() != nil {
|
|
return token.Error()
|
|
}
|
|
go func() {
|
|
<-ctx.Done()
|
|
if c.client != nil && c.client.IsConnected() {
|
|
c.client.Disconnect(250)
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
func (c *MQTTClient) buildOptions(ctx context.Context) (*mqtt.ClientOptions, error) {
|
|
keepAlive := c.cfg.KeepAlive
|
|
if keepAlive <= 0 {
|
|
keepAlive = 20 * time.Second
|
|
}
|
|
connectTimeout := c.cfg.ConnectTimeout
|
|
if connectTimeout <= 0 {
|
|
connectTimeout = 10 * time.Second
|
|
}
|
|
opts := mqtt.NewClientOptions().
|
|
AddBroker(c.cfg.Broker).
|
|
SetClientID(c.cfg.ClientID).
|
|
SetUsername(c.cfg.Username).
|
|
SetPassword(c.cfg.Password).
|
|
SetCleanSession(c.cfg.CleanSession).
|
|
SetAutoReconnect(true).
|
|
SetConnectRetry(true).
|
|
SetConnectRetryInterval(5 * time.Second).
|
|
SetResumeSubs(true).
|
|
SetOrderMatters(false).
|
|
SetKeepAlive(keepAlive).
|
|
SetConnectTimeout(connectTimeout)
|
|
|
|
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
|
|
c.handleMessage(ctx, message.Topic(), message.Payload())
|
|
})
|
|
opts.OnConnect = func(client mqtt.Client) {
|
|
for _, topic := range c.cfg.Topics {
|
|
token := client.Subscribe(topic, c.cfg.QoS, nil)
|
|
if token.Wait() && token.Error() != nil {
|
|
c.cfg.Logger.Error("mqtt subscribe failed", "broker", c.cfg.Broker, "topic", topic, "error", token.Error())
|
|
continue
|
|
}
|
|
c.cfg.Logger.Info("mqtt subscribed", "broker", c.cfg.Broker, "topic", topic, "qos", c.cfg.QoS)
|
|
}
|
|
}
|
|
opts.OnConnectionLost = func(_ mqtt.Client, err error) {
|
|
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
|
|
}
|
|
|
|
tlsConfig, err := c.buildTLSConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tlsConfig != nil {
|
|
opts.SetTLSConfig(tlsConfig)
|
|
}
|
|
return opts, nil
|
|
}
|
|
|
|
func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
|
|
caPath := strings.TrimSpace(c.cfg.TLSCACertPath)
|
|
certPath := strings.TrimSpace(c.cfg.TLSClientCertPath)
|
|
keyPath := strings.TrimSpace(c.cfg.TLSClientKeyPath)
|
|
if caPath == "" && certPath == "" && keyPath == "" {
|
|
return nil, nil
|
|
}
|
|
config := &tls.Config{
|
|
MinVersion: tls.VersionTLS12,
|
|
InsecureSkipVerify: !c.cfg.TLSHostnameVerification,
|
|
}
|
|
if caPath != "" {
|
|
caPEM, err := os.ReadFile(caPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read mqtt ca certificate: %w", err)
|
|
}
|
|
roots := x509.NewCertPool()
|
|
if !roots.AppendCertsFromPEM(caPEM) {
|
|
return nil, fmt.Errorf("parse mqtt ca certificate %s", caPath)
|
|
}
|
|
config.RootCAs = roots
|
|
}
|
|
if certPath != "" || keyPath != "" {
|
|
if certPath == "" || keyPath == "" {
|
|
return nil, errors.New("mqtt client certificate and key must be configured together")
|
|
}
|
|
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load mqtt client certificate: %w", err)
|
|
}
|
|
config.Certificates = []tls.Certificate{cert}
|
|
}
|
|
return config, nil
|
|
}
|
|
|
|
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
|
|
started := time.Now()
|
|
frameStatus := envelope.ParseBadFrame
|
|
defer func() {
|
|
c.recordFrameDuration(frameStatus, time.Since(started))
|
|
}()
|
|
|
|
messageCtx, cancelMessage := context.WithTimeout(context.WithoutCancel(ctx), frameOperationTimeout)
|
|
defer cancelMessage()
|
|
|
|
receivedAtMS := time.Now().UnixMilli()
|
|
env, err := yutongmqtt.ParseMessage(c.cfg.EndpointName, topic, payload, receivedAtMS)
|
|
if err != nil {
|
|
env = envelope.FrameEnvelope{
|
|
Protocol: envelope.ProtocolYutongMQTT,
|
|
MessageID: "MQTT",
|
|
VehicleKeyHint: "unknown",
|
|
SourceEndpoint: "mqtt://" + c.cfg.EndpointName + topic,
|
|
EventTimeMS: receivedAtMS,
|
|
ReceivedAtMS: receivedAtMS,
|
|
RawText: string(payload),
|
|
ParseStatus: envelope.ParseBadFrame,
|
|
ParseError: err.Error(),
|
|
}
|
|
env.EventID = env.StableEventID()
|
|
c.recordParseErrorMetric(err)
|
|
} else if envelope.RequiresVehicleIdentity(env) {
|
|
resolveStarted := time.Now()
|
|
resolved, resolveErr := c.cfg.Resolver.Resolve(messageCtx, env)
|
|
if resolveErr != nil {
|
|
if resolved.Protocol != "" || strings.TrimSpace(resolved.VIN) != "" || strings.TrimSpace(resolved.Phone) != "" {
|
|
env = resolved
|
|
}
|
|
identityStatus := identityErrorStatus(resolveErr)
|
|
c.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
|
|
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
|
|
annotateIdentityError(&env, resolveErr)
|
|
env.ParseStatus = envelope.ParsePartial
|
|
c.recordIdentityMetric(identityStatus)
|
|
c.recordIdentityIssueMetric(identityStatus, env, identityIssueReason(identityStatus, resolveErr))
|
|
} else {
|
|
env = resolved
|
|
annotateIdentityUnresolved(&env)
|
|
identityStatus := identityStatus(env)
|
|
c.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
|
|
c.recordIdentityMetric(identityStatus)
|
|
if identityStatus != "resolved" {
|
|
c.recordIdentityIssueMetric(identityStatus, env, "no_binding")
|
|
}
|
|
recordGatewayIdentityCacheStatus(c.cfg.Metrics, envelope.ProtocolYutongMQTT, env)
|
|
}
|
|
} else {
|
|
c.recordIdentitySkipMetric("non_vehicle_frame")
|
|
}
|
|
frameStatus = env.ParseStatus
|
|
env.EventKind = envelope.EventKindRaw
|
|
c.recordFrameMetric(env.ParseStatus)
|
|
if env.ParseStatus != envelope.ParseBadFrame {
|
|
realtime.EnsureParsedFields(&env)
|
|
}
|
|
canonicalRaw := canonicalRawEnvelope(env)
|
|
if err := c.cfg.Sink.PublishRaw(messageCtx, canonicalRaw); err != nil {
|
|
c.recordPublishMetric("raw", "error")
|
|
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
|
return
|
|
}
|
|
c.recordPublishMetric("raw", "ok")
|
|
if env.ParseStatus == envelope.ParseBadFrame {
|
|
return
|
|
}
|
|
if c.cfg.DelegateFields {
|
|
fieldCount, fieldsStatus, ok := gatewayDelegatedFields(env)
|
|
c.recordFieldsMetric(fieldsStatus)
|
|
if ok {
|
|
c.recordFieldsCount(fieldsStatus, fieldCount)
|
|
c.recordPublishMetric("fields", "delegated")
|
|
}
|
|
} else {
|
|
fieldsEnv, fieldsStatus, ok := gatewayFieldsEnvelope(env)
|
|
if !ok {
|
|
c.recordFieldsMetric(fieldsStatus)
|
|
} else {
|
|
if err := c.cfg.Sink.PublishFields(messageCtx, fieldsEnv); err != nil {
|
|
c.recordFieldsMetric(gatewayFieldsPublishError)
|
|
c.recordFieldsCount(gatewayFieldsPublishError, len(fieldsEnv.Fields))
|
|
c.recordPublishMetric("fields", "error")
|
|
c.cfg.Logger.Error("publish mqtt fields failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
|
} else {
|
|
c.recordFieldsMetric(fieldsStatus)
|
|
c.recordFieldsCount(fieldsStatus, len(fieldsEnv.Fields))
|
|
c.recordPublishMetric("fields", "ok")
|
|
}
|
|
}
|
|
}
|
|
if c.cfg.PublishUnified {
|
|
if err := c.cfg.Sink.PublishUnified(messageCtx, canonicalRaw); err != nil {
|
|
c.recordPublishMetric("unified", "error")
|
|
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
|
} else {
|
|
c.recordPublishMetric("unified", "ok")
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *MQTTClient) recordFrameMetric(status envelope.ParseStatus) {
|
|
if c.cfg.Metrics == nil {
|
|
return
|
|
}
|
|
labels := metrics.Labels{
|
|
"protocol": string(envelope.ProtocolYutongMQTT),
|
|
"status": string(status),
|
|
}
|
|
c.cfg.Metrics.IncCounter("vehicle_gateway_frames_total", labels)
|
|
metrics.RecordLastActivity(c.cfg.Metrics, "vehicle_gateway_last_frame_unix_seconds", labels)
|
|
}
|
|
|
|
func (c *MQTTClient) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
|
|
if c.cfg.Metrics == nil {
|
|
return
|
|
}
|
|
elapsedMS := float64(elapsed.Milliseconds())
|
|
labels := metrics.Labels{
|
|
"protocol": string(envelope.ProtocolYutongMQTT),
|
|
"status": string(status),
|
|
}
|
|
c.cfg.Metrics.SetGauge("vehicle_gateway_frame_duration_ms", metrics.Labels{
|
|
"protocol": string(envelope.ProtocolYutongMQTT),
|
|
"status": string(status),
|
|
}, elapsedMS)
|
|
c.cfg.Metrics.ObserveHistogram("vehicle_gateway_frame_duration_ms_histogram", labels, gatewayFrameDurationBucketsMS, elapsedMS)
|
|
}
|
|
|
|
func (c *MQTTClient) recordPublishMetric(kind string, status string) {
|
|
if c.cfg.Metrics == nil {
|
|
return
|
|
}
|
|
labels := metrics.Labels{
|
|
"protocol": string(envelope.ProtocolYutongMQTT),
|
|
"kind": kind,
|
|
"status": status,
|
|
}
|
|
c.cfg.Metrics.IncCounter("vehicle_gateway_publish_total", labels)
|
|
metrics.RecordLastActivity(c.cfg.Metrics, "vehicle_gateway_last_publish_unix_seconds", labels)
|
|
}
|
|
|
|
func (c *MQTTClient) recordFieldsMetric(status string) {
|
|
recordGatewayFieldsMetric(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status)
|
|
}
|
|
|
|
func (c *MQTTClient) recordFieldsCount(status string, fieldCount int) {
|
|
recordGatewayFieldsCount(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status, fieldCount)
|
|
}
|
|
|
|
func (c *MQTTClient) recordParseErrorMetric(err error) {
|
|
if c.cfg.Metrics == nil {
|
|
return
|
|
}
|
|
c.cfg.Metrics.IncCounter("vehicle_gateway_parse_errors_total", metrics.Labels{
|
|
"protocol": string(envelope.ProtocolYutongMQTT),
|
|
"reason": classifyError(err),
|
|
})
|
|
}
|
|
|
|
func (c *MQTTClient) recordIdentityMetric(status string) {
|
|
if c.cfg.Metrics == nil {
|
|
return
|
|
}
|
|
c.cfg.Metrics.IncCounter("vehicle_gateway_identity_total", metrics.Labels{
|
|
"protocol": string(envelope.ProtocolYutongMQTT),
|
|
"status": status,
|
|
})
|
|
}
|
|
|
|
func (c *MQTTClient) recordIdentitySkipMetric(reason string) {
|
|
if c.cfg.Metrics == nil {
|
|
return
|
|
}
|
|
c.cfg.Metrics.IncCounter("vehicle_gateway_identity_skips_total", metrics.Labels{
|
|
"protocol": string(envelope.ProtocolYutongMQTT),
|
|
"reason": reason,
|
|
})
|
|
}
|
|
|
|
func (c *MQTTClient) recordIdentityIssueMetric(status string, env envelope.FrameEnvelope, reason string) {
|
|
if c.cfg.Metrics == nil {
|
|
return
|
|
}
|
|
messageID := strings.TrimSpace(env.MessageID)
|
|
if messageID == "" {
|
|
messageID = "unknown"
|
|
}
|
|
reason = strings.TrimSpace(reason)
|
|
if reason == "" {
|
|
reason = "unknown"
|
|
}
|
|
c.cfg.Metrics.IncCounter("vehicle_gateway_identity_issues_total", metrics.Labels{
|
|
"protocol": string(envelope.ProtocolYutongMQTT),
|
|
"status": status,
|
|
"message_id": messageID,
|
|
"reason": reason,
|
|
})
|
|
}
|
|
|
|
func (c *MQTTClient) recordIdentityDuration(status string, elapsed time.Duration) {
|
|
recordGatewayIdentityDuration(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status, elapsed)
|
|
}
|