feat: add yutong mqtt ingestion
This commit is contained in:
lingniu
129
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
129
go/vehicle-gateway/internal/gateway/mqtt_client.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
|
||||
)
|
||||
|
||||
type MQTTClientConfig struct {
|
||||
EndpointName string
|
||||
Broker string
|
||||
ClientID string
|
||||
Username string
|
||||
Password string
|
||||
Topics []string
|
||||
QoS byte
|
||||
Sink eventbus.Sink
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
type MQTTClient struct {
|
||||
cfg MQTTClientConfig
|
||||
client mqtt.Client
|
||||
}
|
||||
|
||||
func NewMQTTClient(cfg MQTTClientConfig) (*MQTTClient, error) {
|
||||
if strings.TrimSpace(cfg.Broker) == "" {
|
||||
return nil, errors.New("mqtt broker is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ClientID) == "" {
|
||||
return nil, errors.New("mqtt client id is required")
|
||||
}
|
||||
if len(cfg.Topics) == 0 {
|
||||
return nil, errors.New("mqtt topics are required")
|
||||
}
|
||||
if cfg.Sink == nil {
|
||||
return nil, errors.New("sink is required")
|
||||
}
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
if cfg.EndpointName == "" {
|
||||
cfg.EndpointName = "yutong"
|
||||
}
|
||||
return &MQTTClient{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) Start(ctx context.Context) error {
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker(c.cfg.Broker).
|
||||
SetClientID(c.cfg.ClientID).
|
||||
SetUsername(c.cfg.Username).
|
||||
SetPassword(c.cfg.Password).
|
||||
SetCleanSession(false).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectRetry(true).
|
||||
SetConnectRetryInterval(5 * time.Second).
|
||||
SetKeepAlive(20 * time.Second).
|
||||
SetConnectTimeout(10 * time.Second)
|
||||
|
||||
opts.SetDefaultPublishHandler(func(_ mqtt.Client, message mqtt.Message) {
|
||||
c.handleMessage(ctx, message.Topic(), message.Payload())
|
||||
})
|
||||
opts.OnConnect = func(client mqtt.Client) {
|
||||
for _, topic := range c.cfg.Topics {
|
||||
token := client.Subscribe(topic, c.cfg.QoS, nil)
|
||||
if token.Wait() && token.Error() != nil {
|
||||
c.cfg.Logger.Error("mqtt subscribe failed", "broker", c.cfg.Broker, "topic", topic, "error", token.Error())
|
||||
continue
|
||||
}
|
||||
c.cfg.Logger.Info("mqtt subscribed", "broker", c.cfg.Broker, "topic", topic, "qos", c.cfg.QoS)
|
||||
}
|
||||
}
|
||||
opts.OnConnectionLost = func(_ mqtt.Client, err error) {
|
||||
c.cfg.Logger.Warn("mqtt connection lost", "broker", c.cfg.Broker, "error", err)
|
||||
}
|
||||
|
||||
c.client = mqtt.NewClient(opts)
|
||||
token := c.client.Connect()
|
||||
if token.Wait() && token.Error() != nil {
|
||||
return token.Error()
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
if c.client != nil && c.client.IsConnected() {
|
||||
c.client.Disconnect(250)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := yutongmqtt.ParseMessage(c.cfg.EndpointName, topic, payload, receivedAtMS)
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
MessageID: "MQTT",
|
||||
VehicleKeyHint: "unknown",
|
||||
SourceEndpoint: "mqtt://" + c.cfg.EndpointName + topic,
|
||||
EventTimeMS: receivedAtMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawText: string(payload),
|
||||
RawHex: strings.ToUpper(hex.EncodeToString(payload)),
|
||||
ParseStatus: envelope.ParseBadFrame,
|
||||
ParseError: err.Error(),
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
}
|
||||
if err := c.cfg.Sink.PublishRaw(ctx, env); err != nil {
|
||||
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
}
|
||||
if env.ParseStatus == envelope.ParseBadFrame {
|
||||
return
|
||||
}
|
||||
if err := c.cfg.Sink.PublishUnified(ctx, env); err != nil {
|
||||
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||
}
|
||||
}
|
||||
61
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
61
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "tcp://127.0.0.1:1883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
|
||||
"device":"LTEST000000000001",
|
||||
"time":"20260413100000",
|
||||
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
|
||||
}`))
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
|
||||
t.Fatalf("unexpected raw envelope: %#v", sink.raw[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
client, err := NewMQTTClient(MQTTClientConfig{
|
||||
EndpointName: "endpoint-a",
|
||||
Broker: "tcp://127.0.0.1:1883",
|
||||
ClientID: "test-client",
|
||||
Topics: []string{"/ytforward/shln/+"},
|
||||
Sink: sink,
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||
}
|
||||
|
||||
client.handleMessage(context.Background(), "/ytforward/shln/bad", []byte("{bad-json"))
|
||||
|
||||
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||
}
|
||||
if sink.raw[0].ParseStatus != envelope.ParseBadFrame {
|
||||
t.Fatalf("parse status = %q", sink.raw[0].ParseStatus)
|
||||
}
|
||||
}
|
||||
189
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
189
go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go
Normal file
@@ -0,0 +1,189 @@
|
||||
package yutongmqtt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var ErrInvalidJSON = errors.New("yutong mqtt invalid json")
|
||||
|
||||
func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS int64) (envelope.FrameEnvelope, error) {
|
||||
root := map[string]any{}
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&root); err != nil {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: %v", ErrInvalidJSON, err)
|
||||
}
|
||||
|
||||
data := objectValue(root, "data")
|
||||
if data == nil {
|
||||
data = root
|
||||
}
|
||||
deviceID := firstText(root, "device", "deviceId", "terminalId", "terminalID", "imei", "IMEI", "mqttDeviceId")
|
||||
externalVIN := firstText(root, "vin", "VIN", "externalVin", "vehicleVin")
|
||||
vin := externalVIN
|
||||
if vin == "" && looksLikeVIN(deviceID) {
|
||||
vin = deviceID
|
||||
}
|
||||
plate := firstText(root, "plateNo", "carNo", "plate", "vehicleNo")
|
||||
phone := firstText(root, "sim", "phone", "mobile")
|
||||
eventTimeMS := parseTimeMS(firstText(root, "time", "deviceTime", "eventTime", "gpsTime"), receivedAtMS)
|
||||
fields := fieldsFromData(data)
|
||||
|
||||
parsed := map[string]any{
|
||||
"endpoint": endpoint,
|
||||
"topic": topic,
|
||||
"root": root,
|
||||
"data": data,
|
||||
}
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
MessageID: "MQTT",
|
||||
VIN: vin,
|
||||
VehicleKeyHint: deviceID,
|
||||
Phone: phone,
|
||||
DeviceID: deviceID,
|
||||
Plate: plate,
|
||||
SourceEndpoint: "mqtt://" + endpoint + topic,
|
||||
EventTimeMS: eventTimeMS,
|
||||
ReceivedAtMS: receivedAtMS,
|
||||
RawText: string(payload),
|
||||
Parsed: parsed,
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func fieldsFromData(data map[string]any) map[string]any {
|
||||
fields := map[string]any{}
|
||||
if speed, ok := firstFloat(data, "METER_SPEED", "speed", "speed_kmh"); ok {
|
||||
fields[envelope.FieldSpeedKMH] = speed
|
||||
}
|
||||
if mileage, ok := firstFloat(data, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km"); ok {
|
||||
fields[envelope.FieldTotalMileageKM] = mileage
|
||||
}
|
||||
if soc, ok := firstFloat(data, "BATTERY_CAPACITY_SOC", "soc", "SOC", "soc_percent"); ok {
|
||||
fields[envelope.FieldSOCPercent] = soc
|
||||
}
|
||||
if longitude, ok := firstCoord(data, "LONGITUDE", "longitude", "lng"); ok {
|
||||
fields[envelope.FieldLongitude] = longitude
|
||||
}
|
||||
if latitude, ok := firstCoord(data, "LATITUDE", "latitude", "lat"); ok {
|
||||
fields[envelope.FieldLatitude] = latitude
|
||||
}
|
||||
if direction, ok := firstFloat(data, "GPSDirection", "direction", "direction_deg"); ok {
|
||||
fields["direction_deg"] = direction
|
||||
}
|
||||
if gear, ok := firstFloat(data, "ON_GEAR", "gear"); ok {
|
||||
fields["gear"] = gear
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func objectValue(root map[string]any, key string) map[string]any {
|
||||
value, ok := root[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
typed, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return typed
|
||||
}
|
||||
|
||||
func firstText(root map[string]any, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value, ok := root[key]; ok && value != nil {
|
||||
text := strings.TrimSpace(fmt.Sprint(value))
|
||||
if text != "" && text != "<nil>" {
|
||||
return text
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstFloat(root map[string]any, keys ...string) (float64, bool) {
|
||||
for _, key := range keys {
|
||||
value, ok := root[key]
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
if parsed, ok := toFloat(value); ok {
|
||||
return parsed, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func firstCoord(root map[string]any, keys ...string) (float64, bool) {
|
||||
value, ok := firstFloat(root, keys...)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if value == 0 || int64(value) == 999999 {
|
||||
return 0, false
|
||||
}
|
||||
if value > 1000 || value < -1000 {
|
||||
value = value / 1_000_000
|
||||
}
|
||||
return value, true
|
||||
}
|
||||
|
||||
func toFloat(value any) (float64, bool) {
|
||||
switch typed := value.(type) {
|
||||
case json.Number:
|
||||
parsed, err := typed.Float64()
|
||||
return parsed, err == nil
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func parseTimeMS(raw string, fallback int64) int64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
if len(raw) == 14 {
|
||||
if parsed, err := time.ParseInLocation("20060102150405", raw, time.FixedZone("Asia/Shanghai", 8*3600)); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
}
|
||||
if parsed, err := time.Parse(time.RFC3339, raw); err == nil {
|
||||
return parsed.UnixMilli()
|
||||
}
|
||||
if epoch, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
||||
if epoch > 10_000_000_000 {
|
||||
return epoch
|
||||
}
|
||||
return epoch * 1000
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func looksLikeVIN(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
return len(value) == 17
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package yutongmqtt
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"device": "LTEST000000000001",
|
||||
"time": "20260413100000",
|
||||
"plateNo": "豫A12345",
|
||||
"data": {
|
||||
"METER_SPEED": 52.3,
|
||||
"TOTAL_MILEAGE": 123456.7,
|
||||
"BATTERY_CAPACITY_SOC": 70,
|
||||
"LONGITUDE": 116397128,
|
||||
"LATITUDE": 39916527,
|
||||
"GPSDirection": 88
|
||||
}
|
||||
}`)
|
||||
|
||||
env, err := ParseMessage("endpoint-a", "/ytforward/shln/dev1", payload, 1782745114999)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if env.Protocol != envelope.ProtocolYutongMQTT {
|
||||
t.Fatalf("protocol = %q", env.Protocol)
|
||||
}
|
||||
if env.VIN != "LTEST000000000001" || env.DeviceID != "LTEST000000000001" {
|
||||
t.Fatalf("identity fields = vin:%q device:%q", env.VIN, env.DeviceID)
|
||||
}
|
||||
if env.Plate != "豫A12345" {
|
||||
t.Fatalf("plate = %q", env.Plate)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 52.3)
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 123456.7)
|
||||
assertFloatField(t, env, envelope.FieldSOCPercent, 70)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
|
||||
assertFloatField(t, env, "direction_deg", 88)
|
||||
if env.EventTimeMS == 1782745114999 {
|
||||
t.Fatal("event time should come from device time")
|
||||
}
|
||||
if env.RawText == "" || env.Parsed["data"] == nil {
|
||||
t.Fatalf("raw/parsed missing: %#v", env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) {
|
||||
payload := []byte(`{"device":"YT-DEVICE-001","time":"20260413100000","data":{"METER_SPEED":"12.3"}}`)
|
||||
env, err := ParseMessage("endpoint-a", "/ytforward/shln/YT-DEVICE-001", payload, 1782745114999)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if env.VIN != "" {
|
||||
t.Fatalf("non-vin device should not be treated as vin: %q", env.VIN)
|
||||
}
|
||||
if env.VehicleKey() != "YT-DEVICE-001" {
|
||||
t.Fatalf("vehicle key = %q", env.VehicleKey())
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldSpeedKMH, 12.3)
|
||||
}
|
||||
|
||||
func TestParseMessageRejectsMalformedJSON(t *testing.T) {
|
||||
_, err := ParseMessage("endpoint-a", "/bad", []byte("{bad-json"), 1782745114999)
|
||||
if !errors.Is(err, ErrInvalidJSON) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
|
||||
t.Helper()
|
||||
got, ok := env.Fields[key].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("field %s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user