shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
go/vehicle-gateway/internal/gateway/mqtt_client.go
View File

@@ -41,6 +41,7 @@ type MQTTClientConfig struct {
Logger *slog.Logger
Metrics *metrics.Registry
PublishUnified bool
DelegateFields bool
}
type MQTTClient struct {
@@ -204,28 +205,42 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []
}
env.EventID = env.StableEventID()
c.recordParseErrorMetric(err)
} else {
} else if envelope.RequiresVehicleIdentity(env) {
resolveStarted := time.Now()
resolved, resolveErr := c.cfg.Resolver.Resolve(messageCtx, env)
if resolveErr != nil {
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
if env.Parsed == nil {
env.Parsed = map[string]any{}
if resolved.Protocol != "" || strings.TrimSpace(resolved.VIN) != "" || strings.TrimSpace(resolved.Phone) != "" {
env = resolved
}
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
identityStatus := identityErrorStatus(resolveErr)
c.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr)
annotateIdentityError(&env, resolveErr)
env.ParseStatus = envelope.ParsePartial
c.recordIdentityMetric("error")
c.recordIdentityMetric(identityStatus)
c.recordIdentityIssueMetric(identityStatus, env, identityIssueReason(identityStatus, resolveErr))
} else {
env = resolved
annotateIdentityUnresolved(&env)
c.recordIdentityMetric(identityStatus(env))
identityStatus := identityStatus(env)
c.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
c.recordIdentityMetric(identityStatus)
if identityStatus != "resolved" {
c.recordIdentityIssueMetric(identityStatus, env, "no_binding")
}
recordGatewayIdentityCacheStatus(c.cfg.Metrics, envelope.ProtocolYutongMQTT, env)
}
} else {
c.recordIdentitySkipMetric("non_vehicle_frame")
}
frameStatus = env.ParseStatus
env.EventKind = envelope.EventKindRaw
c.recordFrameMetric(env.ParseStatus)
if env.ParseStatus != envelope.ParseBadFrame {
realtime.EnsureParsedFields(&env)
}
if err := c.cfg.Sink.PublishRaw(messageCtx, env); err != nil {
canonicalRaw := canonicalRawEnvelope(env)
if err := c.cfg.Sink.PublishRaw(messageCtx, canonicalRaw); err != nil {
c.recordPublishMetric("raw", "error")
c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
return
@@ -234,21 +249,37 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []
if env.ParseStatus == envelope.ParseBadFrame {
return
}
if fieldsEnv, ok := realtime.BuildFieldsEnvelope(env); ok {
if err := c.cfg.Sink.PublishFields(messageCtx, fieldsEnv); err != nil {
c.recordPublishMetric("fields", "error")
c.cfg.Logger.Error("publish mqtt fields failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
return
if c.cfg.DelegateFields {
fieldCount, fieldsStatus, ok := gatewayDelegatedFields(env)
c.recordFieldsMetric(fieldsStatus)
if ok {
c.recordFieldsCount(fieldsStatus, fieldCount)
c.recordPublishMetric("fields", "delegated")
}
} else {
fieldsEnv, fieldsStatus, ok := gatewayFieldsEnvelope(env)
if !ok {
c.recordFieldsMetric(fieldsStatus)
} else {
if err := c.cfg.Sink.PublishFields(messageCtx, fieldsEnv); err != nil {
c.recordFieldsMetric(gatewayFieldsPublishError)
c.recordFieldsCount(gatewayFieldsPublishError, len(fieldsEnv.Fields))
c.recordPublishMetric("fields", "error")
c.cfg.Logger.Error("publish mqtt fields failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
} else {
c.recordFieldsMetric(fieldsStatus)
c.recordFieldsCount(fieldsStatus, len(fieldsEnv.Fields))
c.recordPublishMetric("fields", "ok")
}
}
c.recordPublishMetric("fields", "ok")
}
if c.cfg.PublishUnified {
if err := c.cfg.Sink.PublishUnified(messageCtx, env); err != nil {
if err := c.cfg.Sink.PublishUnified(messageCtx, canonicalRaw); err != nil {
c.recordPublishMetric("unified", "error")
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
return
} else {
c.recordPublishMetric("unified", "ok")
}
c.recordPublishMetric("unified", "ok")
}
}
@@ -256,10 +287,12 @@ func (c *MQTTClient) recordFrameMetric(status envelope.ParseStatus) {
if c.cfg.Metrics == nil {
return
}
c.cfg.Metrics.IncCounter("vehicle_gateway_frames_total", metrics.Labels{
labels := metrics.Labels{
"protocol": string(envelope.ProtocolYutongMQTT),
"status": string(status),
})
}
c.cfg.Metrics.IncCounter("vehicle_gateway_frames_total", labels)
metrics.RecordLastActivity(c.cfg.Metrics, "vehicle_gateway_last_frame_unix_seconds", labels)
}
func (c *MQTTClient) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
@@ -282,11 +315,21 @@ func (c *MQTTClient) recordPublishMetric(kind string, status string) {
if c.cfg.Metrics == nil {
return
}
c.cfg.Metrics.IncCounter("vehicle_gateway_publish_total", metrics.Labels{
labels := metrics.Labels{
"protocol": string(envelope.ProtocolYutongMQTT),
"kind": kind,
"status": status,
})
}
c.cfg.Metrics.IncCounter("vehicle_gateway_publish_total", labels)
metrics.RecordLastActivity(c.cfg.Metrics, "vehicle_gateway_last_publish_unix_seconds", labels)
}
func (c *MQTTClient) recordFieldsMetric(status string) {
recordGatewayFieldsMetric(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status)
}
func (c *MQTTClient) recordFieldsCount(status string, fieldCount int) {
recordGatewayFieldsCount(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status, fieldCount)
}
func (c *MQTTClient) recordParseErrorMetric(err error) {
@@ -308,3 +351,37 @@ func (c *MQTTClient) recordIdentityMetric(status string) {
"status": status,
})
}
func (c *MQTTClient) recordIdentitySkipMetric(reason string) {
if c.cfg.Metrics == nil {
return
}
c.cfg.Metrics.IncCounter("vehicle_gateway_identity_skips_total", metrics.Labels{
"protocol": string(envelope.ProtocolYutongMQTT),
"reason": reason,
})
}
func (c *MQTTClient) recordIdentityIssueMetric(status string, env envelope.FrameEnvelope, reason string) {
if c.cfg.Metrics == nil {
return
}
messageID := strings.TrimSpace(env.MessageID)
if messageID == "" {
messageID = "unknown"
}
reason = strings.TrimSpace(reason)
if reason == "" {
reason = "unknown"
}
c.cfg.Metrics.IncCounter("vehicle_gateway_identity_issues_total", metrics.Labels{
"protocol": string(envelope.ProtocolYutongMQTT),
"status": status,
"message_id": messageID,
"reason": reason,
})
}
func (c *MQTTClient) recordIdentityDuration(status string, elapsed time.Duration) {
recordGatewayIdentityDuration(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status, elapsed)
}