shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go): add realtime pipeline observability

Browse Source
This commit is contained in:
lingniu
2026-07-03 15:06:27 +08:00
parent 28c81611a1
commit b8299faefa
10 changed files with 636 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
go/vehicle-gateway/internal/gateway/mqtt_client.go
View File

@@ -197,6 +197,7 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []
ParseError: err.Error(),
}
env.EventID = env.StableEventID()
c.recordParseErrorMetric(err)
} else {
resolved, resolveErr := c.cfg.Resolver.Resolve(messageCtx, env)
if resolveErr != nil {
@@ -206,8 +207,11 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []
}
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
env.ParseStatus = envelope.ParsePartial
c.recordIdentityMetric("error")
} else {
env = resolved
annotateIdentityUnresolved(&env)
c.recordIdentityMetric(identityStatus(env))
}
}
c.recordFrameMetric(env.ParseStatus)
@@ -258,3 +262,23 @@ func (c *MQTTClient) recordPublishMetric(kind string, status string) {
"status": status,
})
}
func (c *MQTTClient) recordParseErrorMetric(err error) {
if c.cfg.Metrics == nil {
return
}
c.cfg.Metrics.IncCounter("vehicle_gateway_parse_errors_total", metrics.Labels{
"protocol": string(envelope.ProtocolYutongMQTT),
"reason": classifyError(err),
})
}
func (c *MQTTClient) recordIdentityMetric(status string) {
if c.cfg.Metrics == nil {
return
}
c.cfg.Metrics.IncCounter("vehicle_gateway_identity_total", metrics.Labels{
"protocol": string(envelope.ProtocolYutongMQTT),
"status": status,
})
}

6
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
View File

@@ -77,6 +77,7 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
text := registry.Render()
for _, want := range []string{
`vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1`,
`vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="resolved"} 1`,
`vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 1`,
} {
if !strings.Contains(text, want) {
@@ -90,6 +91,7 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
sink := &recordingSink{}
registry := metrics.NewRegistry()
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
@@ -97,6 +99,7 @@ func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
Topics: []string{"/ytforward/shln/+"},
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
Metrics: registry,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
@@ -116,6 +119,9 @@ func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
if sink.raw[0].RawHex != "" {
t.Fatalf("bad mqtt raw envelope should not duplicate text payload as hex: %q", sink.raw[0].RawHex)
}
if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_parse_errors_total{protocol="YUTONG_MQTT",reason="json"} 1`) {
t.Fatalf("parse error metric missing:\n%s", text)
}
}
func TestMQTTClientUsesUncancelledMessageContextForReceivedMessage(t *testing.T) {

68
go/vehicle-gateway/internal/gateway/tcp_server.go
View File

@@ -211,6 +211,7 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
ParseError: err.Error(),
}
env.EventID = env.StableEventID()
s.recordParseErrorMetric(err)
} else {
resolved, resolveErr := s.resolver.Resolve(frameCtx, env)
if resolveErr != nil {
@@ -220,8 +221,11 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
}
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
env.ParseStatus = envelope.ParsePartial
s.recordIdentityMetric("error")
} else {
env = resolved
annotateIdentityUnresolved(&env)
s.recordIdentityMetric(identityStatus(env))
}
enrichConnectionPlatform(&env, state)
}
@@ -314,6 +318,26 @@ func (s *TCPServer) recordPublishMetric(kind string, status string) {
})
}
func (s *TCPServer) recordParseErrorMetric(err error) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_parse_errors_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"reason": classifyError(err),
})
}
func (s *TCPServer) recordIdentityMetric(status string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_identity_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": status,
})
}
func (s *TCPServer) recordConnectionMetric(delta float64) {
if s.metrics == nil {
return
@@ -323,6 +347,50 @@ func (s *TCPServer) recordConnectionMetric(delta float64) {
}, delta)
}
func identityStatus(env envelope.FrameEnvelope) string {
if strings.TrimSpace(env.VIN) != "" {
return "resolved"
}
if env.ParseStatus == envelope.ParsePartial {
return "error"
}
return "unresolved"
}
func annotateIdentityUnresolved(env *envelope.FrameEnvelope) {
if env == nil || env.ParseStatus == envelope.ParseBadFrame || strings.TrimSpace(env.VIN) != "" {
return
}
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
if _, exists := env.Parsed["identity"]; exists {
return
}
env.Parsed["identity"] = map[string]any{
"resolved": false,
"reason": "no_binding",
}
}
func classifyError(err error) string {
text := strings.ToLower(strings.TrimSpace(fmt.Sprint(err)))
switch {
case text == "":
return "unknown"
case strings.Contains(text, "bcc") || strings.Contains(text, "checksum"):
return "checksum"
case strings.Contains(text, "short") || strings.Contains(text, "truncated") || strings.Contains(text, "length"):
return "length"
case strings.Contains(text, "json"):
return "json"
case strings.Contains(text, "start"):
return "start_symbol"
default:
return "parse"
}
}
func (p TCPProtocol) String() string {
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
}

38
go/vehicle-gateway/internal/gateway/tcp_server_test.go
View File

@@ -79,6 +79,7 @@ func TestTCPServerRecordsFrameMetrics(t *testing.T) {
text := registry.Render()
for _, want := range []string{
`vehicle_gateway_frames_total{protocol="JT808",status="OK"} 1`,
`vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 1`,
`vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 1`,
} {
if !strings.Contains(text, want) {
@@ -139,12 +140,14 @@ func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
good[len(good)-1] ^= 0xff
sink := &recordingSink{}
registry := metrics.NewRegistry()
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolGB32960,
Addr: ":0",
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
}, sink)
server.metrics = registry
client, done := runPipe(t, server)
if _, err := client.Write(good); err != nil {
@@ -162,6 +165,41 @@ func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
if sink.raw[0].ParseError == "" {
t.Fatal("parse error should be recorded")
}
if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_parse_errors_total{protocol="GB32960",reason="checksum"} 1`) {
t.Fatalf("parse error metric missing:\n%s", text)
}
}
func TestTCPServerAnnotatesUnresolvedIdentity(t *testing.T) {
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolJT808,
Addr: ":0",
Extract: func(raw []byte) ([][]byte, []byte, error) {
return [][]byte{raw}, nil, nil
},
Parse: func(_ []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
Phone: "13307795425",
SourceEndpoint: sourceEndpoint,
ReceivedAtMS: receivedAtMS,
EventTimeMS: receivedAtMS,
ParseStatus: envelope.ParseOK,
}, nil
},
}, sink)
server.handleFrame(context.Background(), nil, []byte{0x01}, "127.0.0.1:808", &connectionState{})
if len(sink.raw) != 1 {
t.Fatalf("raw count = %d", len(sink.raw))
}
identity, ok := sink.raw[0].Parsed["identity"].(map[string]any)
if !ok || identity["resolved"] != false || identity["reason"] != "no_binding" {
t.Fatalf("identity metadata = %#v", sink.raw[0].Parsed["identity"])
}
}
func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {