feat(go): improve realtime pipeline resilience

This commit is contained in:
lingniu
2026-07-03 15:20:22 +08:00
parent b8299faefa
commit 9994f2baf4
6 changed files with 213 additions and 11 deletions

View File

@@ -179,6 +179,12 @@ func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
}
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
started := time.Now()
frameStatus := envelope.ParseBadFrame
defer func() {
c.recordFrameDuration(frameStatus, time.Since(started))
}()
messageCtx, cancelMessage := context.WithTimeout(context.WithoutCancel(ctx), frameOperationTimeout)
defer cancelMessage()
@@ -214,6 +220,7 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []
c.recordIdentityMetric(identityStatus(env))
}
}
frameStatus = env.ParseStatus
c.recordFrameMetric(env.ParseStatus)
if err := c.cfg.Sink.PublishRaw(messageCtx, env); err != nil {
c.recordPublishMetric("raw", "error")
@@ -252,6 +259,16 @@ func (c *MQTTClient) recordFrameMetric(status envelope.ParseStatus) {
})
}
func (c *MQTTClient) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
if c.cfg.Metrics == nil {
return
}
c.cfg.Metrics.SetGauge("vehicle_gateway_frame_duration_ms", metrics.Labels{
"protocol": string(envelope.ProtocolYutongMQTT),
"status": string(status),
}, float64(elapsed.Milliseconds()))
}
func (c *MQTTClient) recordPublishMetric(kind string, status string) {
if c.cfg.Metrics == nil {
return

View File

@@ -79,6 +79,7 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
`vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1`,
`vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="resolved"} 1`,
`vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 1`,
`vehicle_gateway_frame_duration_ms{protocol="YUTONG_MQTT",status="OK"}`,
} {
if !strings.Contains(text, want) {
t.Fatalf("metrics missing %s:\n%s", want, text)

View File

@@ -195,6 +195,12 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
}
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string, state *connectionState) {
started := time.Now()
frameStatus := envelope.ParseBadFrame
defer func() {
s.recordFrameDuration(frameStatus, time.Since(started))
}()
frameCtx, cancelFrame := context.WithTimeout(context.WithoutCancel(ctx), frameOperationTimeout)
defer cancelFrame()
@@ -229,6 +235,7 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
}
enrichConnectionPlatform(&env, state)
}
frameStatus = env.ParseStatus
s.recordFrameMetric(env.ParseStatus)
if err := s.sink.PublishRaw(frameCtx, env); err != nil {
@@ -307,6 +314,16 @@ func (s *TCPServer) recordFrameMetric(status envelope.ParseStatus) {
})
}
func (s *TCPServer) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_gateway_frame_duration_ms", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": string(status),
}, float64(elapsed.Milliseconds()))
}
func (s *TCPServer) recordPublishMetric(kind string, status string) {
if s.metrics == nil {
return

View File

@@ -81,6 +81,7 @@ func TestTCPServerRecordsFrameMetrics(t *testing.T) {
`vehicle_gateway_frames_total{protocol="JT808",status="OK"} 1`,
`vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 1`,
`vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 1`,
`vehicle_gateway_frame_duration_ms{protocol="JT808",status="OK"}`,
} {
if !strings.Contains(text, want) {
t.Fatalf("metrics missing %s:\n%s", want, text)