feat(go): improve realtime pipeline resilience
This commit is contained in:
@@ -179,6 +179,12 @@ func (c *MQTTClient) buildTLSConfig() (*tls.Config, error) {
|
||||
}
|
||||
|
||||
func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []byte) {
|
||||
started := time.Now()
|
||||
frameStatus := envelope.ParseBadFrame
|
||||
defer func() {
|
||||
c.recordFrameDuration(frameStatus, time.Since(started))
|
||||
}()
|
||||
|
||||
messageCtx, cancelMessage := context.WithTimeout(context.WithoutCancel(ctx), frameOperationTimeout)
|
||||
defer cancelMessage()
|
||||
|
||||
@@ -214,6 +220,7 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []
|
||||
c.recordIdentityMetric(identityStatus(env))
|
||||
}
|
||||
}
|
||||
frameStatus = env.ParseStatus
|
||||
c.recordFrameMetric(env.ParseStatus)
|
||||
if err := c.cfg.Sink.PublishRaw(messageCtx, env); err != nil {
|
||||
c.recordPublishMetric("raw", "error")
|
||||
@@ -252,6 +259,16 @@ func (c *MQTTClient) recordFrameMetric(status envelope.ParseStatus) {
|
||||
})
|
||||
}
|
||||
|
||||
func (c *MQTTClient) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
|
||||
if c.cfg.Metrics == nil {
|
||||
return
|
||||
}
|
||||
c.cfg.Metrics.SetGauge("vehicle_gateway_frame_duration_ms", metrics.Labels{
|
||||
"protocol": string(envelope.ProtocolYutongMQTT),
|
||||
"status": string(status),
|
||||
}, float64(elapsed.Milliseconds()))
|
||||
}
|
||||
|
||||
func (c *MQTTClient) recordPublishMetric(kind string, status string) {
|
||||
if c.cfg.Metrics == nil {
|
||||
return
|
||||
|
||||
@@ -79,6 +79,7 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
|
||||
`vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1`,
|
||||
`vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="resolved"} 1`,
|
||||
`vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 1`,
|
||||
`vehicle_gateway_frame_duration_ms{protocol="YUTONG_MQTT",status="OK"}`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("metrics missing %s:\n%s", want, text)
|
||||
|
||||
@@ -195,6 +195,12 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
||||
}
|
||||
|
||||
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string, state *connectionState) {
|
||||
started := time.Now()
|
||||
frameStatus := envelope.ParseBadFrame
|
||||
defer func() {
|
||||
s.recordFrameDuration(frameStatus, time.Since(started))
|
||||
}()
|
||||
|
||||
frameCtx, cancelFrame := context.WithTimeout(context.WithoutCancel(ctx), frameOperationTimeout)
|
||||
defer cancelFrame()
|
||||
|
||||
@@ -229,6 +235,7 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
|
||||
}
|
||||
enrichConnectionPlatform(&env, state)
|
||||
}
|
||||
frameStatus = env.ParseStatus
|
||||
s.recordFrameMetric(env.ParseStatus)
|
||||
|
||||
if err := s.sink.PublishRaw(frameCtx, env); err != nil {
|
||||
@@ -307,6 +314,16 @@ func (s *TCPServer) recordFrameMetric(status envelope.ParseStatus) {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *TCPServer) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
|
||||
if s.metrics == nil {
|
||||
return
|
||||
}
|
||||
s.metrics.SetGauge("vehicle_gateway_frame_duration_ms", metrics.Labels{
|
||||
"protocol": string(s.protocol.Protocol),
|
||||
"status": string(status),
|
||||
}, float64(elapsed.Milliseconds()))
|
||||
}
|
||||
|
||||
func (s *TCPServer) recordPublishMetric(kind string, status string) {
|
||||
if s.metrics == nil {
|
||||
return
|
||||
|
||||
@@ -81,6 +81,7 @@ func TestTCPServerRecordsFrameMetrics(t *testing.T) {
|
||||
`vehicle_gateway_frames_total{protocol="JT808",status="OK"} 1`,
|
||||
`vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 1`,
|
||||
`vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 1`,
|
||||
`vehicle_gateway_frame_duration_ms{protocol="JT808",status="OK"}`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("metrics missing %s:\n%s", want, text)
|
||||
|
||||
Reference in New Issue
Block a user