feat(go): expose gateway connection close metrics

This commit is contained in:
lingniu
2026-07-03 19:28:06 +08:00
parent d08bf17111
commit 06866b0fa7
6 changed files with 56 additions and 0 deletions

View File

@@ -60,6 +60,7 @@ systemd gateway 已配置 `LimitNOFILE=1048576`,需要持续保持。
| 层 | 指标 | 目标 |
| --- | --- | --- |
| Gateway | `vehicle_gateway_active_connections` | 能稳定到阶段目标连接数 |
| Gateway | `vehicle_gateway_connection_closes_total{reason}` | `read_error/extract_error/max_connections` 不异常增长 |
| Gateway | `vehicle_gateway_connection_rejections_total` | 容量目标内不增长 |
| Gateway async sink | `vehicle_async_sink_queue_depth{sink}` | 队列深度不持续增长 |
| Gateway async sink | `vehicle_async_sink_enqueue_total{sink,kind,status}` | `timeout/closed` 不增长 |

View File

@@ -34,6 +34,8 @@ curl -fsS http://127.0.0.1:20200/readyz
| Metric | Meaning |
| --- | --- |
| `vehicle_gateway_active_connections` | Current TCP connections by protocol. Labels: `protocol`. |
| `vehicle_gateway_connection_closes_total` | TCP connection closes by protocol and reason. Labels: `protocol`, `reason`. Reasons include `eof`, `read_timeout`, `read_error`, `extract_error`, `context_cancelled`, `max_connections`. |
| `vehicle_gateway_connection_rejections_total` | TCP connection rejections by protocol and reason. Labels: `protocol`, `reason`. |
| `vehicle_gateway_frames_total` | Protocol frames received and parsed by the gateway. Labels: `protocol`, `status`. |
| `vehicle_gateway_frame_duration_ms` | Last observed Gateway frame handling duration and histogram buckets for p95/p99. Labels: `protocol`, `status`. |
| `vehicle_gateway_publish_total` | Gateway publish result for raw events. Unified appears only when the compatibility stream is explicitly enabled. Labels: `protocol`, `kind`, `status`. |

View File

@@ -409,6 +409,16 @@ vehicle_async_sink_queue_depth{sink}
这些指标是进入带帧率压测前的关键保护栏:如果 `queue_depth` 持续增长、`enqueue_total{status="timeout"}` 增长,或 `vehicle_async_sink_publish_duration_ms_histogram_bucket` 的 p99 持续上升,说明入口 publish 队列或下游 NATS/Kafka 写入已成为瓶颈。
### Gateway connection close reason
2026-07-03 已为 TCP gateway 增加连接关闭原因计数:
```text
vehicle_gateway_connection_closes_total{protocol,reason}
```
当前 `reason` 包括 `eof``read_timeout``read_error``extract_error``context_cancelled``max_connections`。10W 长连接和带帧率压测时,应重点关注 `read_error``extract_error``max_connections` 是否异常增长;`read_timeout` 需要结合协议上报周期和在线数判断。
### NATS fast writer stage histogram
2026-07-03 已为 NATS fast writer 增加阶段耗时 histogram

View File

@@ -115,6 +115,9 @@ go run ./cmd/load-sim \
| --- | --- | --- |
| `/readyz` 非 ok | 立即处理 | 服务或依赖不可用。 |
| Gateway 活跃连接 | 预期有流量时某协议降为 `0` | 上游网络、监听端口或进程可能异常。 |
| `vehicle_gateway_connection_closes_total{reason="read_error"}` | 连续增长 | 入口 TCP 读失败,优先查网络、客户端断连和内核连接状态。 |
| `vehicle_gateway_connection_closes_total{reason="extract_error"}` | 连续增长 | 报文边界或协议提取异常,优先抽查 raw 日志和协议 extractor。 |
| `vehicle_gateway_connection_closes_total{reason="read_timeout"}` | 突然高于历史基线 | 车辆长时间无上报或链路空闲超时,需结合在线数和上游平台状态判断。 |
| `vehicle_gateway_frames_total{status!="OK"}` | 连续 2 分钟增长 | 解析器或上游报文质量异常。 |
| `vehicle_gateway_frame_duration_ms_histogram_bucket` | p99 连续 5 分钟超过容量目标 | Gateway parse、identity resolve、publish enqueue 或响应链路变慢。 |
| `vehicle_async_sink_queue_depth{sink="nats"}` | 持续增长且不回落 | Gateway 到 NATS/Kafka 的异步 publish 队列开始积压。 |

View File

@@ -146,6 +146,7 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error {
}()
default:
s.recordConnectionRejection("max_connections")
s.recordConnectionClose("max_connections")
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
_ = conn.Close()
}
@@ -172,6 +173,7 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
frames, remainder, extractErr := s.protocol.Extract(pending)
if extractErr != nil {
log.Warn("frame extraction failed", "error", extractErr)
s.recordConnectionClose("extract_error")
return
}
pending = remainder
@@ -181,17 +183,21 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
}
if err != nil {
if errors.Is(err, io.EOF) {
s.recordConnectionClose("eof")
return
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
log.Warn("tcp connection idle timeout")
s.recordConnectionClose("read_timeout")
return
}
log.Warn("tcp read failed", "error", err)
s.recordConnectionClose("read_error")
return
}
if ctx.Err() != nil {
s.recordConnectionClose("context_cancelled")
return
}
}
@@ -386,6 +392,16 @@ func (s *TCPServer) recordConnectionRejection(reason string) {
})
}
func (s *TCPServer) recordConnectionClose(reason string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_connection_closes_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"reason": reason,
})
}
func identityStatus(env envelope.FrameEnvelope) string {
if strings.TrimSpace(env.VIN) != "" {
return "resolved"

View File

@@ -164,6 +164,30 @@ func TestTCPServerRecordsConnectionRejectionMetric(t *testing.T) {
}
}
func TestTCPServerRecordsConnectionCloseReasonMetric(t *testing.T) {
registry := metrics.NewRegistry()
server, err := NewTCPServer(TCPServerConfig{
Protocol: TCPProtocol{
Protocol: envelope.ProtocolJT808,
Addr: ":0",
Extract: jt808.ExtractFrames,
Parse: jt808.ParseFrame,
},
Sink: &recordingSink{},
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
Metrics: registry,
})
if err != nil {
t.Fatalf("NewTCPServer() error = %v", err)
}
server.recordConnectionClose("read_timeout")
if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_connection_closes_total{protocol="JT808",reason="read_timeout"} 1`) {
t.Fatalf("connection close metric missing:\n%s", text)
}
}
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
good[len(good)-1] ^= 0xff