feat(go): expose gateway connection close metrics
This commit is contained in:
@@ -60,6 +60,7 @@ systemd gateway 已配置 `LimitNOFILE=1048576`,需要持续保持。
|
||||
| 层 | 指标 | 目标 |
|
||||
| --- | --- | --- |
|
||||
| Gateway | `vehicle_gateway_active_connections` | 能稳定到阶段目标连接数 |
|
||||
| Gateway | `vehicle_gateway_connection_closes_total{reason}` | `read_error/extract_error/max_connections` 不异常增长 |
|
||||
| Gateway | `vehicle_gateway_connection_rejections_total` | 容量目标内不增长 |
|
||||
| Gateway async sink | `vehicle_async_sink_queue_depth{sink}` | 队列深度不持续增长 |
|
||||
| Gateway async sink | `vehicle_async_sink_enqueue_total{sink,kind,status}` | `timeout/closed` 不增长 |
|
||||
|
||||
@@ -34,6 +34,8 @@ curl -fsS http://127.0.0.1:20200/readyz
|
||||
| Metric | Meaning |
|
||||
| --- | --- |
|
||||
| `vehicle_gateway_active_connections` | Current TCP connections by protocol. Labels: `protocol`. |
|
||||
| `vehicle_gateway_connection_closes_total` | TCP connection closes by protocol and reason. Labels: `protocol`, `reason`. Reasons include `eof`, `read_timeout`, `read_error`, `extract_error`, `context_cancelled`, `max_connections`. |
|
||||
| `vehicle_gateway_connection_rejections_total` | TCP connection rejections by protocol and reason. Labels: `protocol`, `reason`. |
|
||||
| `vehicle_gateway_frames_total` | Protocol frames received and parsed by the gateway. Labels: `protocol`, `status`. |
|
||||
| `vehicle_gateway_frame_duration_ms` | Last observed Gateway frame handling duration and histogram buckets for p95/p99. Labels: `protocol`, `status`. |
|
||||
| `vehicle_gateway_publish_total` | Gateway publish result for raw events. Unified appears only when the compatibility stream is explicitly enabled. Labels: `protocol`, `kind`, `status`. |
|
||||
|
||||
@@ -409,6 +409,16 @@ vehicle_async_sink_queue_depth{sink}
|
||||
|
||||
这些指标是进入带帧率压测前的关键保护栏:如果 `queue_depth` 持续增长、`enqueue_total{status="timeout"}` 增长,或 `vehicle_async_sink_publish_duration_ms_histogram_bucket` 的 p99 持续上升,说明入口 publish 队列或下游 NATS/Kafka 写入已成为瓶颈。
|
||||
|
||||
### Gateway connection close reason
|
||||
|
||||
2026-07-03 已为 TCP gateway 增加连接关闭原因计数:
|
||||
|
||||
```text
|
||||
vehicle_gateway_connection_closes_total{protocol,reason}
|
||||
```
|
||||
|
||||
当前 `reason` 包括 `eof`、`read_timeout`、`read_error`、`extract_error`、`context_cancelled`、`max_connections`。10W 长连接和带帧率压测时,应重点关注 `read_error`、`extract_error`、`max_connections` 是否异常增长;`read_timeout` 需要结合协议上报周期和在线数判断。
|
||||
|
||||
### NATS fast writer stage histogram
|
||||
|
||||
2026-07-03 已为 NATS fast writer 增加阶段耗时 histogram:
|
||||
|
||||
@@ -115,6 +115,9 @@ go run ./cmd/load-sim \
|
||||
| --- | --- | --- |
|
||||
| `/readyz` 非 ok | 立即处理 | 服务或依赖不可用。 |
|
||||
| Gateway 活跃连接 | 预期有流量时某协议降为 `0` | 上游网络、监听端口或进程可能异常。 |
|
||||
| `vehicle_gateway_connection_closes_total{reason="read_error"}` | 连续增长 | 入口 TCP 读失败,优先查网络、客户端断连和内核连接状态。 |
|
||||
| `vehicle_gateway_connection_closes_total{reason="extract_error"}` | 连续增长 | 报文边界或协议提取异常,优先抽查 raw 日志和协议 extractor。 |
|
||||
| `vehicle_gateway_connection_closes_total{reason="read_timeout"}` | 突然高于历史基线 | 车辆长时间无上报或链路空闲超时,需结合在线数和上游平台状态判断。 |
|
||||
| `vehicle_gateway_frames_total{status!="OK"}` | 连续 2 分钟增长 | 解析器或上游报文质量异常。 |
|
||||
| `vehicle_gateway_frame_duration_ms_histogram_bucket` | p99 连续 5 分钟超过容量目标 | Gateway parse、identity resolve、publish enqueue 或响应链路变慢。 |
|
||||
| `vehicle_async_sink_queue_depth{sink="nats"}` | 持续增长且不回落 | Gateway 到 NATS/Kafka 的异步 publish 队列开始积压。 |
|
||||
|
||||
@@ -146,6 +146,7 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error {
|
||||
}()
|
||||
default:
|
||||
s.recordConnectionRejection("max_connections")
|
||||
s.recordConnectionClose("max_connections")
|
||||
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
|
||||
_ = conn.Close()
|
||||
}
|
||||
@@ -172,6 +173,7 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
||||
frames, remainder, extractErr := s.protocol.Extract(pending)
|
||||
if extractErr != nil {
|
||||
log.Warn("frame extraction failed", "error", extractErr)
|
||||
s.recordConnectionClose("extract_error")
|
||||
return
|
||||
}
|
||||
pending = remainder
|
||||
@@ -181,17 +183,21 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
s.recordConnectionClose("eof")
|
||||
return
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
log.Warn("tcp connection idle timeout")
|
||||
s.recordConnectionClose("read_timeout")
|
||||
return
|
||||
}
|
||||
log.Warn("tcp read failed", "error", err)
|
||||
s.recordConnectionClose("read_error")
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
s.recordConnectionClose("context_cancelled")
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -386,6 +392,16 @@ func (s *TCPServer) recordConnectionRejection(reason string) {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *TCPServer) recordConnectionClose(reason string) {
|
||||
if s.metrics == nil {
|
||||
return
|
||||
}
|
||||
s.metrics.IncCounter("vehicle_gateway_connection_closes_total", metrics.Labels{
|
||||
"protocol": string(s.protocol.Protocol),
|
||||
"reason": reason,
|
||||
})
|
||||
}
|
||||
|
||||
func identityStatus(env envelope.FrameEnvelope) string {
|
||||
if strings.TrimSpace(env.VIN) != "" {
|
||||
return "resolved"
|
||||
|
||||
@@ -164,6 +164,30 @@ func TestTCPServerRecordsConnectionRejectionMetric(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPServerRecordsConnectionCloseReasonMetric(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
server, err := NewTCPServer(TCPServerConfig{
|
||||
Protocol: TCPProtocol{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Addr: ":0",
|
||||
Extract: jt808.ExtractFrames,
|
||||
Parse: jt808.ParseFrame,
|
||||
},
|
||||
Sink: &recordingSink{},
|
||||
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||
Metrics: registry,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewTCPServer() error = %v", err)
|
||||
}
|
||||
|
||||
server.recordConnectionClose("read_timeout")
|
||||
|
||||
if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_connection_closes_total{protocol="JT808",reason="read_timeout"} 1`) {
|
||||
t.Fatalf("connection close metric missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
|
||||
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||
good[len(good)-1] ^= 0xff
|
||||
|
||||
Reference in New Issue
Block a user