feat(go): expose gateway connection close metrics
This commit is contained in:
@@ -60,6 +60,7 @@ systemd gateway 已配置 `LimitNOFILE=1048576`,需要持续保持。
|
|||||||
| 层 | 指标 | 目标 |
|
| 层 | 指标 | 目标 |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Gateway | `vehicle_gateway_active_connections` | 能稳定到阶段目标连接数 |
|
| Gateway | `vehicle_gateway_active_connections` | 能稳定到阶段目标连接数 |
|
||||||
|
| Gateway | `vehicle_gateway_connection_closes_total{reason}` | `read_error/extract_error/max_connections` 不异常增长 |
|
||||||
| Gateway | `vehicle_gateway_connection_rejections_total` | 容量目标内不增长 |
|
| Gateway | `vehicle_gateway_connection_rejections_total` | 容量目标内不增长 |
|
||||||
| Gateway async sink | `vehicle_async_sink_queue_depth{sink}` | 队列深度不持续增长 |
|
| Gateway async sink | `vehicle_async_sink_queue_depth{sink}` | 队列深度不持续增长 |
|
||||||
| Gateway async sink | `vehicle_async_sink_enqueue_total{sink,kind,status}` | `timeout/closed` 不增长 |
|
| Gateway async sink | `vehicle_async_sink_enqueue_total{sink,kind,status}` | `timeout/closed` 不增长 |
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ curl -fsS http://127.0.0.1:20200/readyz
|
|||||||
| Metric | Meaning |
|
| Metric | Meaning |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `vehicle_gateway_active_connections` | Current TCP connections by protocol. Labels: `protocol`. |
|
| `vehicle_gateway_active_connections` | Current TCP connections by protocol. Labels: `protocol`. |
|
||||||
|
| `vehicle_gateway_connection_closes_total` | TCP connection closes by protocol and reason. Labels: `protocol`, `reason`. Reasons include `eof`, `read_timeout`, `read_error`, `extract_error`, `context_cancelled`, `max_connections`. |
|
||||||
|
| `vehicle_gateway_connection_rejections_total` | TCP connection rejections by protocol and reason. Labels: `protocol`, `reason`. |
|
||||||
| `vehicle_gateway_frames_total` | Protocol frames received and parsed by the gateway. Labels: `protocol`, `status`. |
|
| `vehicle_gateway_frames_total` | Protocol frames received and parsed by the gateway. Labels: `protocol`, `status`. |
|
||||||
| `vehicle_gateway_frame_duration_ms` | Last observed Gateway frame handling duration and histogram buckets for p95/p99. Labels: `protocol`, `status`. |
|
| `vehicle_gateway_frame_duration_ms` | Last observed Gateway frame handling duration and histogram buckets for p95/p99. Labels: `protocol`, `status`. |
|
||||||
| `vehicle_gateway_publish_total` | Gateway publish result for raw events. Unified appears only when the compatibility stream is explicitly enabled. Labels: `protocol`, `kind`, `status`. |
|
| `vehicle_gateway_publish_total` | Gateway publish result for raw events. Unified appears only when the compatibility stream is explicitly enabled. Labels: `protocol`, `kind`, `status`. |
|
||||||
|
|||||||
@@ -409,6 +409,16 @@ vehicle_async_sink_queue_depth{sink}
|
|||||||
|
|
||||||
这些指标是进入带帧率压测前的关键保护栏:如果 `queue_depth` 持续增长、`enqueue_total{status="timeout"}` 增长,或 `vehicle_async_sink_publish_duration_ms_histogram_bucket` 的 p99 持续上升,说明入口 publish 队列或下游 NATS/Kafka 写入已成为瓶颈。
|
这些指标是进入带帧率压测前的关键保护栏:如果 `queue_depth` 持续增长、`enqueue_total{status="timeout"}` 增长,或 `vehicle_async_sink_publish_duration_ms_histogram_bucket` 的 p99 持续上升,说明入口 publish 队列或下游 NATS/Kafka 写入已成为瓶颈。
|
||||||
|
|
||||||
|
### Gateway connection close reason
|
||||||
|
|
||||||
|
2026-07-03 已为 TCP gateway 增加连接关闭原因计数:
|
||||||
|
|
||||||
|
```text
|
||||||
|
vehicle_gateway_connection_closes_total{protocol,reason}
|
||||||
|
```
|
||||||
|
|
||||||
|
当前 `reason` 包括 `eof`、`read_timeout`、`read_error`、`extract_error`、`context_cancelled`、`max_connections`。10W 长连接和带帧率压测时,应重点关注 `read_error`、`extract_error`、`max_connections` 是否异常增长;`read_timeout` 需要结合协议上报周期和在线数判断。
|
||||||
|
|
||||||
### NATS fast writer stage histogram
|
### NATS fast writer stage histogram
|
||||||
|
|
||||||
2026-07-03 已为 NATS fast writer 增加阶段耗时 histogram:
|
2026-07-03 已为 NATS fast writer 增加阶段耗时 histogram:
|
||||||
|
|||||||
@@ -115,6 +115,9 @@ go run ./cmd/load-sim \
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `/readyz` 非 ok | 立即处理 | 服务或依赖不可用。 |
|
| `/readyz` 非 ok | 立即处理 | 服务或依赖不可用。 |
|
||||||
| Gateway 活跃连接 | 预期有流量时某协议降为 `0` | 上游网络、监听端口或进程可能异常。 |
|
| Gateway 活跃连接 | 预期有流量时某协议降为 `0` | 上游网络、监听端口或进程可能异常。 |
|
||||||
|
| `vehicle_gateway_connection_closes_total{reason="read_error"}` | 连续增长 | 入口 TCP 读失败,优先查网络、客户端断连和内核连接状态。 |
|
||||||
|
| `vehicle_gateway_connection_closes_total{reason="extract_error"}` | 连续增长 | 报文边界或协议提取异常,优先抽查 raw 日志和协议 extractor。 |
|
||||||
|
| `vehicle_gateway_connection_closes_total{reason="read_timeout"}` | 突然高于历史基线 | 车辆长时间无上报或链路空闲超时,需结合在线数和上游平台状态判断。 |
|
||||||
| `vehicle_gateway_frames_total{status!="OK"}` | 连续 2 分钟增长 | 解析器或上游报文质量异常。 |
|
| `vehicle_gateway_frames_total{status!="OK"}` | 连续 2 分钟增长 | 解析器或上游报文质量异常。 |
|
||||||
| `vehicle_gateway_frame_duration_ms_histogram_bucket` | p99 连续 5 分钟超过容量目标 | Gateway parse、identity resolve、publish enqueue 或响应链路变慢。 |
|
| `vehicle_gateway_frame_duration_ms_histogram_bucket` | p99 连续 5 分钟超过容量目标 | Gateway parse、identity resolve、publish enqueue 或响应链路变慢。 |
|
||||||
| `vehicle_async_sink_queue_depth{sink="nats"}` | 持续增长且不回落 | Gateway 到 NATS/Kafka 的异步 publish 队列开始积压。 |
|
| `vehicle_async_sink_queue_depth{sink="nats"}` | 持续增长且不回落 | Gateway 到 NATS/Kafka 的异步 publish 队列开始积压。 |
|
||||||
|
|||||||
@@ -146,6 +146,7 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error {
|
|||||||
}()
|
}()
|
||||||
default:
|
default:
|
||||||
s.recordConnectionRejection("max_connections")
|
s.recordConnectionRejection("max_connections")
|
||||||
|
s.recordConnectionClose("max_connections")
|
||||||
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
|
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
|
||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
}
|
}
|
||||||
@@ -172,6 +173,7 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
|||||||
frames, remainder, extractErr := s.protocol.Extract(pending)
|
frames, remainder, extractErr := s.protocol.Extract(pending)
|
||||||
if extractErr != nil {
|
if extractErr != nil {
|
||||||
log.Warn("frame extraction failed", "error", extractErr)
|
log.Warn("frame extraction failed", "error", extractErr)
|
||||||
|
s.recordConnectionClose("extract_error")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
pending = remainder
|
pending = remainder
|
||||||
@@ -181,17 +183,21 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
|||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, io.EOF) {
|
if errors.Is(err, io.EOF) {
|
||||||
|
s.recordConnectionClose("eof")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var netErr net.Error
|
var netErr net.Error
|
||||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||||
log.Warn("tcp connection idle timeout")
|
log.Warn("tcp connection idle timeout")
|
||||||
|
s.recordConnectionClose("read_timeout")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
log.Warn("tcp read failed", "error", err)
|
log.Warn("tcp read failed", "error", err)
|
||||||
|
s.recordConnectionClose("read_error")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
|
s.recordConnectionClose("context_cancelled")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -386,6 +392,16 @@ func (s *TCPServer) recordConnectionRejection(reason string) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *TCPServer) recordConnectionClose(reason string) {
|
||||||
|
if s.metrics == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.metrics.IncCounter("vehicle_gateway_connection_closes_total", metrics.Labels{
|
||||||
|
"protocol": string(s.protocol.Protocol),
|
||||||
|
"reason": reason,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func identityStatus(env envelope.FrameEnvelope) string {
|
func identityStatus(env envelope.FrameEnvelope) string {
|
||||||
if strings.TrimSpace(env.VIN) != "" {
|
if strings.TrimSpace(env.VIN) != "" {
|
||||||
return "resolved"
|
return "resolved"
|
||||||
|
|||||||
@@ -164,6 +164,30 @@ func TestTCPServerRecordsConnectionRejectionMetric(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTCPServerRecordsConnectionCloseReasonMetric(t *testing.T) {
|
||||||
|
registry := metrics.NewRegistry()
|
||||||
|
server, err := NewTCPServer(TCPServerConfig{
|
||||||
|
Protocol: TCPProtocol{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
Addr: ":0",
|
||||||
|
Extract: jt808.ExtractFrames,
|
||||||
|
Parse: jt808.ParseFrame,
|
||||||
|
},
|
||||||
|
Sink: &recordingSink{},
|
||||||
|
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||||
|
Metrics: registry,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewTCPServer() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
server.recordConnectionClose("read_timeout")
|
||||||
|
|
||||||
|
if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_connection_closes_total{protocol="JT808",reason="read_timeout"} 1`) {
|
||||||
|
t.Fatalf("connection close metric missing:\n%s", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
|
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
|
||||||
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
|
||||||
good[len(good)-1] ^= 0xff
|
good[len(good)-1] ^= 0xff
|
||||||
|
|||||||
Reference in New Issue
Block a user