diff --git a/docs/ops/100k-capacity-baseline.md b/docs/ops/100k-capacity-baseline.md index 3d8a19f9..13a529a4 100644 --- a/docs/ops/100k-capacity-baseline.md +++ b/docs/ops/100k-capacity-baseline.md @@ -60,6 +60,7 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 | 层 | 指标 | 目标 | | --- | --- | --- | | Gateway | `vehicle_gateway_active_connections` | 能稳定到阶段目标连接数 | +| Gateway | `vehicle_gateway_connection_closes_total{reason}` | `read_error/extract_error/max_connections` 不异常增长 | | Gateway | `vehicle_gateway_connection_rejections_total` | 容量目标内不增长 | | Gateway async sink | `vehicle_async_sink_queue_depth{sink}` | 队列深度不持续增长 | | Gateway async sink | `vehicle_async_sink_enqueue_total{sink,kind,status}` | `timeout/closed` 不增长 | diff --git a/docs/ops/go-service-observability.md b/docs/ops/go-service-observability.md index 6b5f4c7b..3f37ebf0 100644 --- a/docs/ops/go-service-observability.md +++ b/docs/ops/go-service-observability.md @@ -34,6 +34,8 @@ curl -fsS http://127.0.0.1:20200/readyz | Metric | Meaning | | --- | --- | | `vehicle_gateway_active_connections` | Current TCP connections by protocol. Labels: `protocol`. | +| `vehicle_gateway_connection_closes_total` | TCP connection closes by protocol and reason. Labels: `protocol`, `reason`. Reasons include `eof`, `read_timeout`, `read_error`, `extract_error`, `context_cancelled`, `max_connections`. | +| `vehicle_gateway_connection_rejections_total` | TCP connection rejections by protocol and reason. Labels: `protocol`, `reason`. | | `vehicle_gateway_frames_total` | Protocol frames received and parsed by the gateway. Labels: `protocol`, `status`. | | `vehicle_gateway_frame_duration_ms` | Last observed Gateway frame handling duration and histogram buckets for p95/p99. Labels: `protocol`, `status`. | | `vehicle_gateway_publish_total` | Gateway publish result for raw events. Unified appears only when the compatibility stream is explicitly enabled. Labels: `protocol`, `kind`, `status`. | diff --git a/docs/ops/go-vehicle-ingest-memory.md b/docs/ops/go-vehicle-ingest-memory.md index b3855c86..82af8ca8 100644 --- a/docs/ops/go-vehicle-ingest-memory.md +++ b/docs/ops/go-vehicle-ingest-memory.md @@ -409,6 +409,16 @@ vehicle_async_sink_queue_depth{sink} 这些指标是进入带帧率压测前的关键保护栏：如果 `queue_depth` 持续增长、`enqueue_total{status="timeout"}` 增长，或 `vehicle_async_sink_publish_duration_ms_histogram_bucket` 的 p99 持续上升，说明入口 publish 队列或下游 NATS/Kafka 写入已成为瓶颈。 +### Gateway connection close reason + +2026-07-03 已为 TCP gateway 增加连接关闭原因计数： + +```text +vehicle_gateway_connection_closes_total{protocol,reason} +``` + +当前 `reason` 包括 `eof`、`read_timeout`、`read_error`、`extract_error`、`context_cancelled`、`max_connections`。10W 长连接和带帧率压测时，应重点关注 `read_error`、`extract_error`、`max_connections` 是否异常增长；`read_timeout` 需要结合协议上报周期和在线数判断。 + ### NATS fast writer stage histogram 2026-07-03 已为 NATS fast writer 增加阶段耗时 histogram： diff --git a/docs/ops/vehicle-ingest-runbook.md b/docs/ops/vehicle-ingest-runbook.md index dcc76f1e..7ba03b3b 100644 --- a/docs/ops/vehicle-ingest-runbook.md +++ b/docs/ops/vehicle-ingest-runbook.md @@ -115,6 +115,9 @@ go run ./cmd/load-sim \ | --- | --- | --- | | `/readyz` 非 ok | 立即处理 | 服务或依赖不可用。 | | Gateway 活跃连接 | 预期有流量时某协议降为 `0` | 上游网络、监听端口或进程可能异常。 | +| `vehicle_gateway_connection_closes_total{reason="read_error"}` | 连续增长 | 入口 TCP 读失败，优先查网络、客户端断连和内核连接状态。 | +| `vehicle_gateway_connection_closes_total{reason="extract_error"}` | 连续增长 | 报文边界或协议提取异常，优先抽查 raw 日志和协议 extractor。 | +| `vehicle_gateway_connection_closes_total{reason="read_timeout"}` | 突然高于历史基线 | 车辆长时间无上报或链路空闲超时，需结合在线数和上游平台状态判断。 | | `vehicle_gateway_frames_total{status!="OK"}` | 连续 2 分钟增长 | 解析器或上游报文质量异常。 | | `vehicle_gateway_frame_duration_ms_histogram_bucket` | p99 连续 5 分钟超过容量目标 | Gateway parse、identity resolve、publish enqueue 或响应链路变慢。 | | `vehicle_async_sink_queue_depth{sink="nats"}` | 持续增长且不回落 | Gateway 到 NATS/Kafka 的异步 publish 队列开始积压。 | diff --git a/go/vehicle-gateway/internal/gateway/tcp_server.go b/go/vehicle-gateway/internal/gateway/tcp_server.go index eadd6d47..615fc23d 100644 --- a/go/vehicle-gateway/internal/gateway/tcp_server.go +++ b/go/vehicle-gateway/internal/gateway/tcp_server.go @@ -146,6 +146,7 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error { }() default: s.recordConnectionRejection("max_connections") + s.recordConnectionClose("max_connections") s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String()) _ = conn.Close() } @@ -172,6 +173,7 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) { frames, remainder, extractErr := s.protocol.Extract(pending) if extractErr != nil { log.Warn("frame extraction failed", "error", extractErr) + s.recordConnectionClose("extract_error") return } pending = remainder @@ -181,17 +183,21 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) { } if err != nil { if errors.Is(err, io.EOF) { + s.recordConnectionClose("eof") return } var netErr net.Error if errors.As(err, &netErr) && netErr.Timeout() { log.Warn("tcp connection idle timeout") + s.recordConnectionClose("read_timeout") return } log.Warn("tcp read failed", "error", err) + s.recordConnectionClose("read_error") return } if ctx.Err() != nil { + s.recordConnectionClose("context_cancelled") return } } @@ -386,6 +392,16 @@ func (s *TCPServer) recordConnectionRejection(reason string) { }) } +func (s *TCPServer) recordConnectionClose(reason string) { + if s.metrics == nil { + return + } + s.metrics.IncCounter("vehicle_gateway_connection_closes_total", metrics.Labels{ + "protocol": string(s.protocol.Protocol), + "reason": reason, + }) +} + func identityStatus(env envelope.FrameEnvelope) string { if strings.TrimSpace(env.VIN) != "" { return "resolved" diff --git a/go/vehicle-gateway/internal/gateway/tcp_server_test.go b/go/vehicle-gateway/internal/gateway/tcp_server_test.go index 3220e007..146e819c 100644 --- a/go/vehicle-gateway/internal/gateway/tcp_server_test.go +++ b/go/vehicle-gateway/internal/gateway/tcp_server_test.go @@ -164,6 +164,30 @@ func TestTCPServerRecordsConnectionRejectionMetric(t *testing.T) { } } +func TestTCPServerRecordsConnectionCloseReasonMetric(t *testing.T) { + registry := metrics.NewRegistry() + server, err := NewTCPServer(TCPServerConfig{ + Protocol: TCPProtocol{ + Protocol: envelope.ProtocolJT808, + Addr: ":0", + Extract: jt808.ExtractFrames, + Parse: jt808.ParseFrame, + }, + Sink: &recordingSink{}, + Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)), + Metrics: registry, + }) + if err != nil { + t.Fatalf("NewTCPServer() error = %v", err) + } + + server.recordConnectionClose("read_timeout") + + if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_connection_closes_total{protocol="JT808",reason="read_timeout"} 1`) { + t.Fatalf("connection close metric missing:\n%s", text) + } +} + func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) { good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil) good[len(good)-1] ^= 0xff