From 018015f9dec894e5f59beb05ecbe2bfd9108964a Mon Sep 17 00:00:00 2001 From: lingniu Date: Fri, 3 Jul 2026 18:58:37 +0800 Subject: [PATCH] feat(go): expose bridge batch pressure metrics --- docs/ops/100k-capacity-baseline.md | 2 + docs/ops/go-service-observability.md | 3 ++ docs/ops/go-vehicle-ingest-memory.md | 14 ++++++ docs/ops/vehicle-ingest-runbook.md | 2 + .../cmd/nats-kafka-bridge/main.go | 29 ++++++++++++ .../cmd/nats-kafka-bridge/main_test.go | 47 +++++++++++++++++++ 6 files changed, 97 insertions(+) diff --git a/docs/ops/100k-capacity-baseline.md b/docs/ops/100k-capacity-baseline.md index 8bc34105..e35cfb3a 100644 --- a/docs/ops/100k-capacity-baseline.md +++ b/docs/ops/100k-capacity-baseline.md @@ -73,6 +73,8 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 | History writer | `vehicle_history_batch_flush_duration_ms{status}` | flush 延迟不持续上升 | | NATS bridge | `vehicle_bridge_nats_consumer_ack_pending` | 稳态为 0 | | NATS bridge | `vehicle_bridge_nats_consumer_pending` | burst 后下降 | +| NATS bridge | `vehicle_bridge_batch_pending_messages` | 稳态接近 0，burst 后下降 | +| NATS bridge | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 不持续上升 | | Kafka consumers | `vehicle_*_kafka_lag` | 稳态为 0，burst 后下降 | | Realtime Redis | `vehicle_realtime_store_update_duration_ms{store="redis"}` | 毫秒级 | | Realtime MySQL | async queue depth | 不持续增长 | diff --git a/docs/ops/go-service-observability.md b/docs/ops/go-service-observability.md index 7f553d56..34ca8313 100644 --- a/docs/ops/go-service-observability.md +++ b/docs/ops/go-service-observability.md @@ -43,6 +43,9 @@ curl -fsS http://127.0.0.1:20200/readyz | `vehicle_bridge_nats_consumer_pending` | JetStream messages pending for the bridge durable consumer. | | `vehicle_bridge_nats_consumer_ack_pending` | JetStream messages delivered to bridge but not yet acked. | | `vehicle_bridge_nats_consumer_waiting` | Pull requests waiting on the bridge durable consumer. | +| `vehicle_bridge_batch_pending_messages` | NATS messages fetched by the bridge but not yet written to Kafka and acked. | +| `vehicle_bridge_batch_pending_kafka_messages` | Kafka messages prepared for the current bridge batch. | +| `vehicle_bridge_batch_duration_ms_histogram` | Bridge batch duration histogram covering Kafka write and NATS ack. Labels: `status`. | | `vehicle_history_writes_total` | TDengine history writes. Labels: `topic`, `status`. | | `vehicle_history_batch_pending_messages` | Messages fetched by history writer but not yet flushed and committed. | | `vehicle_history_batch_pending_rows` | Parsed raw envelopes waiting in the current TDengine batch. | diff --git a/docs/ops/go-vehicle-ingest-memory.md b/docs/ops/go-vehicle-ingest-memory.md index 1f14c31b..0efe2ec6 100644 --- a/docs/ops/go-vehicle-ingest-memory.md +++ b/docs/ops/go-vehicle-ingest-memory.md @@ -406,6 +406,20 @@ vehicle_async_sink_queue_depth{sink} 这些指标是进入带帧率压测前的关键保护栏：如果 `queue_depth` 持续增长或 `enqueue_total{status="timeout"}` 增长，说明入口 publish 队列已成为瓶颈。 +### NATS Kafka bridge batch 指标 + +2026-07-03 已为 NATS -> Kafka bridge 增加 batch pending 和 duration histogram： + +```text +vehicle_bridge_batch_pending_messages +vehicle_bridge_batch_pending_kafka_messages +vehicle_bridge_batch_duration_ms_histogram_bucket{status,le} +vehicle_bridge_batch_duration_ms_histogram_count{status} +vehicle_bridge_batch_duration_ms_histogram_sum{status} +``` + +该耗时覆盖一批 NATS 消息路由到 Kafka、Kafka 同步写入、NATS ack 的完整过程。带帧率压测时，如果 pending 持续非 0 或 duration p99 持续上升，应优先排查 Kafka broker、bridge batch size、NATS ack-pending 和网络。 + ### Gateway frame duration histogram 2026-07-03 已为 Gateway 帧处理耗时增加 histogram： diff --git a/docs/ops/vehicle-ingest-runbook.md b/docs/ops/vehicle-ingest-runbook.md index 0f487987..b3bf3629 100644 --- a/docs/ops/vehicle-ingest-runbook.md +++ b/docs/ops/vehicle-ingest-runbook.md @@ -126,6 +126,8 @@ go run ./cmd/load-sim \ | `vehicle_history_batch_flush_duration_ms{status="ok"}` | 持续上升 | TDengine 写入延迟增加，可能需要降低 batch size 或扩容 TDengine。 | | `vehicle_bridge_nats_consumer_ack_pending` | 连续 2 分钟 `> 0` | 消息已投递给 bridge，但 Kafka 写入后未完成 ack。 | | `vehicle_bridge_nats_consumer_pending` | 持续增长且 `> 10000` | bridge 消费 NATS 的速度跟不上生产速度。 | +| `vehicle_bridge_batch_pending_messages` | 持续非 0 或 burst 后不回落 | bridge 已拉取 NATS 消息但尚未完成 Kafka 写入和 NATS ack。 | +| `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢，通常会先于 ack-pending 扩大。 | | Kafka lag | 连续 5 分钟增长或 `> 10000` | 下游 consumer 或存储存在瓶颈。 | | Writer 成功计数 | 入口增长但 writer 不增长 | bridge、Kafka、consumer 或存储链路断开。 | diff --git a/go/vehicle-gateway/cmd/nats-kafka-bridge/main.go b/go/vehicle-gateway/cmd/nats-kafka-bridge/main.go index 2bcad936..901581d6 100644 --- a/go/vehicle-gateway/cmd/nats-kafka-bridge/main.go +++ b/go/vehicle-gateway/cmd/nats-kafka-bridge/main.go @@ -240,20 +240,29 @@ func bridgeBatch(ctx context.Context, registry *metrics.Registry, writer kafkaBa if len(messages) == 0 { return nil } + started := time.Now() + status := "ok" + defer func() { + recordBridgeBatchDuration(registry, status, time.Since(started)) + recordBridgeBatchPending(registry, 0, 0) + }() kafkaMessages := make([]kafka.Message, 0, len(messages)) for _, message := range messages { addBridgeSubjectMetric(registry, "vehicle_bridge_messages_total", message.subject, "received") topic, ok := route[message.subject] if !ok || topic == "" { addBridgeSubjectMetric(registry, "vehicle_bridge_messages_total", message.subject, "route_error") + status = "error" return fmt.Errorf("kafka topic not configured for nats subject %q", message.subject) } kafkaMessages = append(kafkaMessages, kafkaMessage(topic, message.data)) } + recordBridgeBatchPending(registry, len(messages), len(kafkaMessages)) if err := writer.WriteMessages(ctx, kafkaMessages...); err != nil { for _, message := range kafkaMessages { addBridgeTopicMetric(registry, "vehicle_bridge_kafka_writes_total", message.Topic, "error") } + status = "error" return err } for _, message := range kafkaMessages { @@ -265,6 +274,7 @@ func bridgeBatch(ctx context.Context, registry *metrics.Registry, writer kafkaBa } if err := message.ack(); err != nil { addBridgeSubjectMetric(registry, "vehicle_bridge_nats_acks_total", message.subject, "error") + status = "error" return err } addBridgeSubjectMetric(registry, "vehicle_bridge_nats_acks_total", message.subject, "ok") @@ -272,6 +282,8 @@ func bridgeBatch(ctx context.Context, registry *metrics.Registry, writer kafkaBa return nil } +var bridgeBatchDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} + func addBridgeSubjectMetric(registry *metrics.Registry, name string, subject string, status string) { if registry == nil { return @@ -286,6 +298,23 @@ func addBridgeTopicMetric(registry *metrics.Registry, name string, topic string, registry.IncCounter(name, metrics.Labels{"topic": topic, "status": status}) } +func recordBridgeBatchPending(registry *metrics.Registry, messages int, kafkaMessages int) { + if registry == nil { + return + } + registry.SetGauge("vehicle_bridge_batch_pending_messages", nil, float64(messages)) + registry.SetGauge("vehicle_bridge_batch_pending_kafka_messages", nil, float64(kafkaMessages)) +} + +func recordBridgeBatchDuration(registry *metrics.Registry, status string, elapsed time.Duration) { + if registry == nil { + return + } + registry.ObserveHistogram("vehicle_bridge_batch_duration_ms_histogram", metrics.Labels{ + "status": status, + }, bridgeBatchDurationBucketsMS, float64(elapsed.Milliseconds())) +} + func recordNATSConsumerInfoMetrics(registry *metrics.Registry, cfg config, info *nats.ConsumerInfo) { if registry == nil || info == nil { return diff --git a/go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go b/go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go index c8fb6773..3356f966 100644 --- a/go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go +++ b/go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go @@ -91,6 +91,49 @@ func TestBridgeBatchRecordsMetrics(t *testing.T) { } } +func TestBridgeBatchExposesPendingAndDurationMetrics(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + writer := &recordingBridgeWriter{ + onWrite: func() { + text := registry.Render() + for _, want := range []string{ + `vehicle_bridge_batch_pending_messages 2`, + `vehicle_bridge_batch_pending_kafka_messages 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("pending bridge metric missing %s during write:\n%s", want, text) + } + } + }, + } + + err = bridgeBatch(context.Background(), registry, writer, []bridgeMessage{ + {subject: "vehicle.raw.go.gb32960.v1", data: payload, ack: func() error { return nil }}, + {subject: "vehicle.raw.go.gb32960.v1", data: payload, ack: func() error { return nil }}, + }, map[string]string{"vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1"}) + if err != nil { + t.Fatalf("bridgeBatch() error = %v", err) + } + + text := registry.Render() + for _, want := range []string{ + `vehicle_bridge_batch_pending_messages 0`, + `vehicle_bridge_batch_pending_kafka_messages 0`, + `vehicle_bridge_batch_duration_ms_histogram_bucket{le="+Inf",status="ok"} 1`, + `vehicle_bridge_batch_duration_ms_histogram_count{status="ok"} 1`, + `vehicle_bridge_batch_duration_ms_histogram_sum{status="ok"}`, + } { + if !strings.Contains(text, want) { + t.Fatalf("bridge batch metric missing %s:\n%s", want, text) + } + } +} + func TestRecordNATSConsumerInfoMetrics(t *testing.T) { registry := metrics.NewRegistry() cfg := config{NATSStream: "VEHICLE_INGEST", NATSDurable: "vehicle-kafka-bridge"} @@ -148,9 +191,13 @@ func TestLoadConfigIncludesUnifiedOnlyWhenExplicitlyConfigured(t *testing.T) { type recordingBridgeWriter struct { messages []kafka.Message err error + onWrite func() } func (w *recordingBridgeWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error { + if w.onWrite != nil { + w.onWrite() + } if w.err != nil { return w.err }