feat(go): expose fast writer nats consumer metrics

docs/ops/100k-capacity-baseline.md

@@ -76,6 +76,8 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。
 | NATS bridge | `vehicle_bridge_nats_consumer_pending` | burst 后下降 |
 | NATS bridge | `vehicle_bridge_batch_pending_messages` | 稳态接近 0，burst 后下降 |
 | NATS bridge | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 不持续上升 |
+| NATS fast writer | `vehicle_fast_writer_nats_consumer_ack_pending` | 稳态为 0 |
+| NATS fast writer | `vehicle_fast_writer_nats_consumer_pending` | burst 后下降 |
 | NATS fast writer | `vehicle_fast_writer_batch_pending_messages` | 稳态接近 0，burst 后下降 |
 | NATS fast writer | `vehicle_fast_writer_batch_pending_envelopes` | 稳态接近 0，burst 后下降 |
 | NATS fast writer | `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | TDengine/Redis/ack 各阶段 p99 不持续上升 |

docs/ops/go-service-observability.md

@@ -47,6 +47,9 @@ curl -fsS http://127.0.0.1:20200/readyz
 | `vehicle_bridge_batch_pending_messages` | NATS messages fetched by the bridge but not yet written to Kafka and acked. |
 | `vehicle_bridge_batch_pending_kafka_messages` | Kafka messages prepared for the current bridge batch. |
 | `vehicle_bridge_batch_duration_ms_histogram` | Bridge batch duration histogram covering Kafka write and NATS ack. Labels: `status`. |
+| `vehicle_fast_writer_nats_consumer_pending` | JetStream messages pending for the fast writer durable consumer. |
+| `vehicle_fast_writer_nats_consumer_ack_pending` | JetStream messages delivered to fast writer but not yet acked. |
+| `vehicle_fast_writer_nats_consumer_waiting` | Pull requests waiting on the fast writer durable consumer. |
 | `vehicle_fast_writer_batch_pending_messages` | NATS fast writer messages fetched but not yet written to TDengine/Redis and acked. |
 | `vehicle_fast_writer_batch_pending_envelopes` | Valid parsed envelopes in the current NATS fast writer batch. |
 | `vehicle_fast_writer_stage_duration_ms_histogram` | Fast writer stage duration histogram for TDengine, Redis, and NATS ack. Labels: `subject`, `stage`, `status`. |

docs/ops/go-vehicle-ingest-memory.md

@@ -426,11 +426,14 @@ vehicle_fast_writer_stage_duration_ms_histogram_sum{subject,stage,status}
 同时新增 batch pending gauge：
 
 ```text
+vehicle_fast_writer_nats_consumer_pending{stream,consumer}
+vehicle_fast_writer_nats_consumer_ack_pending{stream,consumer}
+vehicle_fast_writer_nats_consumer_waiting{stream,consumer}
 vehicle_fast_writer_batch_pending_messages
 vehicle_fast_writer_batch_pending_envelopes
 ```
 
-这两个指标表示 fast-writer 已拉取但尚未完成 TDengine/Redis/ack 的批内压力。带帧率压测时它们应在稳态接近 0，burst 后能够回落；如果持续非 0，需要结合 `vehicle_fast_writer_stage_duration_ms_histogram` 判断卡在 TDengine、Redis 还是 ack。
+`vehicle_fast_writer_nats_consumer_*` 表示 JetStream durable consumer 层面的 backlog/ack-pending；`vehicle_fast_writer_batch_pending_*` 表示 fast-writer 已拉取但尚未完成 TDengine/Redis/ack 的批内压力。带帧率压测时它们应在稳态接近 0，burst 后能够回落；如果持续非 0，需要结合 `vehicle_fast_writer_stage_duration_ms_histogram` 判断卡在 NATS 拉取、TDengine、Redis 还是 ack。
 
 ### NATS Kafka bridge batch 指标
 

docs/ops/vehicle-ingest-runbook.md

@@ -129,6 +129,8 @@ go run ./cmd/load-sim \
 | `vehicle_bridge_nats_consumer_pending` | 持续增长且 `> 10000` | bridge 消费 NATS 的速度跟不上生产速度。 |
 | `vehicle_bridge_batch_pending_messages` | 持续非 0 或 burst 后不回落 | bridge 已拉取 NATS 消息但尚未完成 Kafka 写入和 NATS ack。 |
 | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢，通常会先于 ack-pending 扩大。 |
+| `vehicle_fast_writer_nats_consumer_ack_pending` | 连续 2 分钟 `> 0` | 消息已投递给 fast-writer，但 TDengine/Redis 写入后未完成 ack。 |
+| `vehicle_fast_writer_nats_consumer_pending` | 持续增长且 `> 10000` | fast-writer 消费 NATS 的速度跟不上入口写入速度。 |
 | `vehicle_fast_writer_batch_pending_messages` | 持续非 0 或 burst 后不回落 | fast-writer 已拉取 NATS 消息但尚未完成 TDengine/Redis 写入和 ack。 |
 | `vehicle_fast_writer_batch_pending_envelopes` | 持续非 0 或 burst 后不回落 | fast-writer 当前批次已有有效 envelope 在等待落库或 ack。 |
 | `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 连续 5 分钟上升 | NATS 快速写链路在 TDengine、Redis 或 NATS ack 某一阶段变慢。 |

go/vehicle-gateway/cmd/nats-fast-writer/main.go

@@ -119,7 +119,7 @@ func main() {
 		"workers", cfg.Workers,
 		"operation_timeout_ms", cfg.OperationWait.Milliseconds())
 	for i := 0; i < cfg.Workers; i++ {
-		go runFastWorker(ctx, logger, registry, sub, historyWriter, realtimeRepo, cfg)
+		go runFastWorker(ctx, logger, registry, js, sub, historyWriter, realtimeRepo, cfg)
 	}
 	<-ctx.Done()
 }
@@ -197,11 +197,23 @@ type fastMessage struct {
 	ack     func() error
 }
 
-func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, sub natsPullSubscription, appender fastAppender, updater fastUpdater, cfg config) {
+func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, infoReader natsConsumerInfoReader, sub natsPullSubscription, appender fastAppender, updater fastUpdater, cfg config) {
+	var lastConsumerInfoAt time.Time
 	for {
 		if ctx.Err() != nil {
 			return
 		}
+		if time.Since(lastConsumerInfoAt) >= time.Duration(envInt("NATS_CONSUMER_METRICS_INTERVAL_SECONDS", 10))*time.Second {
+			lastConsumerInfoAt = time.Now()
+			if infoReader != nil {
+				info, err := infoReader.ConsumerInfo(cfg.NATSStream, cfg.NATSDurable, nats.Context(ctx))
+				if err != nil {
+					logger.Warn("nats consumer info failed", "stream", cfg.NATSStream, "durable", cfg.NATSDurable, "error", err)
+				} else {
+					recordFastNATSConsumerInfoMetrics(registry, cfg, info)
+				}
+			}
+		}
 		msgs, err := sub.Fetch(cfg.BatchSize, nats.MaxWait(cfg.FetchWait))
 		if err != nil {
 			if errors.Is(err, nats.ErrTimeout) {
@@ -240,6 +252,10 @@ type natsPullSubscription interface {
 	Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error)
 }
 
+type natsConsumerInfoReader interface {
+	ConsumerInfo(stream string, name string, opts ...nats.JSOpt) (*nats.ConsumerInfo, error)
+}
+
 var fastWriterStageDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
 
 func processFastBatch(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, messages []*fastMessage) error {
@@ -377,6 +393,16 @@ func setFastBatchPending(registry *metrics.Registry, messages int, envelopes int
 	registry.SetGauge("vehicle_fast_writer_batch_pending_envelopes", nil, float64(envelopes))
 }
 
+func recordFastNATSConsumerInfoMetrics(registry *metrics.Registry, cfg config, info *nats.ConsumerInfo) {
+	if registry == nil || info == nil {
+		return
+	}
+	labels := metrics.Labels{"stream": cfg.NATSStream, "consumer": cfg.NATSDurable}
+	registry.SetGauge("vehicle_fast_writer_nats_consumer_pending", labels, float64(info.NumPending))
+	registry.SetGauge("vehicle_fast_writer_nats_consumer_ack_pending", labels, float64(info.NumAckPending))
+	registry.SetGauge("vehicle_fast_writer_nats_consumer_waiting", labels, float64(info.NumWaiting))
+}
+
 func recordFastWriterStageDuration(registry *metrics.Registry, subject string, stage string, status string, elapsed time.Duration) {
 	if registry == nil {
 		return

go/vehicle-gateway/cmd/nats-fast-writer/main_test.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/nats-io/nats.go"
+
 	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
 	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
 )
@@ -173,6 +175,28 @@ func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) {
 	}
 }
 
+func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) {
+	registry := metrics.NewRegistry()
+	cfg := config{NATSStream: "VEHICLE_INGEST", NATSDurable: "vehicle-fast-writer"}
+
+	recordFastNATSConsumerInfoMetrics(registry, cfg, &nats.ConsumerInfo{
+		NumPending:    23,
+		NumAckPending: 5,
+		NumWaiting:    3,
+	})
+
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_fast_writer_nats_consumer_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 23`,
+		`vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 5`,
+		`vehicle_fast_writer_nats_consumer_waiting{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 3`,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("metrics missing %s:\n%s", want, text)
+		}
+	}
+}
+
 type recordingFastAppender struct {
 	count         int
 	batchCount    int