diff --git a/docs/ops/100k-capacity-baseline.md b/docs/ops/100k-capacity-baseline.md index 0985b93c..3d8a19f9 100644 --- a/docs/ops/100k-capacity-baseline.md +++ b/docs/ops/100k-capacity-baseline.md @@ -76,6 +76,8 @@ systemd gateway 已配置 `LimitNOFILE=1048576`,需要持续保持。 | NATS bridge | `vehicle_bridge_nats_consumer_pending` | burst 后下降 | | NATS bridge | `vehicle_bridge_batch_pending_messages` | 稳态接近 0,burst 后下降 | | NATS bridge | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 不持续上升 | +| NATS fast writer | `vehicle_fast_writer_nats_consumer_ack_pending` | 稳态为 0 | +| NATS fast writer | `vehicle_fast_writer_nats_consumer_pending` | burst 后下降 | | NATS fast writer | `vehicle_fast_writer_batch_pending_messages` | 稳态接近 0,burst 后下降 | | NATS fast writer | `vehicle_fast_writer_batch_pending_envelopes` | 稳态接近 0,burst 后下降 | | NATS fast writer | `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | TDengine/Redis/ack 各阶段 p99 不持续上升 | diff --git a/docs/ops/go-service-observability.md b/docs/ops/go-service-observability.md index e65bfd11..6b5f4c7b 100644 --- a/docs/ops/go-service-observability.md +++ b/docs/ops/go-service-observability.md @@ -47,6 +47,9 @@ curl -fsS http://127.0.0.1:20200/readyz | `vehicle_bridge_batch_pending_messages` | NATS messages fetched by the bridge but not yet written to Kafka and acked. | | `vehicle_bridge_batch_pending_kafka_messages` | Kafka messages prepared for the current bridge batch. | | `vehicle_bridge_batch_duration_ms_histogram` | Bridge batch duration histogram covering Kafka write and NATS ack. Labels: `status`. | +| `vehicle_fast_writer_nats_consumer_pending` | JetStream messages pending for the fast writer durable consumer. | +| `vehicle_fast_writer_nats_consumer_ack_pending` | JetStream messages delivered to fast writer but not yet acked. | +| `vehicle_fast_writer_nats_consumer_waiting` | Pull requests waiting on the fast writer durable consumer. | | `vehicle_fast_writer_batch_pending_messages` | NATS fast writer messages fetched but not yet written to TDengine/Redis and acked. | | `vehicle_fast_writer_batch_pending_envelopes` | Valid parsed envelopes in the current NATS fast writer batch. | | `vehicle_fast_writer_stage_duration_ms_histogram` | Fast writer stage duration histogram for TDengine, Redis, and NATS ack. Labels: `subject`, `stage`, `status`. | diff --git a/docs/ops/go-vehicle-ingest-memory.md b/docs/ops/go-vehicle-ingest-memory.md index be27098e..b3855c86 100644 --- a/docs/ops/go-vehicle-ingest-memory.md +++ b/docs/ops/go-vehicle-ingest-memory.md @@ -426,11 +426,14 @@ vehicle_fast_writer_stage_duration_ms_histogram_sum{subject,stage,status} 同时新增 batch pending gauge: ```text +vehicle_fast_writer_nats_consumer_pending{stream,consumer} +vehicle_fast_writer_nats_consumer_ack_pending{stream,consumer} +vehicle_fast_writer_nats_consumer_waiting{stream,consumer} vehicle_fast_writer_batch_pending_messages vehicle_fast_writer_batch_pending_envelopes ``` -这两个指标表示 fast-writer 已拉取但尚未完成 TDengine/Redis/ack 的批内压力。带帧率压测时它们应在稳态接近 0,burst 后能够回落;如果持续非 0,需要结合 `vehicle_fast_writer_stage_duration_ms_histogram` 判断卡在 TDengine、Redis 还是 ack。 +`vehicle_fast_writer_nats_consumer_*` 表示 JetStream durable consumer 层面的 backlog/ack-pending;`vehicle_fast_writer_batch_pending_*` 表示 fast-writer 已拉取但尚未完成 TDengine/Redis/ack 的批内压力。带帧率压测时它们应在稳态接近 0,burst 后能够回落;如果持续非 0,需要结合 `vehicle_fast_writer_stage_duration_ms_histogram` 判断卡在 NATS 拉取、TDengine、Redis 还是 ack。 ### NATS Kafka bridge batch 指标 diff --git a/docs/ops/vehicle-ingest-runbook.md b/docs/ops/vehicle-ingest-runbook.md index f03ae720..dcc76f1e 100644 --- a/docs/ops/vehicle-ingest-runbook.md +++ b/docs/ops/vehicle-ingest-runbook.md @@ -129,6 +129,8 @@ go run ./cmd/load-sim \ | `vehicle_bridge_nats_consumer_pending` | 持续增长且 `> 10000` | bridge 消费 NATS 的速度跟不上生产速度。 | | `vehicle_bridge_batch_pending_messages` | 持续非 0 或 burst 后不回落 | bridge 已拉取 NATS 消息但尚未完成 Kafka 写入和 NATS ack。 | | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢,通常会先于 ack-pending 扩大。 | +| `vehicle_fast_writer_nats_consumer_ack_pending` | 连续 2 分钟 `> 0` | 消息已投递给 fast-writer,但 TDengine/Redis 写入后未完成 ack。 | +| `vehicle_fast_writer_nats_consumer_pending` | 持续增长且 `> 10000` | fast-writer 消费 NATS 的速度跟不上入口写入速度。 | | `vehicle_fast_writer_batch_pending_messages` | 持续非 0 或 burst 后不回落 | fast-writer 已拉取 NATS 消息但尚未完成 TDengine/Redis 写入和 ack。 | | `vehicle_fast_writer_batch_pending_envelopes` | 持续非 0 或 burst 后不回落 | fast-writer 当前批次已有有效 envelope 在等待落库或 ack。 | | `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 连续 5 分钟上升 | NATS 快速写链路在 TDengine、Redis 或 NATS ack 某一阶段变慢。 | diff --git a/go/vehicle-gateway/cmd/nats-fast-writer/main.go b/go/vehicle-gateway/cmd/nats-fast-writer/main.go index d520d2d3..7eb95da8 100644 --- a/go/vehicle-gateway/cmd/nats-fast-writer/main.go +++ b/go/vehicle-gateway/cmd/nats-fast-writer/main.go @@ -119,7 +119,7 @@ func main() { "workers", cfg.Workers, "operation_timeout_ms", cfg.OperationWait.Milliseconds()) for i := 0; i < cfg.Workers; i++ { - go runFastWorker(ctx, logger, registry, sub, historyWriter, realtimeRepo, cfg) + go runFastWorker(ctx, logger, registry, js, sub, historyWriter, realtimeRepo, cfg) } <-ctx.Done() } @@ -197,11 +197,23 @@ type fastMessage struct { ack func() error } -func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, sub natsPullSubscription, appender fastAppender, updater fastUpdater, cfg config) { +func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, infoReader natsConsumerInfoReader, sub natsPullSubscription, appender fastAppender, updater fastUpdater, cfg config) { + var lastConsumerInfoAt time.Time for { if ctx.Err() != nil { return } + if time.Since(lastConsumerInfoAt) >= time.Duration(envInt("NATS_CONSUMER_METRICS_INTERVAL_SECONDS", 10))*time.Second { + lastConsumerInfoAt = time.Now() + if infoReader != nil { + info, err := infoReader.ConsumerInfo(cfg.NATSStream, cfg.NATSDurable, nats.Context(ctx)) + if err != nil { + logger.Warn("nats consumer info failed", "stream", cfg.NATSStream, "durable", cfg.NATSDurable, "error", err) + } else { + recordFastNATSConsumerInfoMetrics(registry, cfg, info) + } + } + } msgs, err := sub.Fetch(cfg.BatchSize, nats.MaxWait(cfg.FetchWait)) if err != nil { if errors.Is(err, nats.ErrTimeout) { @@ -240,6 +252,10 @@ type natsPullSubscription interface { Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) } +type natsConsumerInfoReader interface { + ConsumerInfo(stream string, name string, opts ...nats.JSOpt) (*nats.ConsumerInfo, error) +} + var fastWriterStageDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} func processFastBatch(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, messages []*fastMessage) error { @@ -377,6 +393,16 @@ func setFastBatchPending(registry *metrics.Registry, messages int, envelopes int registry.SetGauge("vehicle_fast_writer_batch_pending_envelopes", nil, float64(envelopes)) } +func recordFastNATSConsumerInfoMetrics(registry *metrics.Registry, cfg config, info *nats.ConsumerInfo) { + if registry == nil || info == nil { + return + } + labels := metrics.Labels{"stream": cfg.NATSStream, "consumer": cfg.NATSDurable} + registry.SetGauge("vehicle_fast_writer_nats_consumer_pending", labels, float64(info.NumPending)) + registry.SetGauge("vehicle_fast_writer_nats_consumer_ack_pending", labels, float64(info.NumAckPending)) + registry.SetGauge("vehicle_fast_writer_nats_consumer_waiting", labels, float64(info.NumWaiting)) +} + func recordFastWriterStageDuration(registry *metrics.Registry, subject string, stage string, status string, elapsed time.Duration) { if registry == nil { return diff --git a/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go b/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go index fc8054da..4052a9c5 100644 --- a/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go +++ b/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go @@ -7,6 +7,8 @@ import ( "sync" "testing" + "github.com/nats-io/nats.go" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" ) @@ -173,6 +175,28 @@ func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) { } } +func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) { + registry := metrics.NewRegistry() + cfg := config{NATSStream: "VEHICLE_INGEST", NATSDurable: "vehicle-fast-writer"} + + recordFastNATSConsumerInfoMetrics(registry, cfg, &nats.ConsumerInfo{ + NumPending: 23, + NumAckPending: 5, + NumWaiting: 3, + }) + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_nats_consumer_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 23`, + `vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 5`, + `vehicle_fast_writer_nats_consumer_waiting{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 3`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metrics missing %s:\n%s", want, text) + } + } +} + type recordingFastAppender struct { count int batchCount int