diff --git a/docs/ops/100k-capacity-baseline.md b/docs/ops/100k-capacity-baseline.md index 4f9e028d..0985b93c 100644 --- a/docs/ops/100k-capacity-baseline.md +++ b/docs/ops/100k-capacity-baseline.md @@ -76,6 +76,8 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 | NATS bridge | `vehicle_bridge_nats_consumer_pending` | burst 后下降 | | NATS bridge | `vehicle_bridge_batch_pending_messages` | 稳态接近 0，burst 后下降 | | NATS bridge | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 不持续上升 | +| NATS fast writer | `vehicle_fast_writer_batch_pending_messages` | 稳态接近 0，burst 后下降 | +| NATS fast writer | `vehicle_fast_writer_batch_pending_envelopes` | 稳态接近 0，burst 后下降 | | NATS fast writer | `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | TDengine/Redis/ack 各阶段 p99 不持续上升 | | Kafka consumers | `vehicle_*_kafka_lag` | 稳态为 0，burst 后下降 | | Realtime Redis | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 毫秒级且不持续上升 | diff --git a/docs/ops/go-service-observability.md b/docs/ops/go-service-observability.md index c2b09e84..e65bfd11 100644 --- a/docs/ops/go-service-observability.md +++ b/docs/ops/go-service-observability.md @@ -47,6 +47,8 @@ curl -fsS http://127.0.0.1:20200/readyz | `vehicle_bridge_batch_pending_messages` | NATS messages fetched by the bridge but not yet written to Kafka and acked. | | `vehicle_bridge_batch_pending_kafka_messages` | Kafka messages prepared for the current bridge batch. | | `vehicle_bridge_batch_duration_ms_histogram` | Bridge batch duration histogram covering Kafka write and NATS ack. Labels: `status`. | +| `vehicle_fast_writer_batch_pending_messages` | NATS fast writer messages fetched but not yet written to TDengine/Redis and acked. | +| `vehicle_fast_writer_batch_pending_envelopes` | Valid parsed envelopes in the current NATS fast writer batch. | | `vehicle_fast_writer_stage_duration_ms_histogram` | Fast writer stage duration histogram for TDengine, Redis, and NATS ack. Labels: `subject`, `stage`, `status`. | | `vehicle_history_writes_total` | TDengine history writes. Labels: `topic`, `status`. | | `vehicle_history_batch_pending_messages` | Messages fetched by history writer but not yet flushed and committed. | diff --git a/docs/ops/go-vehicle-ingest-memory.md b/docs/ops/go-vehicle-ingest-memory.md index 06aac1e6..be27098e 100644 --- a/docs/ops/go-vehicle-ingest-memory.md +++ b/docs/ops/go-vehicle-ingest-memory.md @@ -423,6 +423,15 @@ vehicle_fast_writer_stage_duration_ms_histogram_sum{subject,stage,status} 2026-07-03 后续优化：`nats-fast-writer` 已从“Fetch batch 后逐条写 TDengine”改为“Fetch batch 后先调用 `history.Writer.AppendAllBatch` 批量写 TDengine，再逐条更新 Redis 和 ack”。这样可以减少 TDengine round trip；Redis/ack 仍按单条执行，避免某条实时投影失败时误 ack。 +同时新增 batch pending gauge： + +```text +vehicle_fast_writer_batch_pending_messages +vehicle_fast_writer_batch_pending_envelopes +``` + +这两个指标表示 fast-writer 已拉取但尚未完成 TDengine/Redis/ack 的批内压力。带帧率压测时它们应在稳态接近 0，burst 后能够回落；如果持续非 0，需要结合 `vehicle_fast_writer_stage_duration_ms_histogram` 判断卡在 TDengine、Redis 还是 ack。 + ### NATS Kafka bridge batch 指标 2026-07-03 已为 NATS -> Kafka bridge 增加 batch pending 和 duration histogram： diff --git a/docs/ops/vehicle-ingest-runbook.md b/docs/ops/vehicle-ingest-runbook.md index a91ee44c..f03ae720 100644 --- a/docs/ops/vehicle-ingest-runbook.md +++ b/docs/ops/vehicle-ingest-runbook.md @@ -129,6 +129,8 @@ go run ./cmd/load-sim \ | `vehicle_bridge_nats_consumer_pending` | 持续增长且 `> 10000` | bridge 消费 NATS 的速度跟不上生产速度。 | | `vehicle_bridge_batch_pending_messages` | 持续非 0 或 burst 后不回落 | bridge 已拉取 NATS 消息但尚未完成 Kafka 写入和 NATS ack。 | | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢，通常会先于 ack-pending 扩大。 | +| `vehicle_fast_writer_batch_pending_messages` | 持续非 0 或 burst 后不回落 | fast-writer 已拉取 NATS 消息但尚未完成 TDengine/Redis 写入和 ack。 | +| `vehicle_fast_writer_batch_pending_envelopes` | 持续非 0 或 burst 后不回落 | fast-writer 当前批次已有有效 envelope 在等待落库或 ack。 | | `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 连续 5 分钟上升 | NATS 快速写链路在 TDengine、Redis 或 NATS ack 某一阶段变慢。 | | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 连续 5 分钟上升 | Redis 实时投影变慢，会直接影响 realtime consumer 追平能力。 | | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 连续 5 分钟上升 | MySQL 当前态/位置投影变慢，需结合 async queue depth 和 dropped 计数判断。 | diff --git a/go/vehicle-gateway/cmd/nats-fast-writer/main.go b/go/vehicle-gateway/cmd/nats-fast-writer/main.go index 26dbd3a1..d520d2d3 100644 --- a/go/vehicle-gateway/cmd/nats-fast-writer/main.go +++ b/go/vehicle-gateway/cmd/nats-fast-writer/main.go @@ -262,6 +262,8 @@ func processFastBatch(ctx context.Context, registry *metrics.Registry, appender if len(envelopes) == 0 { return nil } + setFastBatchPending(registry, len(messages), len(envelopes)) + defer setFastBatchPending(registry, 0, 0) subject := fastBatchSubject(validMessages) started := time.Now() err := appender.AppendAllBatch(ctx, envelopes) @@ -367,6 +369,14 @@ func addFastMetric(registry *metrics.Registry, subject string, status string) { registry.IncCounter("vehicle_fast_writer_messages_total", metrics.Labels{"subject": subject, "status": status}) } +func setFastBatchPending(registry *metrics.Registry, messages int, envelopes int) { + if registry == nil { + return + } + registry.SetGauge("vehicle_fast_writer_batch_pending_messages", nil, float64(messages)) + registry.SetGauge("vehicle_fast_writer_batch_pending_envelopes", nil, float64(envelopes)) +} + func recordFastWriterStageDuration(registry *metrics.Registry, subject string, stage string, status string, elapsed time.Duration) { if registry == nil { return diff --git a/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go b/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go index 591fa86b..fc8054da 100644 --- a/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go +++ b/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "strings" + "sync" "testing" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" @@ -113,11 +114,71 @@ func TestProcessFastBatchAppendsTDengineBatchBeforeRedisAndAck(t *testing.T) { } } +func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-6"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-7"} + firstPayload, err := first.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + secondPayload, err := second.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + appenderStarted := make(chan struct{}) + appenderRelease := make(chan struct{}) + appender := &recordingFastAppender{ + onAppendBatch: func() { + close(appenderStarted) + <-appenderRelease + }, + } + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { return nil }}, + {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { return nil }}, + } + var wg sync.WaitGroup + wg.Add(1) + var processErr error + go func() { + defer wg.Done() + processErr = processFastBatch(context.Background(), registry, appender, &recordingFastUpdater{}, msgs) + }() + <-appenderStarted + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_batch_pending_messages 2`, + `vehicle_fast_writer_batch_pending_envelopes 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("pending metric missing %s while appending:\n%s", want, text) + } + } + + close(appenderRelease) + wg.Wait() + if processErr != nil { + t.Fatalf("processFastBatch() error = %v", processErr) + } + text = registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_batch_pending_messages 0`, + `vehicle_fast_writer_batch_pending_envelopes 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("pending metric should reset after append, missing %s:\n%s", want, text) + } + } +} + type recordingFastAppender struct { - count int - batchCount int - batchRows int - err error + count int + batchCount int + batchRows int + err error + onAppendBatch func() } func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelope) error { @@ -128,6 +189,9 @@ func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelop func (a *recordingFastAppender) AppendAllBatch(_ context.Context, envs []envelope.FrameEnvelope) error { a.batchCount++ a.batchRows += len(envs) + if a.onAppendBatch != nil { + a.onAppendBatch() + } return a.err }