From 07d17e5adf7e97fbde2ce9f122fbb51c22f84de5 Mon Sep 17 00:00:00 2001 From: lingniu Date: Fri, 3 Jul 2026 19:08:49 +0800 Subject: [PATCH] feat(go): expose stat writer latency histogram --- docs/ops/100k-capacity-baseline.md | 1 + docs/ops/go-service-observability.md | 1 + docs/ops/go-vehicle-ingest-memory.md | 12 ++++++++++ docs/ops/vehicle-ingest-runbook.md | 1 + go/vehicle-gateway/cmd/stat-writer/main.go | 24 ++++++++++++++++++- .../cmd/stat-writer/main_test.go | 3 +++ 6 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/ops/100k-capacity-baseline.md b/docs/ops/100k-capacity-baseline.md index f709c3ef..e6094c4d 100644 --- a/docs/ops/100k-capacity-baseline.md +++ b/docs/ops/100k-capacity-baseline.md @@ -80,6 +80,7 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 | Realtime Redis | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 毫秒级且不持续上升 | | Realtime MySQL | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 不持续上升 | | Realtime MySQL | async queue depth | 不持续增长 | +| Stat writer | `vehicle_stat_write_duration_ms_histogram_bucket` | p99 不持续上升 | ## 分阶段压测 diff --git a/docs/ops/go-service-observability.md b/docs/ops/go-service-observability.md index cd569c59..38e9e9df 100644 --- a/docs/ops/go-service-observability.md +++ b/docs/ops/go-service-observability.md @@ -52,6 +52,7 @@ curl -fsS http://127.0.0.1:20200/readyz | `vehicle_history_batch_pending_rows` | Parsed raw envelopes waiting in the current TDengine batch. | | `vehicle_history_batch_flush_duration_ms` | Last TDengine batch flush duration. Labels: `status`. | | `vehicle_stat_writes_total` | MySQL metric writes. Labels: `topic`, `status`. | +| `vehicle_stat_write_duration_ms_histogram` | MySQL stat writer append duration histogram. Labels: `topic`, `status`. | | `vehicle_realtime_updates_total` | Redis/MySQL realtime projector updates. Labels: `topic`, `status`. | | `vehicle_realtime_store_update_duration_ms_histogram` | Redis/MySQL store update duration histogram. Labels: `store`, `protocol`, `status`. | | `vehicle_history_kafka_lag` | Estimated Kafka lag for history writer by topic and partition. | diff --git a/docs/ops/go-vehicle-ingest-memory.md b/docs/ops/go-vehicle-ingest-memory.md index 8e29f5d7..6bd3cb4d 100644 --- a/docs/ops/go-vehicle-ingest-memory.md +++ b/docs/ops/go-vehicle-ingest-memory.md @@ -435,6 +435,18 @@ vehicle_realtime_store_update_duration_ms_histogram_sum{store,protocol,status} 该耗时覆盖 Redis 快速实时投影和 MySQL 异步当前态/位置投影的实际写入耗时。带帧率压测时，Redis p99 上升会直接影响 realtime consumer 追平；MySQL p99 上升需要结合 `vehicle_realtime_async_queue_depth` 和 `vehicle_realtime_async_queue_total{status="dropped"}` 判断是否开始丢弃低优先级当前态写入。 +### Stat writer write histogram + +2026-07-03 已为 stat-writer MySQL append 增加 duration histogram： + +```text +vehicle_stat_write_duration_ms_histogram_bucket{topic,status,le} +vehicle_stat_write_duration_ms_histogram_count{topic,status} +vehicle_stat_write_duration_ms_histogram_sum{topic,status} +``` + +该耗时覆盖单条 Kafka 消息解析后写入 `vehicle_daily_mileage` 的 MySQL append 路径。带帧率压测时，如果该 p99 持续上升并伴随 `vehicle_stat_kafka_lag` 增长，优先排查 MySQL 写入、每日里程表索引和 stat-writer 消费能力。 + ### Gateway frame duration histogram 2026-07-03 已为 Gateway 帧处理耗时增加 histogram： diff --git a/docs/ops/vehicle-ingest-runbook.md b/docs/ops/vehicle-ingest-runbook.md index 437f683e..d1f9edd6 100644 --- a/docs/ops/vehicle-ingest-runbook.md +++ b/docs/ops/vehicle-ingest-runbook.md @@ -131,6 +131,7 @@ go run ./cmd/load-sim \ | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢，通常会先于 ack-pending 扩大。 | | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 连续 5 分钟上升 | Redis 实时投影变慢，会直接影响 realtime consumer 追平能力。 | | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 连续 5 分钟上升 | MySQL 当前态/位置投影变慢，需结合 async queue depth 和 dropped 计数判断。 | +| `vehicle_stat_write_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | MySQL 每日里程统计写入变慢，可能导致 stat Kafka lag 增长。 | | Kafka lag | 连续 5 分钟增长或 `> 10000` | 下游 consumer 或存储存在瓶颈。 | | Writer 成功计数 | 入口增长但 writer 不增长 | bridge、Kafka、consumer 或存储链路断开。 | diff --git a/go/vehicle-gateway/cmd/stat-writer/main.go b/go/vehicle-gateway/cmd/stat-writer/main.go index b590a595..8bbd57fd 100644 --- a/go/vehicle-gateway/cmd/stat-writer/main.go +++ b/go/vehicle-gateway/cmd/stat-writer/main.go @@ -83,6 +83,8 @@ type kafkaMessageCommitter interface { CommitMessages(context.Context, ...kafka.Message) error } +var statWriteDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} + func processStatMessage(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) @@ -99,7 +101,10 @@ func processStatMessage(ctx context.Context, logger interface { _ = committer.CommitMessages(messageCtx, message) return } - if err := appender.Append(messageCtx, env); err != nil { + started := time.Now() + err := appender.Append(messageCtx, env) + recordStatWriteDuration(registry, message, statusFromError(err), time.Since(started)) + if err != nil { addStatMetric(registry, "vehicle_stat_writes_total", message, "error") logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) return @@ -127,6 +132,23 @@ func addStatLagMetric(registry *metrics.Registry, message kafka.Message) { registry.SetKafkaLag("vehicle_stat_kafka_lag", message.Topic, message.Partition, message.Offset, message.HighWaterMark) } +func recordStatWriteDuration(registry *metrics.Registry, message kafka.Message, status string, elapsed time.Duration) { + if registry == nil { + return + } + registry.ObserveHistogram("vehicle_stat_write_duration_ms_histogram", metrics.Labels{ + "topic": message.Topic, + "status": status, + }, statWriteDurationBucketsMS, float64(elapsed.Milliseconds())) +} + +func statusFromError(err error) string { + if err != nil { + return "error" + } + return "ok" +} + type config struct { KafkaBrokers []string KafkaTopics []string diff --git a/go/vehicle-gateway/cmd/stat-writer/main_test.go b/go/vehicle-gateway/cmd/stat-writer/main_test.go index fd18bd7d..2a3903f8 100644 --- a/go/vehicle-gateway/cmd/stat-writer/main_test.go +++ b/go/vehicle-gateway/cmd/stat-writer/main_test.go @@ -67,6 +67,9 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) { `vehicle_stat_writes_total{status="ok",topic="vehicle.raw.jt808.v1"} 1`, `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.raw.jt808.v1"} 1`, `vehicle_stat_kafka_lag{partition="4",topic="vehicle.raw.jt808.v1"} 10`, + `vehicle_stat_write_duration_ms_histogram_bucket{le="+Inf",status="ok",topic="vehicle.raw.jt808.v1"} 1`, + `vehicle_stat_write_duration_ms_histogram_count{status="ok",topic="vehicle.raw.jt808.v1"} 1`, + `vehicle_stat_write_duration_ms_histogram_sum{status="ok",topic="vehicle.raw.jt808.v1"}`, } { if !strings.Contains(text, want) { t.Fatalf("metrics missing %s:\n%s", want, text)