feat(go): expose fast writer batch pending metrics
This commit is contained in:
@@ -76,6 +76,8 @@ systemd gateway 已配置 `LimitNOFILE=1048576`,需要持续保持。
|
|||||||
| NATS bridge | `vehicle_bridge_nats_consumer_pending` | burst 后下降 |
|
| NATS bridge | `vehicle_bridge_nats_consumer_pending` | burst 后下降 |
|
||||||
| NATS bridge | `vehicle_bridge_batch_pending_messages` | 稳态接近 0,burst 后下降 |
|
| NATS bridge | `vehicle_bridge_batch_pending_messages` | 稳态接近 0,burst 后下降 |
|
||||||
| NATS bridge | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 不持续上升 |
|
| NATS bridge | `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 不持续上升 |
|
||||||
|
| NATS fast writer | `vehicle_fast_writer_batch_pending_messages` | 稳态接近 0,burst 后下降 |
|
||||||
|
| NATS fast writer | `vehicle_fast_writer_batch_pending_envelopes` | 稳态接近 0,burst 后下降 |
|
||||||
| NATS fast writer | `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | TDengine/Redis/ack 各阶段 p99 不持续上升 |
|
| NATS fast writer | `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | TDengine/Redis/ack 各阶段 p99 不持续上升 |
|
||||||
| Kafka consumers | `vehicle_*_kafka_lag` | 稳态为 0,burst 后下降 |
|
| Kafka consumers | `vehicle_*_kafka_lag` | 稳态为 0,burst 后下降 |
|
||||||
| Realtime Redis | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 毫秒级且不持续上升 |
|
| Realtime Redis | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 毫秒级且不持续上升 |
|
||||||
|
|||||||
@@ -47,6 +47,8 @@ curl -fsS http://127.0.0.1:20200/readyz
|
|||||||
| `vehicle_bridge_batch_pending_messages` | NATS messages fetched by the bridge but not yet written to Kafka and acked. |
|
| `vehicle_bridge_batch_pending_messages` | NATS messages fetched by the bridge but not yet written to Kafka and acked. |
|
||||||
| `vehicle_bridge_batch_pending_kafka_messages` | Kafka messages prepared for the current bridge batch. |
|
| `vehicle_bridge_batch_pending_kafka_messages` | Kafka messages prepared for the current bridge batch. |
|
||||||
| `vehicle_bridge_batch_duration_ms_histogram` | Bridge batch duration histogram covering Kafka write and NATS ack. Labels: `status`. |
|
| `vehicle_bridge_batch_duration_ms_histogram` | Bridge batch duration histogram covering Kafka write and NATS ack. Labels: `status`. |
|
||||||
|
| `vehicle_fast_writer_batch_pending_messages` | NATS fast writer messages fetched but not yet written to TDengine/Redis and acked. |
|
||||||
|
| `vehicle_fast_writer_batch_pending_envelopes` | Valid parsed envelopes in the current NATS fast writer batch. |
|
||||||
| `vehicle_fast_writer_stage_duration_ms_histogram` | Fast writer stage duration histogram for TDengine, Redis, and NATS ack. Labels: `subject`, `stage`, `status`. |
|
| `vehicle_fast_writer_stage_duration_ms_histogram` | Fast writer stage duration histogram for TDengine, Redis, and NATS ack. Labels: `subject`, `stage`, `status`. |
|
||||||
| `vehicle_history_writes_total` | TDengine history writes. Labels: `topic`, `status`. |
|
| `vehicle_history_writes_total` | TDengine history writes. Labels: `topic`, `status`. |
|
||||||
| `vehicle_history_batch_pending_messages` | Messages fetched by history writer but not yet flushed and committed. |
|
| `vehicle_history_batch_pending_messages` | Messages fetched by history writer but not yet flushed and committed. |
|
||||||
|
|||||||
@@ -423,6 +423,15 @@ vehicle_fast_writer_stage_duration_ms_histogram_sum{subject,stage,status}
|
|||||||
|
|
||||||
2026-07-03 后续优化:`nats-fast-writer` 已从“Fetch batch 后逐条写 TDengine”改为“Fetch batch 后先调用 `history.Writer.AppendAllBatch` 批量写 TDengine,再逐条更新 Redis 和 ack”。这样可以减少 TDengine round trip;Redis/ack 仍按单条执行,避免某条实时投影失败时误 ack。
|
2026-07-03 后续优化:`nats-fast-writer` 已从“Fetch batch 后逐条写 TDengine”改为“Fetch batch 后先调用 `history.Writer.AppendAllBatch` 批量写 TDengine,再逐条更新 Redis 和 ack”。这样可以减少 TDengine round trip;Redis/ack 仍按单条执行,避免某条实时投影失败时误 ack。
|
||||||
|
|
||||||
|
同时新增 batch pending gauge:
|
||||||
|
|
||||||
|
```text
|
||||||
|
vehicle_fast_writer_batch_pending_messages
|
||||||
|
vehicle_fast_writer_batch_pending_envelopes
|
||||||
|
```
|
||||||
|
|
||||||
|
这两个指标表示 fast-writer 已拉取但尚未完成 TDengine/Redis/ack 的批内压力。带帧率压测时它们应在稳态接近 0,burst 后能够回落;如果持续非 0,需要结合 `vehicle_fast_writer_stage_duration_ms_histogram` 判断卡在 TDengine、Redis 还是 ack。
|
||||||
|
|
||||||
### NATS Kafka bridge batch 指标
|
### NATS Kafka bridge batch 指标
|
||||||
|
|
||||||
2026-07-03 已为 NATS -> Kafka bridge 增加 batch pending 和 duration histogram:
|
2026-07-03 已为 NATS -> Kafka bridge 增加 batch pending 和 duration histogram:
|
||||||
|
|||||||
@@ -129,6 +129,8 @@ go run ./cmd/load-sim \
|
|||||||
| `vehicle_bridge_nats_consumer_pending` | 持续增长且 `> 10000` | bridge 消费 NATS 的速度跟不上生产速度。 |
|
| `vehicle_bridge_nats_consumer_pending` | 持续增长且 `> 10000` | bridge 消费 NATS 的速度跟不上生产速度。 |
|
||||||
| `vehicle_bridge_batch_pending_messages` | 持续非 0 或 burst 后不回落 | bridge 已拉取 NATS 消息但尚未完成 Kafka 写入和 NATS ack。 |
|
| `vehicle_bridge_batch_pending_messages` | 持续非 0 或 burst 后不回落 | bridge 已拉取 NATS 消息但尚未完成 Kafka 写入和 NATS ack。 |
|
||||||
| `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢,通常会先于 ack-pending 扩大。 |
|
| `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢,通常会先于 ack-pending 扩大。 |
|
||||||
|
| `vehicle_fast_writer_batch_pending_messages` | 持续非 0 或 burst 后不回落 | fast-writer 已拉取 NATS 消息但尚未完成 TDengine/Redis 写入和 ack。 |
|
||||||
|
| `vehicle_fast_writer_batch_pending_envelopes` | 持续非 0 或 burst 后不回落 | fast-writer 当前批次已有有效 envelope 在等待落库或 ack。 |
|
||||||
| `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 连续 5 分钟上升 | NATS 快速写链路在 TDengine、Redis 或 NATS ack 某一阶段变慢。 |
|
| `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 连续 5 分钟上升 | NATS 快速写链路在 TDengine、Redis 或 NATS ack 某一阶段变慢。 |
|
||||||
| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 连续 5 分钟上升 | Redis 实时投影变慢,会直接影响 realtime consumer 追平能力。 |
|
| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 连续 5 分钟上升 | Redis 实时投影变慢,会直接影响 realtime consumer 追平能力。 |
|
||||||
| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 连续 5 分钟上升 | MySQL 当前态/位置投影变慢,需结合 async queue depth 和 dropped 计数判断。 |
|
| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 连续 5 分钟上升 | MySQL 当前态/位置投影变慢,需结合 async queue depth 和 dropped 计数判断。 |
|
||||||
|
|||||||
@@ -262,6 +262,8 @@ func processFastBatch(ctx context.Context, registry *metrics.Registry, appender
|
|||||||
if len(envelopes) == 0 {
|
if len(envelopes) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
setFastBatchPending(registry, len(messages), len(envelopes))
|
||||||
|
defer setFastBatchPending(registry, 0, 0)
|
||||||
subject := fastBatchSubject(validMessages)
|
subject := fastBatchSubject(validMessages)
|
||||||
started := time.Now()
|
started := time.Now()
|
||||||
err := appender.AppendAllBatch(ctx, envelopes)
|
err := appender.AppendAllBatch(ctx, envelopes)
|
||||||
@@ -367,6 +369,14 @@ func addFastMetric(registry *metrics.Registry, subject string, status string) {
|
|||||||
registry.IncCounter("vehicle_fast_writer_messages_total", metrics.Labels{"subject": subject, "status": status})
|
registry.IncCounter("vehicle_fast_writer_messages_total", metrics.Labels{"subject": subject, "status": status})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setFastBatchPending(registry *metrics.Registry, messages int, envelopes int) {
|
||||||
|
if registry == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
registry.SetGauge("vehicle_fast_writer_batch_pending_messages", nil, float64(messages))
|
||||||
|
registry.SetGauge("vehicle_fast_writer_batch_pending_envelopes", nil, float64(envelopes))
|
||||||
|
}
|
||||||
|
|
||||||
func recordFastWriterStageDuration(registry *metrics.Registry, subject string, stage string, status string, elapsed time.Duration) {
|
func recordFastWriterStageDuration(registry *metrics.Registry, subject string, stage string, status string, elapsed time.Duration) {
|
||||||
if registry == nil {
|
if registry == nil {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
@@ -113,11 +114,71 @@ func TestProcessFastBatchAppendsTDengineBatchBeforeRedisAndAck(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) {
|
||||||
|
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-6"}
|
||||||
|
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-7"}
|
||||||
|
firstPayload, err := first.MarshalJSONBytes()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
secondPayload, err := second.MarshalJSONBytes()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
registry := metrics.NewRegistry()
|
||||||
|
appenderStarted := make(chan struct{})
|
||||||
|
appenderRelease := make(chan struct{})
|
||||||
|
appender := &recordingFastAppender{
|
||||||
|
onAppendBatch: func() {
|
||||||
|
close(appenderStarted)
|
||||||
|
<-appenderRelease
|
||||||
|
},
|
||||||
|
}
|
||||||
|
msgs := []*fastMessage{
|
||||||
|
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { return nil }},
|
||||||
|
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { return nil }},
|
||||||
|
}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
var processErr error
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
processErr = processFastBatch(context.Background(), registry, appender, &recordingFastUpdater{}, msgs)
|
||||||
|
}()
|
||||||
|
<-appenderStarted
|
||||||
|
|
||||||
|
text := registry.Render()
|
||||||
|
for _, want := range []string{
|
||||||
|
`vehicle_fast_writer_batch_pending_messages 2`,
|
||||||
|
`vehicle_fast_writer_batch_pending_envelopes 2`,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(text, want) {
|
||||||
|
t.Fatalf("pending metric missing %s while appending:\n%s", want, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
close(appenderRelease)
|
||||||
|
wg.Wait()
|
||||||
|
if processErr != nil {
|
||||||
|
t.Fatalf("processFastBatch() error = %v", processErr)
|
||||||
|
}
|
||||||
|
text = registry.Render()
|
||||||
|
for _, want := range []string{
|
||||||
|
`vehicle_fast_writer_batch_pending_messages 0`,
|
||||||
|
`vehicle_fast_writer_batch_pending_envelopes 0`,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(text, want) {
|
||||||
|
t.Fatalf("pending metric should reset after append, missing %s:\n%s", want, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type recordingFastAppender struct {
|
type recordingFastAppender struct {
|
||||||
count int
|
count int
|
||||||
batchCount int
|
batchCount int
|
||||||
batchRows int
|
batchRows int
|
||||||
err error
|
err error
|
||||||
|
onAppendBatch func()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelope) error {
|
func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelope) error {
|
||||||
@@ -128,6 +189,9 @@ func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelop
|
|||||||
func (a *recordingFastAppender) AppendAllBatch(_ context.Context, envs []envelope.FrameEnvelope) error {
|
func (a *recordingFastAppender) AppendAllBatch(_ context.Context, envs []envelope.FrameEnvelope) error {
|
||||||
a.batchCount++
|
a.batchCount++
|
||||||
a.batchRows += len(envs)
|
a.batchRows += len(envs)
|
||||||
|
if a.onAppendBatch != nil {
|
||||||
|
a.onAppendBatch()
|
||||||
|
}
|
||||||
return a.err
|
return a.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user