shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(stats): flush throttled daily mileage projections

Browse Source
This commit is contained in:
lingniu
2026-07-19 16:00:13 +08:00
parent 81532d91b7
commit fa60bbb2c8
6 changed files with 381 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
go/vehicle-gateway/cmd/stat-writer/main.go
View File

@@ -95,6 +95,14 @@ func main() {
quarantiner := fileStatMessageQuarantiner{dir: cfg.QuarantineDir}
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "baseline_hit_ttl_seconds", cfg.BaselineHitTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds(), "quarantine_dir", cfg.QuarantineDir)
var background sync.WaitGroup
if cfg.ProjectInterval > 0 {
background.Add(1)
go func() {
defer background.Done()
runPendingProjectionFlusher(ctx, logger, registry, writer, cfg.ProjectInterval)
}()
}
var workers sync.WaitGroup
for workerID := 1; workerID <= cfg.Workers; workerID++ {
workers.Add(1)
@@ -104,6 +112,67 @@ func main() {
}(workerID)
}
workers.Wait()
background.Wait()
if cfg.ProjectInterval > 0 {
flushCtx, cancel := context.WithTimeout(context.Background(), kafkaMessageOperationTimeout)
flushPendingProjections(flushCtx, logger, registry, writer, time.Time{})
cancel()
}
}
type pendingProjectionFlusher interface {
FlushPendingProjections(context.Context, time.Time) (stats.ProjectionFlushResult, error)
}
func runPendingProjectionFlusher(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, flusher pendingProjectionFlusher, interval time.Duration) {
if flusher == nil || interval <= 0 {
return
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case now := <-ticker.C:
operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
flushPendingProjections(operationCtx, logger, registry, flusher, now.Add(-interval))
cancel()
}
}
}
func flushPendingProjections(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, flusher pendingProjectionFlusher, readyBefore time.Time) {
if flusher == nil {
return
}
result, err := flusher.FlushPendingProjections(ctx, readyBefore)
if registry != nil {
if result.Attempted > 0 {
registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "attempted"}, float64(result.Attempted))
}
if result.Written > 0 {
registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "written"}, float64(result.Written))
metrics.RecordLastActivity(registry, "vehicle_stat_last_pending_projection_unix_seconds", metrics.Labels{"status": "written"})
}
if result.Failed > 0 {
registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "error"}, float64(result.Failed))
} else if err != nil {
registry.IncCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "error"})
}
if err != nil {
metrics.RecordLastActivity(registry, "vehicle_stat_last_pending_projection_unix_seconds", metrics.Labels{"status": "error"})
}
}
if err != nil && logger != nil {
logger.Warn("pending daily mileage projection flush failed", "attempted", result.Attempted, "written", result.Written, "failed", result.Failed, "error", err)
}
}
func runStatConsumer(ctx context.Context, logger interface {
@@ -763,6 +832,7 @@ func recordStatCacheMetrics(registry *metrics.Registry, appender statAppender) {
setStatCacheGauge(registry, "last_total_mileage", stats.LastTotalMileageEntries, stats.MaxEntries, stats.LastCleanupTotalMileage, stats.TotalMileageEvictions)
setStatCacheGauge(registry, "source_seen", stats.LastSourceSeenEntries, stats.MaxEntries, stats.LastCleanupSourceSeen, stats.SourceSeenEvictions)
setStatCacheGauge(registry, "projection", stats.LastProjectionEntries, stats.MaxEntries, stats.LastCleanupProjection, stats.ProjectionEvictions)
registry.SetGauge("vehicle_stat_pending_projection_entries", nil, float64(stats.PendingProjectionEntries))
setStatCacheGauge(registry, "baseline", stats.BaselineEntries, stats.MaxEntries, stats.LastCleanupBaseline, stats.BaselineEvictions)
if !stats.LastCleanupAt.IsZero() {
registry.SetGauge("vehicle_stat_cache_last_cleanup_unix_seconds", nil, float64(stats.LastCleanupAt.Unix()))

89
go/vehicle-gateway/cmd/stat-writer/main_test.go
View File

@@ -105,20 +105,21 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) {
ProjectionsWritten: 1,
ProjectionsSkippedThrottled: 2,
}, cacheStats: stats.CacheStats{
LastTotalMileageEntries: 300,
LastSourceSeenEntries: 3,
LastProjectionEntries: 280,
BaselineEntries: 295,
MaxEntries: 1000000,
LastCleanupAt: time.Unix(1782918600, 0),
LastCleanupTotalMileage: 11,
LastCleanupSourceSeen: 1,
LastCleanupProjection: 9,
LastCleanupBaseline: 10,
TotalMileageEvictions: 4,
SourceSeenEvictions: 1,
ProjectionEvictions: 2,
BaselineEvictions: 3,
LastTotalMileageEntries: 300,
LastSourceSeenEntries: 3,
LastProjectionEntries: 280,
PendingProjectionEntries: 12,
BaselineEntries: 295,
MaxEntries: 1000000,
LastCleanupAt: time.Unix(1782918600, 0),
LastCleanupTotalMileage: 11,
LastCleanupSourceSeen: 1,
LastCleanupProjection: 9,
LastCleanupBaseline: 10,
TotalMileageEvictions: 4,
SourceSeenEvictions: 1,
ProjectionEvictions: 2,
BaselineEvictions: 3,
}},
&contextCheckingStatCommitter{},
kafka.Message{Topic: "vehicle.fields.go.jt808.v1", Partition: 4, Offset: 30, HighWaterMark: 41, Value: payload},
@@ -150,6 +151,7 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) {
`vehicle_stat_cache_entries{cache="last_total_mileage"} 300`,
`vehicle_stat_cache_entries{cache="source_seen"} 3`,
`vehicle_stat_cache_entries{cache="projection"} 280`,
`vehicle_stat_pending_projection_entries 12`,
`vehicle_stat_cache_entries{cache="baseline"} 295`,
`vehicle_stat_cache_max_entries{cache="last_total_mileage"} 1000000`,
`vehicle_stat_cache_max_entries{cache="source_seen"} 1000000`,
@@ -177,6 +179,52 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) {
}
}
func TestFlushPendingProjectionsRecordsEventualProjectionMetrics(t *testing.T) {
registry := metrics.NewRegistry()
flusher := &recordingPendingProjectionFlusher{
result: stats.ProjectionFlushResult{Attempted: 3, Written: 3},
}
readyBefore := time.Date(2026, 7, 19, 15, 4, 45, 0, time.Local)
flushPendingProjections(context.Background(), discardStatLogger{}, registry, flusher, readyBefore)
if flusher.calls != 1 || !flusher.readyBefore.Equal(readyBefore) {
t.Fatalf("flusher calls=%d readyBefore=%s, want one call at %s", flusher.calls, flusher.readyBefore, readyBefore)
}
text := registry.Render()
for _, want := range []string{
`vehicle_stat_pending_projections_total{status="attempted"} 3`,
`vehicle_stat_pending_projections_total{status="written"} 3`,
`vehicle_stat_last_pending_projection_unix_seconds{status="written"} `,
} {
if !strings.Contains(text, want) {
t.Fatalf("pending projection metrics missing %s:\n%s", want, text)
}
}
}
func TestFlushPendingProjectionsRetainsErrorMetricForRetry(t *testing.T) {
registry := metrics.NewRegistry()
flusher := &recordingPendingProjectionFlusher{
result: stats.ProjectionFlushResult{Attempted: 3, Written: 1, Failed: 2},
err: errors.New("temporary projection failure"),
}
flushPendingProjections(context.Background(), discardStatLogger{}, registry, flusher, time.Time{})
text := registry.Render()
for _, want := range []string{
`vehicle_stat_pending_projections_total{status="attempted"} 3`,
`vehicle_stat_pending_projections_total{status="written"} 1`,
`vehicle_stat_pending_projections_total{status="error"} 2`,
`vehicle_stat_last_pending_projection_unix_seconds{status="error"} `,
} {
if !strings.Contains(text, want) {
t.Fatalf("pending projection error metrics missing %s:\n%s", want, text)
}
}
}
func TestRecordStatWriteE2EDurationSkipsMissingReceivedAt(t *testing.T) {
registry := metrics.NewRegistry()
@@ -1232,6 +1280,19 @@ type discardStatLogger struct{}
func (discardStatLogger) Error(string, ...any) {}
func (discardStatLogger) Warn(string, ...any) {}
type recordingPendingProjectionFlusher struct {
result stats.ProjectionFlushResult
err error
calls int
readyBefore time.Time
}
func (f *recordingPendingProjectionFlusher) FlushPendingProjections(_ context.Context, readyBefore time.Time) (stats.ProjectionFlushResult, error) {
f.calls++
f.readyBefore = readyBefore
return f.result, f.err
}
var errTestStatAppend = errors.New("test stat append failed")
func committedOffsets(messages []kafka.Message) []int64 {