shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(stats): flush throttled daily mileage projections

Browse Source
This commit is contained in:
lingniu
2026-07-19 16:00:13 +08:00
parent 81532d91b7
commit fa60bbb2c8
6 changed files with 381 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
go/vehicle-gateway/cmd/stat-writer/main.go
View File

@@ -95,6 +95,14 @@ func main() {
quarantiner := fileStatMessageQuarantiner{dir: cfg.QuarantineDir}
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "baseline_hit_ttl_seconds", cfg.BaselineHitTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds(), "quarantine_dir", cfg.QuarantineDir)
var background sync.WaitGroup
if cfg.ProjectInterval > 0 {
background.Add(1)
go func() {
defer background.Done()
runPendingProjectionFlusher(ctx, logger, registry, writer, cfg.ProjectInterval)
}()
}
var workers sync.WaitGroup
for workerID := 1; workerID <= cfg.Workers; workerID++ {
workers.Add(1)
@@ -104,6 +112,67 @@ func main() {
}(workerID)
}
workers.Wait()
background.Wait()
if cfg.ProjectInterval > 0 {
flushCtx, cancel := context.WithTimeout(context.Background(), kafkaMessageOperationTimeout)
flushPendingProjections(flushCtx, logger, registry, writer, time.Time{})
cancel()
}
}
type pendingProjectionFlusher interface {
FlushPendingProjections(context.Context, time.Time) (stats.ProjectionFlushResult, error)
}
func runPendingProjectionFlusher(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, flusher pendingProjectionFlusher, interval time.Duration) {
if flusher == nil || interval <= 0 {
return
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case now := <-ticker.C:
operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
flushPendingProjections(operationCtx, logger, registry, flusher, now.Add(-interval))
cancel()
}
}
}
func flushPendingProjections(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, flusher pendingProjectionFlusher, readyBefore time.Time) {
if flusher == nil {
return
}
result, err := flusher.FlushPendingProjections(ctx, readyBefore)
if registry != nil {
if result.Attempted > 0 {
registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "attempted"}, float64(result.Attempted))
}
if result.Written > 0 {
registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "written"}, float64(result.Written))
metrics.RecordLastActivity(registry, "vehicle_stat_last_pending_projection_unix_seconds", metrics.Labels{"status": "written"})
}
if result.Failed > 0 {
registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "error"}, float64(result.Failed))
} else if err != nil {
registry.IncCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "error"})
}
if err != nil {
metrics.RecordLastActivity(registry, "vehicle_stat_last_pending_projection_unix_seconds", metrics.Labels{"status": "error"})
}
}
if err != nil && logger != nil {
logger.Warn("pending daily mileage projection flush failed", "attempted", result.Attempted, "written", result.Written, "failed", result.Failed, "error", err)
}
}
func runStatConsumer(ctx context.Context, logger interface {
@@ -763,6 +832,7 @@ func recordStatCacheMetrics(registry *metrics.Registry, appender statAppender) {
setStatCacheGauge(registry, "last_total_mileage", stats.LastTotalMileageEntries, stats.MaxEntries, stats.LastCleanupTotalMileage, stats.TotalMileageEvictions)
setStatCacheGauge(registry, "source_seen", stats.LastSourceSeenEntries, stats.MaxEntries, stats.LastCleanupSourceSeen, stats.SourceSeenEvictions)
setStatCacheGauge(registry, "projection", stats.LastProjectionEntries, stats.MaxEntries, stats.LastCleanupProjection, stats.ProjectionEvictions)
registry.SetGauge("vehicle_stat_pending_projection_entries", nil, float64(stats.PendingProjectionEntries))
setStatCacheGauge(registry, "baseline", stats.BaselineEntries, stats.MaxEntries, stats.LastCleanupBaseline, stats.BaselineEvictions)
if !stats.LastCleanupAt.IsZero() {
registry.SetGauge("vehicle_stat_cache_last_cleanup_unix_seconds", nil, float64(stats.LastCleanupAt.Unix()))