shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(mileage): recover sparse source daily distance

Browse Source
This commit is contained in:
lingniu
2026-07-19 22:57:31 +08:00
parent e45daff528
commit 4c64c4c738
11 changed files with 364 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
go/vehicle-gateway/internal/stats/daily_metric.go
View File

@@ -16,12 +16,13 @@ import (
)
const (
maxNegativeMileageJitterKM = 1.0
defaultCacheRetention = 72 * time.Hour
defaultCacheCleanupInterval = 10 * time.Minute
defaultBaselineMissTTL = time.Minute
defaultBaselineHitTTL = 5 * time.Minute
defaultMaxCacheEntries = 1000000
maxNegativeMileageJitterKM = 1.0
defaultCacheRetention = 72 * time.Hour
defaultCacheCleanupInterval = 10 * time.Minute
defaultBaselineMissTTL = time.Minute
defaultBaselineHitTTL = 5 * time.Minute
defaultMaxCacheEntries = 1000000
defaultGPSAccumulationInterval = 10 * time.Second
)
type Execer interface {
@@ -29,28 +30,30 @@ type Execer interface {
}
type Writer struct {
exec Execer
query Queryer
loc *time.Location
sourceTouchInterval time.Duration
projectionInterval time.Duration
cacheRetention time.Duration
cacheCleanupInterval time.Duration
baselineMissTTL time.Duration
baselineHitTTL time.Duration
maxCacheEntries int
lastCacheCleanup time.Time
lastCacheCleanupStats cacheCleanupStats
cacheEvictions cacheCleanupStats
mu sync.Mutex
lastTotalMileage map[string]float64
lastSourceSeen map[string]time.Time
lastProjection map[string]projectionCacheEntry
pendingProjection map[string]pendingProjectionEntry
baselineCache map[string]sourceBaselineCacheEntry
mileageKeysByPrefix map[string]map[string]struct{}
projectionKeysByPrefix map[string]map[string]struct{}
baselineKeysByPrefix map[string]map[string]struct{}
exec Execer
query Queryer
loc *time.Location
sourceTouchInterval time.Duration
projectionInterval time.Duration
cacheRetention time.Duration
cacheCleanupInterval time.Duration
baselineMissTTL time.Duration
baselineHitTTL time.Duration
maxCacheEntries int
lastCacheCleanup time.Time
lastCacheCleanupStats cacheCleanupStats
cacheEvictions cacheCleanupStats
mu sync.Mutex
lastTotalMileage map[string]float64
lastSourceSeen map[string]time.Time
lastProjection map[string]projectionCacheEntry
pendingProjection map[string]pendingProjectionEntry
baselineCache map[string]sourceBaselineCacheEntry
mileageKeysByPrefix map[string]map[string]struct{}
projectionKeysByPrefix map[string]map[string]struct{}
baselineKeysByPrefix map[string]map[string]struct{}
lastGPSAccumulation map[string]time.Time
gpsAccumulationInterval time.Duration
}
type MetricSample struct {
@@ -109,16 +112,19 @@ type CacheStats struct {
LastProjectionEntries int
PendingProjectionEntries int
BaselineEntries int
GPSAccumulationEntries int
MaxEntries int
LastCleanupAt time.Time
LastCleanupTotalMileage int
LastCleanupSourceSeen int
LastCleanupProjection int
LastCleanupBaseline int
LastCleanupGPS int
TotalMileageEvictions int
SourceSeenEvictions int
ProjectionEvictions int
BaselineEvictions int
GPSEvictions int
}
type cacheCleanupStats struct {
@@ -126,6 +132,7 @@ type cacheCleanupStats struct {
sourceSeen int
projection int
baseline int
gps int
}
func NewWriter(exec Execer, loc *time.Location) *Writer {
@@ -136,23 +143,25 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
loc = time.FixedZone("Asia/Shanghai", 8*3600)
}
writer := &Writer{
exec: exec,
loc: loc,
sourceTouchInterval: time.Minute,
projectionInterval: 15 * time.Second,
cacheRetention: defaultCacheRetention,
cacheCleanupInterval: defaultCacheCleanupInterval,
baselineMissTTL: defaultBaselineMissTTL,
baselineHitTTL: defaultBaselineHitTTL,
maxCacheEntries: defaultMaxCacheEntries,
lastTotalMileage: map[string]float64{},
lastSourceSeen: map[string]time.Time{},
lastProjection: map[string]projectionCacheEntry{},
pendingProjection: map[string]pendingProjectionEntry{},
baselineCache: map[string]sourceBaselineCacheEntry{},
mileageKeysByPrefix: map[string]map[string]struct{}{},
projectionKeysByPrefix: map[string]map[string]struct{}{},
baselineKeysByPrefix: map[string]map[string]struct{}{},
exec: exec,
loc: loc,
sourceTouchInterval: time.Minute,
projectionInterval: 15 * time.Second,
cacheRetention: defaultCacheRetention,
cacheCleanupInterval: defaultCacheCleanupInterval,
baselineMissTTL: defaultBaselineMissTTL,
baselineHitTTL: defaultBaselineHitTTL,
maxCacheEntries: defaultMaxCacheEntries,
lastTotalMileage: map[string]float64{},
lastSourceSeen: map[string]time.Time{},
lastProjection: map[string]projectionCacheEntry{},
pendingProjection: map[string]pendingProjectionEntry{},
baselineCache: map[string]sourceBaselineCacheEntry{},
mileageKeysByPrefix: map[string]map[string]struct{}{},
projectionKeysByPrefix: map[string]map[string]struct{}{},
baselineKeysByPrefix: map[string]map[string]struct{}{},
lastGPSAccumulation: map[string]time.Time{},
gpsAccumulationInterval: defaultGPSAccumulationInterval,
}
if query, ok := exec.(Queryer); ok {
writer.query = query
@@ -224,6 +233,15 @@ func (w *Writer) SetMaxCacheEntries(maxEntries int) {
w.enforceCacheLimitsLocked()
}
func (w *Writer) SetGPSAccumulationInterval(interval time.Duration) {
w.mu.Lock()
defer w.mu.Unlock()
if interval < 0 {
interval = 0
}
w.gpsAccumulationInterval = interval
}
func (w *Writer) CacheStats() CacheStats {
w.mu.Lock()
defer w.mu.Unlock()
@@ -233,16 +251,19 @@ func (w *Writer) CacheStats() CacheStats {
LastProjectionEntries: len(w.lastProjection),
PendingProjectionEntries: len(w.pendingProjection),
BaselineEntries: len(w.baselineCache),
GPSAccumulationEntries: len(w.lastGPSAccumulation),
MaxEntries: w.maxCacheEntries,
LastCleanupAt: w.lastCacheCleanup,
LastCleanupTotalMileage: w.lastCacheCleanupStats.totalMileage,
LastCleanupSourceSeen: w.lastCacheCleanupStats.sourceSeen,
LastCleanupProjection: w.lastCacheCleanupStats.projection,
LastCleanupBaseline: w.lastCacheCleanupStats.baseline,
LastCleanupGPS: w.lastCacheCleanupStats.gps,
TotalMileageEvictions: w.cacheEvictions.totalMileage,
SourceSeenEvictions: w.cacheEvictions.sourceSeen,
ProjectionEvictions: w.cacheEvictions.projection,
BaselineEvictions: w.cacheEvictions.baseline,
GPSEvictions: w.cacheEvictions.gps,
}
}
@@ -306,11 +327,12 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
}
var stationaryCandidate *SourceMileageSample
if len(samples) == 0 && hasSource && result.SamplesSkippedMissingMileage > 0 {
if point, ok := GPSMileagePointFromEnvelope(env, identity, w.loc); ok {
_, recovered, recoverErr := AccumulateJT808GPSMileage(ctx, w.exec, point)
if point, ok := GPSMileagePointFromEnvelope(env, identity, w.loc); ok && w.shouldAccumulateGPS(point) {
_, recovered, recoverErr := AccumulateGPSMileage(ctx, w.exec, point)
if recoverErr != nil {
return result, recoverErr
}
w.markGPSAccumulated(point)
if recovered {
result.SamplesFound++
result.SamplesWritten++
@@ -371,6 +393,31 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
return result, nil
}
func (w *Writer) shouldAccumulateGPS(point GPSMileagePoint) bool {
key := gpsMileageStateCacheKey(point)
w.mu.Lock()
defer w.mu.Unlock()
last, ok := w.lastGPSAccumulation[key]
if !ok || w.gpsAccumulationInterval == 0 {
return true
}
return point.EventTime.After(last.Add(w.gpsAccumulationInterval))
}
func (w *Writer) markGPSAccumulated(point GPSMileagePoint) {
key := gpsMileageStateCacheKey(point)
w.mu.Lock()
if previous, ok := w.lastGPSAccumulation[key]; !ok || point.EventTime.After(previous) {
w.lastGPSAccumulation[key] = point.EventTime
}
w.enforceCacheLimitsLocked()
w.mu.Unlock()
}
func gpsMileageStateCacheKey(point GPSMileagePoint) string {
return point.VIN + "|" + point.StatDate + "|" + string(point.Protocol) + "|" + point.SourceKey
}
func (w *Writer) stationaryCarryForward(ctx context.Context, env envelope.FrameEnvelope, identity SourceIdentity) (MetricSample, SourceMileageSample, bool, error) {
if env.Protocol != envelope.ProtocolYutongMQTT || strings.TrimSpace(env.VIN) == "" {
return MetricSample{}, SourceMileageSample{}, false, nil
@@ -739,6 +786,12 @@ func (w *Writer) maybeCleanupCaches(now time.Time) {
cleanupStats.baseline++
}
}
for key, eventTime := range w.lastGPSAccumulation {
if eventTime.IsZero() || eventTime.Before(cutoff) {
delete(w.lastGPSAccumulation, key)
cleanupStats.gps++
}
}
w.lastCacheCleanupStats = cleanupStats
w.enforceCacheLimitsLocked()
}
@@ -966,6 +1019,14 @@ func (w *Writer) enforceCacheLimitsLocked() {
delete(w.lastSourceSeen, victim)
w.cacheEvictions.sourceSeen++
}
for len(w.lastGPSAccumulation) > w.maxCacheEntries {
victim := oldestSourceSeenKey(w.lastGPSAccumulation)
if victim == "" {
break
}
delete(w.lastGPSAccumulation, victim)
w.cacheEvictions.gps++
}
}
func mileageCachePrefix(sample MetricSample) string {