shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(stats): flush throttled daily mileage projections

Browse Source
This commit is contained in:
lingniu
2026-07-19 16:00:13 +08:00
parent 81532d91b7
commit fa60bbb2c8
6 changed files with 381 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
go/vehicle-gateway/internal/stats/daily_metric.go
View File

@@ -3,7 +3,9 @@ package stats
import (
"context"
"database/sql"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
@@ -44,6 +46,7 @@ type Writer struct {
lastTotalMileage map[string]float64
lastSourceSeen map[string]time.Time
lastProjection map[string]projectionCacheEntry
pendingProjection map[string]pendingProjectionEntry
baselineCache map[string]sourceBaselineCacheEntry
mileageKeysByPrefix map[string]map[string]struct{}
projectionKeysByPrefix map[string]map[string]struct{}
@@ -86,21 +89,28 @@ type AppendResult struct {
ProjectionsSkippedThrottled int
}
type ProjectionFlushResult struct {
Attempted int
Written int
Failed int
}
type CacheStats struct {
LastTotalMileageEntries int
LastSourceSeenEntries int
LastProjectionEntries int
BaselineEntries int
MaxEntries int
LastCleanupAt time.Time
LastCleanupTotalMileage int
LastCleanupSourceSeen int
LastCleanupProjection int
LastCleanupBaseline int
TotalMileageEvictions int
SourceSeenEvictions int
ProjectionEvictions int
BaselineEvictions int
LastTotalMileageEntries int
LastSourceSeenEntries int
LastProjectionEntries int
PendingProjectionEntries int
BaselineEntries int
MaxEntries int
LastCleanupAt time.Time
LastCleanupTotalMileage int
LastCleanupSourceSeen int
LastCleanupProjection int
LastCleanupBaseline int
TotalMileageEvictions int
SourceSeenEvictions int
ProjectionEvictions int
BaselineEvictions int
}
type cacheCleanupStats struct {
@@ -130,6 +140,7 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
lastTotalMileage: map[string]float64{},
lastSourceSeen: map[string]time.Time{},
lastProjection: map[string]projectionCacheEntry{},
pendingProjection: map[string]pendingProjectionEntry{},
baselineCache: map[string]sourceBaselineCacheEntry{},
mileageKeysByPrefix: map[string]map[string]struct{}{},
projectionKeysByPrefix: map[string]map[string]struct{}{},
@@ -209,20 +220,21 @@ func (w *Writer) CacheStats() CacheStats {
w.mu.Lock()
defer w.mu.Unlock()
return CacheStats{
LastTotalMileageEntries: len(w.lastTotalMileage),
LastSourceSeenEntries: len(w.lastSourceSeen),
LastProjectionEntries: len(w.lastProjection),
BaselineEntries: len(w.baselineCache),
MaxEntries: w.maxCacheEntries,
LastCleanupAt: w.lastCacheCleanup,
LastCleanupTotalMileage: w.lastCacheCleanupStats.totalMileage,
LastCleanupSourceSeen: w.lastCacheCleanupStats.sourceSeen,
LastCleanupProjection: w.lastCacheCleanupStats.projection,
LastCleanupBaseline: w.lastCacheCleanupStats.baseline,
TotalMileageEvictions: w.cacheEvictions.totalMileage,
SourceSeenEvictions: w.cacheEvictions.sourceSeen,
ProjectionEvictions: w.cacheEvictions.projection,
BaselineEvictions: w.cacheEvictions.baseline,
LastTotalMileageEntries: len(w.lastTotalMileage),
LastSourceSeenEntries: len(w.lastSourceSeen),
LastProjectionEntries: len(w.lastProjection),
PendingProjectionEntries: len(w.pendingProjection),
BaselineEntries: len(w.baselineCache),
MaxEntries: w.maxCacheEntries,
LastCleanupAt: w.lastCacheCleanup,
LastCleanupTotalMileage: w.lastCacheCleanupStats.totalMileage,
LastCleanupSourceSeen: w.lastCacheCleanupStats.sourceSeen,
LastCleanupProjection: w.lastCacheCleanupStats.projection,
LastCleanupBaseline: w.lastCacheCleanupStats.baseline,
TotalMileageEvictions: w.cacheEvictions.totalMileage,
SourceSeenEvictions: w.cacheEvictions.sourceSeen,
ProjectionEvictions: w.cacheEvictions.projection,
BaselineEvictions: w.cacheEvictions.baseline,
}
}
@@ -327,6 +339,8 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
if projectDaily {
w.markProjected(sample)
result.ProjectionsWritten++
} else {
w.markProjectionPending(sample, time.Now())
}
w.markMileageWritten(sample)
result.SamplesWritten++
@@ -474,12 +488,94 @@ func (w *Writer) markProjected(sample MetricSample) {
entry.sourceKeys = map[string]struct{}{}
}
entry.sourceKeys[sample.SourceKey] = struct{}{}
entry.projectedAt = sample.EventTime
if entry.projectedAt.IsZero() || sample.EventTime.After(entry.projectedAt) {
entry.projectedAt = sample.EventTime
}
w.lastProjection[key] = entry
if pending, ok := w.pendingProjection[key]; ok && !pending.sample.EventTime.After(sample.EventTime) {
delete(w.pendingProjection, key)
}
w.addCacheKeyLocked(w.projectionKeysByPrefix, prefix, key)
w.enforceCacheLimitsLocked()
}
func (w *Writer) markProjectionPending(sample MetricSample, queuedAt time.Time) {
key := projectionCacheKey(sample)
prefix := projectionCachePrefix(sample)
w.mu.Lock()
defer w.mu.Unlock()
current, ok := w.pendingProjection[key]
if !ok || sample.EventTime.After(current.sample.EventTime) ||
(sample.EventTime.Equal(current.sample.EventTime) && queuedAt.After(current.queuedAt)) {
w.pendingProjection[key] = pendingProjectionEntry{sample: sample, queuedAt: queuedAt}
}
w.addCacheKeyLocked(w.projectionKeysByPrefix, prefix, key)
w.enforceCacheLimitsLocked()
}
// FlushPendingProjections closes the eventual-consistency gap introduced by
// projection throttling. Source candidates are persisted immediately, while
// final projections are flushed after their throttle window even when a
// vehicle stops reporting before another message can trigger projection.
func (w *Writer) FlushPendingProjections(ctx context.Context, readyBefore time.Time) (ProjectionFlushResult, error) {
type pendingItem struct {
key string
entry pendingProjectionEntry
}
w.mu.Lock()
items := make([]pendingItem, 0, len(w.pendingProjection))
for key, entry := range w.pendingProjection {
if readyBefore.IsZero() || !entry.queuedAt.After(readyBefore) {
items = append(items, pendingItem{key: key, entry: entry})
}
}
w.mu.Unlock()
sort.Slice(items, func(i, j int) bool {
return items[i].key < items[j].key
})
var result ProjectionFlushResult
var flushErr error
for _, item := range items {
if err := ctx.Err(); err != nil {
return result, errors.Join(flushErr, err)
}
result.Attempted++
if err := w.projectPendingMileage(ctx, item.entry.sample); err != nil {
result.Failed++
flushErr = errors.Join(flushErr, fmt.Errorf("project pending mileage %s: %w", item.key, err))
continue
}
w.markProjected(item.entry.sample)
result.Written++
}
return result, flushErr
}
func (w *Writer) projectPendingMileage(ctx context.Context, sample MetricSample) error {
project := func(exec Execer) error {
if strings.Contains(sample.SourceKey, platformSourceKeyPrefix) {
if err := NormalizePlatformSourceMileage(ctx, exec, sample.VIN, sample.StatDate, sample.Protocol); err != nil {
return err
}
}
return projectDailyMileageWithExec(ctx, exec, sample.VIN, sample.StatDate, sample.Protocol)
}
if beginner, ok := w.exec.(txBeginner); ok {
tx, err := beginner.BeginTx(ctx, nil)
if err != nil {
return err
}
if err := project(tx); err != nil {
_ = tx.Rollback()
return err
}
return tx.Commit()
}
return project(w.exec)
}
func (w *Writer) seenSameMileage(sample MetricSample) bool {
key := mileageCacheKey(sample)
w.mu.Lock()
@@ -719,6 +815,7 @@ func (w *Writer) deleteBaselineKeyLocked(key string) {
func (w *Writer) deleteProjectionKeyLocked(key string) {
delete(w.lastProjection, key)
delete(w.pendingProjection, key)
w.deleteIndexedCacheKeyLocked(w.projectionKeysByPrefix, cacheKeyPrefix(key), key)
}
@@ -866,6 +963,11 @@ type projectionCacheEntry struct {
sourceKeys map[string]struct{}
}
type pendingProjectionEntry struct {
sample MetricSample
queuedAt time.Time
}
type sourceBaselineCacheEntry struct {
baseline sourceBaseline
found bool

102
go/vehicle-gateway/internal/stats/daily_metric_test.go
View File

@@ -797,6 +797,97 @@ func TestWriterAppendThrottlesDailyProjectionPerVehicleProtocolDate(t *testing.T
}
}
func TestWriterFlushesFinalProjectionWhenVehicleStopsInsideThrottleWindow(t *testing.T) {
exec := &recordingExec{}
loc := time.FixedZone("Asia/Shanghai", 8*3600)
writer := NewWriter(exec, loc)
base := managedTestSource(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "LMRKH9AC7R1004120",
Phone: "64341233712",
SourceEndpoint: "115.159.85.149:7339",
Fields: map[string]any{
"jt808.location.total_mileage_km": 4732.6,
},
})
first := base
first.EventTimeMS = time.Date(2026, 7, 19, 15, 4, 45, 0, loc).UnixMilli()
if err := writer.Append(context.Background(), first); err != nil {
t.Fatalf("first Append() error = %v", err)
}
final := base
final.EventTimeMS = time.Date(2026, 7, 19, 15, 4, 52, 0, loc).UnixMilli()
final.Fields = map[string]any{"jt808.location.total_mileage_km": 4740.7}
if err := writer.Append(context.Background(), final); err != nil {
t.Fatalf("final Append() error = %v", err)
}
if got := writer.CacheStats().PendingProjectionEntries; got != 1 {
t.Fatalf("pending projections = %d, want final throttled sample queued", got)
}
beforeDue, err := writer.FlushPendingProjections(context.Background(), time.Now().Add(-time.Hour))
if err != nil {
t.Fatalf("early FlushPendingProjections() error = %v", err)
}
if beforeDue.Attempted != 0 || beforeDue.Written != 0 {
t.Fatalf("early flush result = %+v, want no eligible projection", beforeDue)
}
flushed, err := writer.FlushPendingProjections(context.Background(), time.Time{})
if err != nil {
t.Fatalf("FlushPendingProjections() error = %v", err)
}
if flushed.Attempted != 1 || flushed.Written != 1 {
t.Fatalf("flush result = %+v, want final projection written", flushed)
}
if got := countExecQueries(exec.calls, "INSERT INTO vehicle_daily_mileage\n"); got != 2 {
t.Fatalf("daily projections = %d, want initial plus eventual final projection", got)
}
if got := writer.CacheStats().PendingProjectionEntries; got != 0 {
t.Fatalf("pending projections after flush = %d, want 0", got)
}
key := projectionCacheKey(MetricSample{
VIN: "LMRKH9AC7R1004120",
Protocol: envelope.ProtocolJT808,
StatDate: "2026-07-19",
})
wantProjectedAt := time.UnixMilli(final.EventTimeMS).In(loc)
if got := writer.lastProjection[key].projectedAt; !got.Equal(wantProjectedAt) {
t.Fatalf("last projected event = %s, want %s", got, wantProjectedAt)
}
}
func TestWriterPendingProjectionFailureDoesNotBlockOtherVehicles(t *testing.T) {
exec := &selectiveFailingExec{failVIN: "FAIL-VIN"}
loc := time.FixedZone("Asia/Shanghai", 8*3600)
writer := NewWriter(exec, loc)
writer.projectionInterval = time.Hour
for _, vin := range []string{"FAIL-VIN", "OK-VIN"} {
sample := MetricSample{
VIN: vin,
Protocol: envelope.ProtocolJT808,
StatDate: "2026-07-19",
SourceKey: "jt808:test",
EventTime: time.Date(2026, 7, 19, 15, 4, 52, 0, loc),
TotalMileageKM: 100,
}
writer.markProjectionPending(sample, time.Now().Add(-time.Hour))
}
result, err := writer.FlushPendingProjections(context.Background(), time.Time{})
if err == nil {
t.Fatal("FlushPendingProjections() error = nil, want failed vehicle reported")
}
if result.Attempted != 2 || result.Written != 1 || result.Failed != 1 {
t.Fatalf("flush result = %+v, want one failure and one successful projection", result)
}
if got := writer.CacheStats().PendingProjectionEntries; got != 1 {
t.Fatalf("pending projections after partial failure = %d, want only failed vehicle retained", got)
}
}
func TestWriterAppendProjectsImmediatelyForNewSource(t *testing.T) {
exec := &recordingExec{}
loc := time.FixedZone("Asia/Shanghai", 8*3600)
@@ -2321,6 +2412,17 @@ type recordingExec struct {
errs []error
}
type selectiveFailingExec struct {
failVIN string
}
func (e *selectiveFailingExec) ExecContext(_ context.Context, _ string, args ...any) (sql.Result, error) {
if len(args) > 0 && fmt.Sprint(args[0]) == e.failVIN {
return nil, errors.New("test projection failure")
}
return nil, nil
}
type concurrentExec struct {
mu sync.Mutex
calls int