fix(stats): track data sources before mileage sampling

This commit is contained in:
lingniu
2026-07-08 16:46:06 +08:00
parent 44e119331d
commit da669fae94
2 changed files with 80 additions and 14 deletions

View File

@@ -21,8 +21,10 @@ type Writer struct {
exec Execer
query Queryer
loc *time.Location
sourceTouchInterval time.Duration
mu sync.Mutex
lastTotalMileage map[string]float64
lastSourceSeen map[string]time.Time
baselineCache map[string]sourceBaselineCacheEntry
}
@@ -48,7 +50,9 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
writer := &Writer{
exec: exec,
loc: loc,
sourceTouchInterval: time.Minute,
lastTotalMileage: map[string]float64{},
lastSourceSeen: map[string]time.Time{},
baselineCache: map[string]sourceBaselineCacheEntry{},
}
if query, ok := exec.(Queryer); ok {
@@ -76,11 +80,20 @@ func (w *Writer) EnsureSchema(ctx context.Context) error {
}
func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
identity, hasSource := NewSourceIdentity(env.Protocol, env.SourceEndpoint)
if hasSource {
seenAt := w.sourceSeenAt(env)
if w.shouldTouchSource(identity, seenAt) {
if err := UpsertDataSource(ctx, w.exec, identity, seenAt); err != nil {
return err
}
w.markSourceTouched(identity, seenAt)
}
}
samples, err := SamplesFromEnvelope(env, w.loc)
if err != nil {
return err
}
identity, hasSource := NewSourceIdentity(env.Protocol, env.SourceEndpoint)
for _, sample := range samples {
if w.seenSameMileage(sample) {
continue
@@ -88,9 +101,6 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
if !hasSource {
continue
}
if err := UpsertDataSource(ctx, w.exec, identity, sample.EventTime); err != nil {
return err
}
candidate := SourceMileageSampleFromMetric(sample, identity)
if err := w.applyRealtimeBaseline(ctx, &candidate); err != nil {
return err
@@ -106,6 +116,35 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
return nil
}
func (w *Writer) sourceSeenAt(env envelope.FrameEnvelope) time.Time {
eventMS := env.EventTimeMS
if eventMS <= 0 {
eventMS = env.ReceivedAtMS
}
if eventMS > 0 {
return time.UnixMilli(eventMS).In(w.loc)
}
return time.Now().In(w.loc)
}
func (w *Writer) shouldTouchSource(identity SourceIdentity, seenAt time.Time) bool {
key := string(identity.Protocol) + "|" + identity.SourceIP
w.mu.Lock()
defer w.mu.Unlock()
last, ok := w.lastSourceSeen[key]
if ok && !seenAt.After(last.Add(w.sourceTouchInterval)) {
return false
}
return true
}
func (w *Writer) markSourceTouched(identity SourceIdentity, seenAt time.Time) {
key := string(identity.Protocol) + "|" + identity.SourceIP
w.mu.Lock()
w.lastSourceSeen[key] = seenAt
w.mu.Unlock()
}
func (w *Writer) seenSameMileage(sample MetricSample) bool {
key := mileageCacheKey(sample)
w.mu.Lock()

View File

@@ -211,6 +211,33 @@ func TestWriterAppendSkipsMileageWithoutSourceIdentity(t *testing.T) {
}
}
func TestWriterAppendTracksSourceWithoutMileageSample(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
event := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "13307795518",
SourceEndpoint: "222.66.200.68:43614",
EventTimeMS: time.Date(2026, 7, 8, 16, 30, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
Fields: map[string]any{
"jt808.location.speed_kmh": 38.5,
},
}
if err := writer.Append(context.Background(), event); err != nil {
t.Fatalf("Append() error = %v", err)
}
if len(exec.calls) != 1 {
t.Fatalf("exec calls = %d, want source only", len(exec.calls))
}
if !strings.Contains(exec.calls[0].query, "INSERT INTO vehicle_data_source") {
t.Fatalf("source upsert query missing: %s", exec.calls[0].query)
}
if got := exec.calls[0].args[1]; got != "222.66.200.68" {
t.Fatalf("source ip arg = %#v", got)
}
}
func TestWriterAppendDedupesRealtimeMileagePerSource(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))