shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: tighten source-aware mileage projection

Browse Source
This commit is contained in:
lingniu
2026-07-08 15:41:59 +08:00
parent 800ca1a8b1
commit e0ae57828e
7 changed files with 500 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
go/vehicle-gateway/internal/stats/daily_metric_test.go
View File

@@ -3,11 +3,14 @@ package stats
import (
"context"
"database/sql"
"database/sql/driver"
"errors"
"math"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
@@ -173,8 +176,8 @@ func TestWriterAppendWritesSourceCandidateAndProjection(t *testing.T) {
if err := writer.Append(context.Background(), event); err != nil {
t.Fatalf("Append() error = %v", err)
}
if len(exec.calls) != 5 {
t.Fatalf("exec calls = %d, want source + candidate + clear + project + mark", len(exec.calls))
if len(exec.calls) != 6 {
t.Fatalf("exec calls = %d, want source + candidate + clear + project + mark + cleanup", len(exec.calls))
}
if !strings.Contains(exec.calls[0].query, "INSERT INTO vehicle_data_source") {
t.Fatalf("first call should upsert source: %s", exec.calls[0].query)
@@ -187,6 +190,216 @@ func TestWriterAppendWritesSourceCandidateAndProjection(t *testing.T) {
}
}
func TestWriterAppendDedupesRealtimeMileagePerSource(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
base := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "LA9GG64L7PBAF4001",
Phone: "13307765812",
EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(),
Fields: map[string]any{
"jt808.location.total_mileage_km": 4123.9,
},
}
sourceA := base
sourceA.SourceEndpoint = "115.231.168.135:20215"
if err := writer.Append(context.Background(), sourceA); err != nil {
t.Fatalf("Append(sourceA) error = %v", err)
}
sourceB := base
sourceB.SourceEndpoint = "115.159.85.149:28316"
sourceB.ReceivedAtMS = sourceB.EventTimeMS + 1000
if err := writer.Append(context.Background(), sourceB); err != nil {
t.Fatalf("Append(sourceB) error = %v", err)
}
sourceADuplicate := sourceA
sourceADuplicate.ReceivedAtMS = sourceA.EventTimeMS + 2000
if err := writer.Append(context.Background(), sourceADuplicate); err != nil {
t.Fatalf("Append(sourceADuplicate) error = %v", err)
}
if got := countExecQueries(exec.calls, "INSERT INTO vehicle_daily_mileage_source"); got != 2 {
t.Fatalf("candidate upserts = %d, want 2", got)
}
if got := countExecQueries(exec.calls, "INSERT INTO vehicle_data_source"); got != 2 {
t.Fatalf("source upserts = %d, want 2", got)
}
}
func TestWriterAppendWritesNoPreviousBaselineCandidateAndCleansProjection(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
writer := NewWriter(db, loc)
eventTime := time.Date(2026, 7, 8, 13, 20, 0, 0, loc)
event := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "LA9GG64L7PBAF4001",
Phone: "13307765812",
SourceEndpoint: "115.231.168.135:20215",
EventTimeMS: eventTime.UnixMilli(),
Fields: map[string]any{
"jt808.location.total_mileage_km": 4123.9,
},
}
mock.ExpectExec(`INSERT INTO vehicle_data_source`).
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-07", "JT808", "JT808:13307765812@115.231.168.135").
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}))
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
"JT808:13307765812@115.231.168.135",
"115.231.168.135",
"115.231.168.135:20215",
"13307765812",
"",
"",
float64(4123.9),
float64(4123.9),
float64(0),
int64(1),
eventTime,
eventTime,
QualityNoPreviousBaseline,
"missing_previous_source",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)).
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
int64(1000),
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
).
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
).
WillReturnResult(sqlmock.NewResult(0, 1))
if err := writer.Append(context.Background(), event); err != nil {
t.Fatalf("Append() error = %v", err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
writer := NewWriter(db, loc)
previousTime := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
currentTime := time.Date(2026, 7, 8, 13, 20, 0, 0, loc)
event := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "LA9GG64L7PBAF4001",
Phone: "13307765812",
SourceEndpoint: "115.231.168.135:20215",
EventTimeMS: currentTime.UnixMilli(),
Fields: map[string]any{
"jt808.location.total_mileage_km": 4123.9,
},
}
mock.ExpectExec(`INSERT INTO vehicle_data_source`).
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-07", "JT808", "JT808:13307765812@115.231.168.135").
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).
AddRow(4100.8, previousTime))
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
"JT808:13307765812@115.231.168.135",
"115.231.168.135",
"115.231.168.135:20215",
"13307765812",
"",
"",
float64(4100.8),
float64(4123.9),
approxFloat64{want: 23.1, tolerance: 0.000001},
int64(1),
previousTime,
currentTime,
QualityOK,
"same_source_previous_day",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
int64(1000),
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
).
WillReturnResult(sqlmock.NewResult(0, 0))
if err := writer.Append(context.Background(), event); err != nil {
t.Fatalf("Append() error = %v", err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
@@ -206,7 +419,7 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
t.Fatalf("Append() error = %v", err)
}
if len(exec.calls) != 11 {
if len(exec.calls) != 12 {
t.Fatalf("exec calls = %d", len(exec.calls))
}
for i, want := range []string{
@@ -277,8 +490,8 @@ func TestWriterSkipsConsecutiveDuplicateMileageSamples(t *testing.T) {
t.Fatalf("duplicate Append() error = %v", err)
}
if len(exec.calls) != 3 {
t.Fatalf("exec calls = %d, want 3", len(exec.calls))
if len(exec.calls) != 4 {
t.Fatalf("exec calls = %d, want 4", len(exec.calls))
}
}
@@ -301,8 +514,8 @@ func TestWriterDoesNotCacheMileageWhenUpsertFails(t *testing.T) {
t.Fatalf("retry Append() error = %v", err)
}
if len(exec.calls) != 4 {
t.Fatalf("exec calls = %d, want 4", len(exec.calls))
if len(exec.calls) != 5 {
t.Fatalf("exec calls = %d, want 5", len(exec.calls))
}
}
@@ -326,14 +539,24 @@ func TestWriterKeepsOnlyLatestDateInMileageCache(t *testing.T) {
t.Fatalf("next-day Append() error = %v", err)
}
if len(exec.calls) != 6 {
t.Fatalf("exec calls = %d, want 6", len(exec.calls))
if len(exec.calls) != 8 {
t.Fatalf("exec calls = %d, want 8", len(exec.calls))
}
if len(writer.lastTotalMileage) != 1 {
t.Fatalf("cache entries = %d, want 1", len(writer.lastTotalMileage))
}
}
func countExecQueries(calls []execCall, fragment string) int {
count := 0
for _, call := range calls {
if strings.Contains(call.query, fragment) {
count++
}
}
return count
}
type execCall struct {
query string
args []any
@@ -353,3 +576,16 @@ func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any
}
return nil, nil
}
type approxFloat64 struct {
want float64
tolerance float64
}
func (m approxFloat64) Match(value driver.Value) bool {
got, ok := value.(float64)
if !ok {
return false
}
return math.Abs(got-m.want) <= m.tolerance
}