shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(stats): elect trusted mileage source

Browse Source
This commit is contained in:
lingniu
2026-07-08 13:31:56 +08:00
parent abfed27846
commit a60a628d25
5 changed files with 284 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
go/vehicle-gateway/internal/stats/daily_metric.go
View File

@@ -29,6 +29,9 @@ type MetricSample struct {
Protocol envelope.Protocol
StatDate string
TotalMileageKM float64
SourceKey string
Phone string
SourceEndpoint string
}
func NewWriter(exec Execer, loc *time.Location) *Writer {
@@ -45,6 +48,11 @@ func (w *Writer) EnsureSchema(ctx context.Context) error {
if _, err := w.exec.ExecContext(ctx, DailyMileageTableSQL); err != nil {
return err
}
for _, statement := range DailyMileageAlterSQL {
if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateColumnError(err) {
return err
}
}
return nil
}
@@ -62,7 +70,10 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
sample.StatDate,
string(sample.Protocol),
sample.TotalMileageKM,
sample.TotalMileageKM); err != nil {
sample.TotalMileageKM,
sample.SourceKey,
sample.Phone,
sample.SourceEndpoint); err != nil {
return err
}
w.markMileageWritten(sample)
@@ -122,9 +133,35 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
Protocol: env.Protocol,
StatDate: statDate,
TotalMileageKM: totalMileage,
SourceKey: sourceKey(env),
Phone: strings.TrimSpace(env.Phone),
SourceEndpoint: strings.TrimSpace(env.SourceEndpoint),
}}, nil
}
func sourceKey(env envelope.FrameEnvelope) string {
parts := []string{
strings.TrimSpace(env.Phone),
strings.TrimSpace(env.DeviceID),
strings.TrimSpace(env.SourceEndpoint),
}
var kept []string
for _, part := range parts {
if part != "" {
kept = append(kept, part)
}
}
return strings.Join(kept, "@")
}
func isDuplicateColumnError(err error) bool {
if err == nil {
return false
}
text := strings.ToLower(err.Error())
return strings.Contains(text, "duplicate column") || strings.Contains(text, "1060")
}
type mileageFieldMapping struct {
key string
scale float64
@@ -163,32 +200,86 @@ func mileageMappingsByProtocol(protocol envelope.Protocol) []mileageFieldMapping
const upsertDailyMileageSQL = `
INSERT INTO vehicle_daily_mileage
(vin, stat_date, protocol, daily_mileage_km,
first_total_mileage_km, latest_total_mileage_km, sample_count)
VALUES (?, ?, ?, 0, ?, ?, 1)
first_total_mileage_km, latest_total_mileage_km,
trusted_source_key, trusted_phone, trusted_source_endpoint, sample_count)
VALUES (?, ?, ?, 0, ?, ?, ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE
first_total_mileage_km = CASE
WHEN trusted_source_key IS NOT NULL
AND trusted_source_key <> ''
AND trusted_source_key <> VALUES(trusted_source_key)
AND ABS(COALESCE(latest_total_mileage_km, VALUES(latest_total_mileage_km)) - VALUES(latest_total_mileage_km)) > 50
THEN first_total_mileage_km
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END,
latest_total_mileage_km = CASE
WHEN trusted_source_key IS NOT NULL
AND trusted_source_key <> ''
AND trusted_source_key <> VALUES(trusted_source_key)
AND ABS(COALESCE(latest_total_mileage_km, VALUES(latest_total_mileage_km)) - VALUES(latest_total_mileage_km)) > 50
THEN latest_total_mileage_km
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
END,
daily_mileage_km = GREATEST(
CASE
WHEN trusted_source_key IS NOT NULL
AND trusted_source_key <> ''
AND trusted_source_key <> VALUES(trusted_source_key)
AND ABS(COALESCE(latest_total_mileage_km, VALUES(latest_total_mileage_km)) - VALUES(latest_total_mileage_km)) > 50
THEN latest_total_mileage_km
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE latest_total_mileage_km
END,
VALUES(latest_total_mileage_km)
CASE
WHEN trusted_source_key IS NOT NULL
AND trusted_source_key <> ''
AND trusted_source_key <> VALUES(trusted_source_key)
AND ABS(COALESCE(latest_total_mileage_km, VALUES(latest_total_mileage_km)) - VALUES(latest_total_mileage_km)) > 50
THEN latest_total_mileage_km
ELSE VALUES(latest_total_mileage_km)
END
) - CASE
WHEN trusted_source_key IS NOT NULL
AND trusted_source_key <> ''
AND trusted_source_key <> VALUES(trusted_source_key)
AND ABS(COALESCE(latest_total_mileage_km, VALUES(latest_total_mileage_km)) - VALUES(latest_total_mileage_km)) > 50
THEN first_total_mileage_km
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END,
sample_count = sample_count + 1,
trusted_source_key = CASE
WHEN trusted_source_key IS NULL OR trusted_source_key = ''
THEN VALUES(trusted_source_key)
WHEN trusted_source_key = VALUES(trusted_source_key)
THEN trusted_source_key
WHEN ABS(COALESCE(latest_total_mileage_km, VALUES(latest_total_mileage_km)) - VALUES(latest_total_mileage_km)) <= 50
THEN VALUES(trusted_source_key)
ELSE trusted_source_key
END,
trusted_phone = CASE
WHEN trusted_source_key IS NULL OR trusted_source_key = '' OR trusted_source_key = VALUES(trusted_source_key)
THEN VALUES(trusted_phone)
ELSE trusted_phone
END,
trusted_source_endpoint = CASE
WHEN trusted_source_key IS NULL OR trusted_source_key = '' OR trusted_source_key = VALUES(trusted_source_key)
THEN VALUES(trusted_source_endpoint)
ELSE trusted_source_endpoint
END,
sample_count = CASE
WHEN trusted_source_key IS NOT NULL
AND trusted_source_key <> ''
AND trusted_source_key <> VALUES(trusted_source_key)
AND ABS(COALESCE(latest_total_mileage_km, VALUES(latest_total_mileage_km)) - VALUES(latest_total_mileage_km)) > 50
THEN sample_count
ELSE sample_count + 1
END,
updated_at = CURRENT_TIMESTAMP
`

20
go/vehicle-gateway/internal/stats/daily_metric_test.go
View File

@@ -187,19 +187,23 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
if strings.Contains(exec.calls[0].query, "KEY idx_vin (vin)") {
t.Fatalf("daily mileage table should not keep redundant vin index covered by the primary key: %s", exec.calls[0].query)
}
if len(exec.calls) != 2 {
if len(exec.calls) != 5 {
t.Fatalf("exec calls = %d", len(exec.calls))
}
if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query)
upsertCall := exec.calls[4]
if !strings.Contains(upsertCall.query, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("unexpected upsert sql: %s", upsertCall.query)
}
if strings.Contains(exec.calls[1].query, "metric_key") || strings.Contains(exec.calls[1].query, "metric_unit") {
t.Fatalf("daily mileage upsert should not use generic metric columns: %s", exec.calls[1].query)
if strings.Contains(upsertCall.query, "metric_key") || strings.Contains(upsertCall.query, "metric_unit") {
t.Fatalf("daily mileage upsert should not use generic metric columns: %s", upsertCall.query)
}
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
if !strings.Contains(upsertCall.query, "first_total_mileage_km <= 0") {
t.Fatalf("upsert should ignore legacy zero first mileage: %s", upsertCall.query)
}
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
if !strings.Contains(upsertCall.query, "trusted_source_key") {
t.Fatalf("upsert should track trusted source: %s", upsertCall.query)
}
if got := upsertCall.args[0]; got != "LNBVIN00000000002" {
t.Fatalf("first upsert arg should be vin, got %#v", got)
}
}

9
go/vehicle-gateway/internal/stats/schema.go
View File

@@ -7,9 +7,18 @@ const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage (
daily_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0,
first_total_mileage_km DECIMAL(18,3) NULL,
latest_total_mileage_km DECIMAL(18,3) NULL,
trusted_source_key VARCHAR(256) NULL,
trusted_phone VARCHAR(32) NULL,
trusted_source_endpoint VARCHAR(128) NULL,
sample_count BIGINT NOT NULL DEFAULT 0,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (vin, stat_date, protocol),
KEY idx_stat_date (stat_date),
KEY idx_protocol_date (protocol, stat_date)
)`
var DailyMileageAlterSQL = []string{
"ALTER TABLE vehicle_daily_mileage ADD COLUMN trusted_source_key VARCHAR(256) NULL",
"ALTER TABLE vehicle_daily_mileage ADD COLUMN trusted_phone VARCHAR(32) NULL",
"ALTER TABLE vehicle_daily_mileage ADD COLUMN trusted_source_endpoint VARCHAR(128) NULL",
}