shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(stats): store source mileage candidates

Browse Source
This commit is contained in:
lingniu
2026-07-08 14:25:09 +08:00
parent 009e50a902
commit 08e2d8c0f0
5 changed files with 298 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
go/vehicle-gateway/internal/stats/daily_metric.go
View File

@@ -29,8 +29,10 @@ type MetricSample struct {
Protocol envelope.Protocol
StatDate string
TotalMileageKM float64
EventTime time.Time
SourceKey string
Phone string
DeviceID string
SourceEndpoint string
}
@@ -133,25 +135,16 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
Protocol: env.Protocol,
StatDate: statDate,
TotalMileageKM: totalMileage,
EventTime: time.UnixMilli(eventMS).In(loc),
SourceKey: sourceKey(env),
Phone: strings.TrimSpace(env.Phone),
DeviceID: strings.TrimSpace(env.DeviceID),
SourceEndpoint: strings.TrimSpace(env.SourceEndpoint),
}}, nil
}
func sourceKey(env envelope.FrameEnvelope) string {
parts := []string{
strings.TrimSpace(env.Phone),
strings.TrimSpace(env.DeviceID),
strings.TrimSpace(env.SourceEndpoint),
}
var kept []string
for _, part := range parts {
if part != "" {
kept = append(kept, part)
}
}
return strings.Join(kept, "@")
return SourceKey(env.Protocol, env.Phone, env.DeviceID, NormalizeSourceIP(env.SourceEndpoint))
}
func isDuplicateColumnError(err error) bool {

27
go/vehicle-gateway/internal/stats/schema.go
View File

@@ -40,3 +40,30 @@ const DataSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_data_source (
UNIQUE KEY uk_protocol_source_ip (protocol, source_ip),
KEY idx_protocol_enabled_priority (protocol, enabled, trust_priority)
)`
const DailyMileageSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage_source (
vin VARCHAR(32) NOT NULL,
stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL,
source_key VARCHAR(256) NOT NULL,
source_ip VARCHAR(64) NOT NULL,
source_endpoint VARCHAR(128) NULL,
phone VARCHAR(32) NULL,
device_id VARCHAR(64) NULL,
platform_name VARCHAR(128) NULL,
first_total_mileage_km DECIMAL(18,3) NULL,
latest_total_mileage_km DECIMAL(18,3) NULL,
daily_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0,
sample_count BIGINT NOT NULL DEFAULT 0,
first_event_time DATETIME NULL,
latest_event_time DATETIME NULL,
quality_status VARCHAR(32) NOT NULL DEFAULT 'OK',
quality_reason VARCHAR(255) NULL,
is_selected TINYINT(1) NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (vin, stat_date, protocol, source_key),
KEY idx_protocol_date (protocol, stat_date),
KEY idx_source_ip (protocol, source_ip),
KEY idx_selected (stat_date, protocol, is_selected)
)`

151
go/vehicle-gateway/internal/stats/source_mileage.go Normal file
View File

@@ -0,0 +1,151 @@
package stats
import (
"context"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
const (
QualityOK = "OK"
QualityNoPreviousBaseline = "NO_PREVIOUS_BASELINE"
QualityInvalidDelta = "INVALID_DELTA"
)
type SourceMileageSample struct {
VIN string
StatDate string
Protocol envelope.Protocol
SourceKey string
SourceIP string
SourceEndpoint string
Phone string
DeviceID string
PlatformName string
FirstTotalKM float64
LatestTotalKM float64
DailyKM float64
SampleCount int64
FirstEventTime time.Time
LatestEventTime time.Time
QualityStatus string
QualityReason string
}
func SourceKey(protocol envelope.Protocol, phone string, deviceID string, sourceIP string) string {
identity := strings.TrimSpace(phone)
if identity == "" {
identity = strings.TrimSpace(deviceID)
}
if identity == "" {
identity = "unknown"
}
return string(protocol) + ":" + identity + "@" + strings.TrimSpace(sourceIP)
}
func SourceMileageSampleFromMetric(sample MetricSample, identity SourceIdentity) SourceMileageSample {
eventTime := sample.EventTime
return SourceMileageSample{
VIN: sample.VIN,
StatDate: sample.StatDate,
Protocol: sample.Protocol,
SourceKey: SourceKey(sample.Protocol, sample.Phone, sample.DeviceID, identity.SourceIP),
SourceIP: identity.SourceIP,
SourceEndpoint: identity.SourceEndpoint,
Phone: sample.Phone,
DeviceID: sample.DeviceID,
FirstTotalKM: sample.TotalMileageKM,
LatestTotalKM: sample.TotalMileageKM,
DailyKM: 0,
SampleCount: 1,
FirstEventTime: eventTime,
LatestEventTime: eventTime,
QualityStatus: QualityOK,
QualityReason: "realtime_sample",
}
}
func UpsertSourceMileage(ctx context.Context, exec Execer, sample SourceMileageSample) error {
if exec == nil {
panic("stats execer must not be nil")
}
if sample.VIN == "" || sample.SourceKey == "" {
return nil
}
if sample.QualityStatus == "" {
sample.QualityStatus = QualityOK
}
_, err := exec.ExecContext(ctx, upsertSourceMileageSQL,
sample.VIN,
sample.StatDate,
string(sample.Protocol),
sample.SourceKey,
sample.SourceIP,
sample.SourceEndpoint,
sample.Phone,
sample.DeviceID,
sample.PlatformName,
sample.FirstTotalKM,
sample.LatestTotalKM,
sample.DailyKM,
sample.SampleCount,
sample.FirstEventTime,
sample.LatestEventTime,
sample.QualityStatus,
sample.QualityReason,
)
return err
}
const upsertSourceMileageSQL = `
INSERT INTO vehicle_daily_mileage_source
(vin, stat_date, protocol, source_key, source_ip, source_endpoint, phone, device_id, platform_name,
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count,
first_event_time, latest_event_time, quality_status, quality_reason)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
source_ip = VALUES(source_ip),
source_endpoint = VALUES(source_endpoint),
phone = VALUES(phone),
device_id = VALUES(device_id),
platform_name = COALESCE(VALUES(platform_name), platform_name),
first_total_mileage_km = CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END,
latest_total_mileage_km = CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
END,
/* daily_mileage_km = VALUES(daily_mileage_km) */
daily_mileage_km = GREATEST(
CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE latest_total_mileage_km
END,
VALUES(latest_total_mileage_km)
) - CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END,
sample_count = sample_count + VALUES(sample_count),
first_event_time = CASE
WHEN first_event_time IS NULL OR VALUES(first_event_time) < first_event_time
THEN VALUES(first_event_time)
ELSE first_event_time
END,
latest_event_time = CASE
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) > latest_event_time
THEN VALUES(latest_event_time)
ELSE latest_event_time
END,
quality_status = VALUES(quality_status),
quality_reason = VALUES(quality_reason),
updated_at = CURRENT_TIMESTAMP
`

63
go/vehicle-gateway/internal/stats/source_mileage_test.go Normal file
View File

@@ -0,0 +1,63 @@
package stats
import (
"context"
"strings"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestSourceKeyUsesProtocolDeviceAndSourceIP(t *testing.T) {
got := SourceKey(envelope.ProtocolJT808, "13307765812", "", "115.231.168.135")
if got != "JT808:13307765812@115.231.168.135" {
t.Fatalf("source key = %q", got)
}
got = SourceKey(envelope.ProtocolGB32960, "", "iccid-1", "202.98.117.132")
if got != "GB32960:iccid-1@202.98.117.132" {
t.Fatalf("source key = %q", got)
}
}
func TestUpsertSourceMileageWritesCandidateRow(t *testing.T) {
exec := &recordingExec{}
sample := SourceMileageSample{
VIN: "LA9GG64L7PBAF4001",
StatDate: "2026-07-08",
Protocol: envelope.ProtocolJT808,
SourceKey: "JT808:13307765812@115.231.168.135",
SourceIP: "115.231.168.135",
SourceEndpoint: "115.231.168.135:20215",
Phone: "13307765812",
FirstTotalKM: 4100.8,
LatestTotalKM: 4123.9,
DailyKM: 23.1,
SampleCount: 2,
FirstEventTime: time.Date(2026, 7, 8, 0, 1, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
LatestEventTime: time.Date(2026, 7, 8, 23, 59, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
QualityStatus: QualityOK,
QualityReason: "same_source_continuous",
}
if err := UpsertSourceMileage(context.Background(), exec, sample); err != nil {
t.Fatalf("UpsertSourceMileage() error = %v", err)
}
if len(exec.calls) != 1 {
t.Fatalf("exec calls = %d", len(exec.calls))
}
sql := exec.calls[0].query
for _, want := range []string{
"INSERT INTO vehicle_daily_mileage_source",
"ON DUPLICATE KEY UPDATE",
"daily_mileage_km = VALUES(daily_mileage_km)",
"quality_status = VALUES(quality_status)",
} {
if !strings.Contains(sql, want) {
t.Fatalf("candidate upsert missing %q: %s", want, sql)
}
}
if got := exec.calls[0].args[0]; got != "LA9GG64L7PBAF4001" {
t.Fatalf("first arg = %#v", got)
}
}