shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: expand vehicle data platform capabilities

Browse Source
This commit is contained in:
lingniu
2026-07-27 16:46:15 +08:00
parent e3a1f80f86
commit 3c4bece72c
650 changed files with 62155 additions and 2552 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
go/vehicle-gateway/internal/stats/source_mileage.go
View File

@@ -24,23 +24,27 @@ const (
)
type SourceMileageSample struct {
VIN string
StatDate string
Protocol envelope.Protocol
SourceKey string
SourceIP string
SourceEndpoint string
Phone string
DeviceID string
PlatformName string
FirstTotalKM float64
LatestTotalKM float64
DailyKM float64
SampleCount int64
FirstEventTime time.Time
LatestEventTime time.Time
QualityStatus string
QualityReason string
VIN string
StatDate string
Protocol envelope.Protocol
SourceKey string
SourceIP string
SourceEndpoint string
Phone string
DeviceID string
PlatformName string
FirstTotalKM float64
LatestTotalKM float64
DailyKM float64
PureHydrogenMileageKM float64
SampleCount int64
PureHydrogenSampleCount int64
FirstEventTime time.Time
LatestEventTime time.Time
LatestPureHydrogenActive bool
LatestPureHydrogenModeKnown bool
QualityStatus string
QualityReason string
}
func SourceKey(protocol envelope.Protocol, phone string, deviceID string, sourceIP string) string {
@@ -80,26 +84,37 @@ func SourceKeyForSource(protocol envelope.Protocol, phone string, deviceID strin
func SourceMileageSampleFromMetric(sample MetricSample, identity SourceIdentity) SourceMileageSample {
eventTime := sample.EventTime
return SourceMileageSample{
VIN: sample.VIN,
StatDate: sample.StatDate,
Protocol: sample.Protocol,
SourceKey: SourceKeyForSource(sample.Protocol, sample.Phone, sample.DeviceID, identity.SourceIP, identity.SourceKind, identity.SourceCode),
SourceIP: identity.SourceIP,
SourceEndpoint: identity.SourceEndpoint,
Phone: sample.Phone,
DeviceID: sample.DeviceID,
PlatformName: firstNonEmpty(sample.PlatformName, identity.PlatformName),
FirstTotalKM: sample.TotalMileageKM,
LatestTotalKM: sample.TotalMileageKM,
DailyKM: 0,
SampleCount: 1,
FirstEventTime: eventTime,
LatestEventTime: eventTime,
QualityStatus: QualityOK,
QualityReason: "realtime_sample",
VIN: sample.VIN,
StatDate: sample.StatDate,
Protocol: sample.Protocol,
SourceKey: SourceKeyForSource(sample.Protocol, sample.Phone, sample.DeviceID, identity.SourceIP, identity.SourceKind, identity.SourceCode),
SourceIP: identity.SourceIP,
SourceEndpoint: identity.SourceEndpoint,
Phone: sample.Phone,
DeviceID: sample.DeviceID,
PlatformName: firstNonEmpty(sample.PlatformName, identity.PlatformName),
FirstTotalKM: sample.TotalMileageKM,
LatestTotalKM: sample.TotalMileageKM,
DailyKM: 0,
PureHydrogenMileageKM: 0,
SampleCount: 1,
PureHydrogenSampleCount: boolCount(sample.PureHydrogenActive && sample.PureHydrogenModeKnown),
FirstEventTime: eventTime,
LatestEventTime: eventTime,
LatestPureHydrogenActive: sample.PureHydrogenActive,
LatestPureHydrogenModeKnown: sample.PureHydrogenModeKnown,
QualityStatus: QualityOK,
QualityReason: "realtime_sample",
}
}
func boolCount(value bool) int64 {
if value {
return 1
}
return 0
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
value = strings.TrimSpace(value)
@@ -203,9 +218,13 @@ func UpsertSourceMileage(ctx context.Context, exec Execer, sample SourceMileageS
sample.FirstTotalKM,
sample.LatestTotalKM,
sample.DailyKM,
sample.PureHydrogenMileageKM,
sample.SampleCount,
sample.PureHydrogenSampleCount,
sample.FirstEventTime,
sample.LatestEventTime,
sample.LatestPureHydrogenActive,
sample.LatestPureHydrogenModeKnown,
sample.QualityStatus,
sample.QualityReason,
)
@@ -422,18 +441,44 @@ const normalizePlatformMaxMileageSQL = `(` + normalizePlatformQualityWindowDaysS
const normalizePlatformOutsideDailyRangeSQL = `(` + normalizePlatformDailySQL + ` < 0 OR ` + normalizePlatformDailySQL + ` > ` + normalizePlatformMaxMileageSQL + `)`
const upsertSourcePureHydrogenIncrementSQL = `CASE
WHEN latest_event_time IS NOT NULL
AND VALUES(latest_event_time) > latest_event_time
AND latest_pure_hydrogen_mode_known = 1
AND latest_pure_hydrogen_active = 1
AND VALUES(latest_pure_hydrogen_mode_known) = 1
AND VALUES(latest_pure_hydrogen_active) = 1
AND VALUES(latest_total_mileage_km) >= latest_total_mileage_km
AND VALUES(latest_total_mileage_km) - latest_total_mileage_km <= ` + maxSelectedDailyMileageKMSQL + `
THEN VALUES(latest_total_mileage_km) - latest_total_mileage_km
ELSE 0
END`
const upsertSourceMileageSQL = `
INSERT INTO vehicle_daily_mileage_source
(vin, stat_date, protocol, source_key, source_ip, source_endpoint, phone, device_id, platform_name,
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count,
first_event_time, latest_event_time, quality_status, quality_reason)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, pure_hydrogen_mileage_km,
sample_count, pure_hydrogen_sample_count, first_event_time, latest_event_time,
latest_pure_hydrogen_active, latest_pure_hydrogen_mode_known, quality_status, quality_reason)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
source_ip = VALUES(source_ip),
source_endpoint = VALUES(source_endpoint),
phone = VALUES(phone),
device_id = VALUES(device_id),
platform_name = COALESCE(NULLIF(TRIM(VALUES(platform_name)), ''), platform_name),
pure_hydrogen_mileage_km = pure_hydrogen_mileage_km + ` + upsertSourcePureHydrogenIncrementSQL + `,
pure_hydrogen_sample_count = pure_hydrogen_sample_count + VALUES(pure_hydrogen_sample_count),
latest_pure_hydrogen_active = CASE
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time
THEN VALUES(latest_pure_hydrogen_active)
ELSE latest_pure_hydrogen_active
END,
latest_pure_hydrogen_mode_known = CASE
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time
THEN VALUES(latest_pure_hydrogen_mode_known)
ELSE latest_pure_hydrogen_mode_known
END,
first_total_mileage_km = ` + upsertSourceMergedFirstTotalSQL + `,
latest_total_mileage_km = ` + upsertSourceMergedLatestTotalSQL + `,
daily_mileage_km = ` + upsertSourceMergedDailySQL + `,
@@ -456,8 +501,9 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
const normalizePlatformSourceMileageInsertSQL = `
INSERT INTO vehicle_daily_mileage_source
(vin, stat_date, protocol, source_key, source_ip, source_endpoint, phone, device_id, platform_name,
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count,
first_event_time, latest_event_time, quality_status, quality_reason)
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, pure_hydrogen_mileage_km,
sample_count, pure_hydrogen_sample_count, first_event_time, latest_event_time,
latest_pure_hydrogen_active, latest_pure_hydrogen_mode_known, quality_status, quality_reason)
SELECT
s.vin,
s.stat_date,
@@ -471,9 +517,13 @@ SELECT
` + normalizePlatformFirstTotalSQL + ` AS first_total_mileage_km,
` + normalizePlatformLatestTotalSQL + ` AS latest_total_mileage_km,
` + normalizePlatformDailySQL + ` AS daily_mileage_km,
SUM(s.pure_hydrogen_mileage_km) AS pure_hydrogen_mileage_km,
SUM(s.sample_count) AS sample_count,
SUM(s.pure_hydrogen_sample_count) AS pure_hydrogen_sample_count,
MIN(s.first_event_time) AS first_event_time,
MAX(s.latest_event_time) AS latest_event_time,
CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_pure_hydrogen_active ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS UNSIGNED) AS latest_pure_hydrogen_active,
CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_pure_hydrogen_mode_known ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS UNSIGNED) AS latest_pure_hydrogen_mode_known,
CASE
WHEN ` + normalizePlatformOutsideDailyRangeSQL + ` THEN '` + QualityInvalidDelta + `'
ELSE '` + QualityOK + `'
@@ -530,6 +580,16 @@ ON DUPLICATE KEY UPDATE
phone = COALESCE(NULLIF(TRIM(VALUES(phone)), ''), phone),
device_id = COALESCE(NULLIF(TRIM(VALUES(device_id)), ''), device_id),
platform_name = COALESCE(NULLIF(TRIM(VALUES(platform_name)), ''), platform_name),
pure_hydrogen_mileage_km = VALUES(pure_hydrogen_mileage_km),
pure_hydrogen_sample_count = VALUES(pure_hydrogen_sample_count),
latest_pure_hydrogen_active = CASE
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(latest_pure_hydrogen_active)
ELSE latest_pure_hydrogen_active
END,
latest_pure_hydrogen_mode_known = CASE
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(latest_pure_hydrogen_mode_known)
ELSE latest_pure_hydrogen_mode_known
END,
first_total_mileage_km = ` + upsertSourceMergedFirstTotalSQL + `,
latest_total_mileage_km = ` + upsertSourceMergedLatestTotalSQL + `,
daily_mileage_km = ` + upsertSourceMergedDailySQL + `,
@@ -673,13 +733,14 @@ const selectedSourceIdentitySampleCountSQL = `(
const projectDailyMileageSQL = `
INSERT INTO vehicle_daily_mileage
(vin, stat_date, protocol, source_id, daily_mileage_km, latest_total_mileage_km)
(vin, stat_date, protocol, source_id, daily_mileage_km, pure_hydrogen_mileage_km, latest_total_mileage_km)
SELECT
s.vin,
s.stat_date,
s.protocol,
ds.id,
s.daily_mileage_km,
LEAST(s.pure_hydrogen_mileage_km, s.daily_mileage_km),
s.latest_total_mileage_km
FROM vehicle_daily_mileage_source s
LEFT JOIN vehicle_data_source ds
@@ -688,6 +749,7 @@ WHERE s.vin = ?
AND s.stat_date = ?
AND s.protocol = ?
AND s.quality_status = '` + QualityOK + `'
AND s.latest_total_mileage_km IS NOT NULL
AND s.daily_mileage_km BETWEEN 0 AND (? * GREATEST(1, DATEDIFF(DATE(s.latest_event_time), DATE(s.first_event_time))))
AND (ds.id IS NULL OR ds.enabled = 1 OR (
COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN'
@@ -710,6 +772,7 @@ LIMIT 1
ON DUPLICATE KEY UPDATE
source_id = VALUES(source_id),
daily_mileage_km = VALUES(daily_mileage_km),
pure_hydrogen_mileage_km = VALUES(pure_hydrogen_mileage_km),
latest_total_mileage_km = VALUES(latest_total_mileage_km),
updated_at = CURRENT_TIMESTAMP
`