shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(monitor): arbitrate conflicting location sources

Browse Source
This commit is contained in:
lingniu
2026-07-16 11:05:28 +08:00
parent 9384d6acf5
commit b9fcf476ec
11 changed files with 360 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
go/vehicle-gateway/internal/realtime/snapshot_writer.go
View File

@@ -5,6 +5,8 @@ import (
"database/sql"
"encoding/json"
"errors"
"math"
"net"
"strings"
"sync"
"time"
@@ -179,6 +181,12 @@ func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error {
return err
}
}
if _, err := w.exec.ExecContext(ctx, jt808RealtimeLocationSourceTableSQL); err != nil {
return err
}
if _, err := w.exec.ExecContext(ctx, vehicleLocationSourcePolicyTableSQL); err != nil {
return err
}
for _, statement := range cleanupInvalidRealtimeTotalMileageSQL {
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
return err
@@ -230,7 +238,7 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
if !ok {
return nil
}
_, err = w.exec.ExecContext(ctx, upsertRealtimeLocationSQL,
locationArgs := []any{
location.Protocol,
location.VIN,
location.Plate,
@@ -247,10 +255,66 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
location.StatusFlag,
location.ReceivedAt,
location.EventID,
)
}
if env.Protocol != envelope.ProtocolJT808 {
_, err = w.exec.ExecContext(ctx, upsertRealtimeLocationSQL, locationArgs...)
return err
}
sourceKey := realtimeLocationSourceKey(env, vin)
sourceArgs := append([]any{sourceKey, strings.TrimSpace(env.Phone), strings.TrimSpace(env.DeviceID), strings.TrimSpace(env.SourceCode), normalizedSourceKind(env.SourceKind), strings.TrimSpace(env.SourceEndpoint)}, locationArgs...)
sourceArgs = append(sourceArgs, location.ReceivedAt)
if _, err = w.exec.ExecContext(ctx, upsertJT808RealtimeLocationSourceSQL, sourceArgs...); err != nil {
return err
}
_, err = w.exec.ExecContext(ctx, electJT808RealtimeLocationSQL, vin)
return err
}
func realtimeLocationSourceKey(env envelope.FrameEnvelope, vin string) string {
identity := strings.TrimSpace(env.Phone)
if identity == "" {
identity = strings.TrimSpace(env.DeviceID)
}
if identity == "" {
identity = strings.TrimSpace(vin)
}
scope := strings.TrimSpace(env.SourceCode)
if scope == "" {
scope = normalizedSourceKind(env.SourceKind)
}
if scope == "" || scope == "UNKNOWN" {
scope = sourceHost(env.SourceEndpoint)
}
if scope == "" {
scope = "unknown"
}
return string(env.Protocol) + ":" + identity + "@" + scope
}
func normalizedSourceKind(value string) string {
value = strings.ToUpper(strings.TrimSpace(value))
switch value {
case "DIRECT", "PLATFORM":
return value
default:
return "UNKNOWN"
}
}
func sourceHost(endpoint string) string {
endpoint = strings.TrimSpace(endpoint)
if endpoint == "" {
return ""
}
if host, _, err := net.SplitHostPort(endpoint); err == nil {
return strings.TrimSpace(host)
}
if index := strings.LastIndex(endpoint, ":"); index > 0 && strings.Count(endpoint, ":") == 1 {
return endpoint[:index]
}
return endpoint
}
func snapshotFieldsForEnvelope(env envelope.FrameEnvelope) map[string]any {
if len(env.ParsedFields) == 0 {
return nil
@@ -347,6 +411,11 @@ func realtimeLocationFromEnvelope(env envelope.FrameEnvelope, vin string, plate
if !ok {
return realtimeLocationRow{}, false
}
if math.IsNaN(location.Latitude) || math.IsNaN(location.Longitude) || math.IsInf(location.Latitude, 0) || math.IsInf(location.Longitude, 0) ||
location.Latitude < -90 || location.Latitude > 90 || location.Longitude < -180 || location.Longitude > 180 ||
(math.Abs(location.Latitude) < 0.000001 && math.Abs(location.Longitude) < 0.000001) {
return realtimeLocationRow{}, false
}
eventTimeMS, _ := envelope.NormalizedEventTimeMS(env)
totalMileageKM, hasTotalMileage := telemetry.TotalMileageKM(env.Protocol, env.ParsedFields)
if totalMileageKM <= 0 {
@@ -554,8 +623,137 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo
var realtimeLocationCompatibilitySQL = []string{
"ALTER TABLE vehicle_realtime_location ADD COLUMN total_mileage_event_time DATETIME(3) NULL AFTER total_mileage_km",
"ALTER TABLE vehicle_realtime_location ADD COLUMN source_key VARCHAR(256) NOT NULL DEFAULT '' AFTER plate",
"ALTER TABLE vehicle_realtime_location ADD COLUMN location_conflict TINYINT(1) NOT NULL DEFAULT 0 AFTER status_flag",
"ALTER TABLE vehicle_realtime_location ADD COLUMN location_conflict_distance_m DECIMAL(12,1) NULL AFTER location_conflict",
}
const jt808RealtimeLocationSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location_source (
protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL,
source_key VARCHAR(256) NOT NULL,
phone VARCHAR(32) NOT NULL DEFAULT '',
device_id VARCHAR(64) NOT NULL DEFAULT '',
source_code VARCHAR(64) NOT NULL DEFAULT '',
source_kind VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN',
source_endpoint VARCHAR(128) NOT NULL DEFAULT '',
plate VARCHAR(32) NOT NULL DEFAULT '',
event_time DATETIME(3) NULL,
latitude DECIMAL(12,6) NOT NULL,
longitude DECIMAL(12,6) NOT NULL,
speed_kmh DECIMAL(10,3) NULL,
total_mileage_km DECIMAL(18,3) NULL,
total_mileage_event_time DATETIME(3) NULL,
soc_percent DECIMAL(6,2) NULL,
altitude_m DECIMAL(10,3) NULL,
direction_deg DECIMAL(10,3) NULL,
alarm_flag BIGINT NULL,
status_flag BIGINT NULL,
received_at DATETIME(3) NULL,
event_id VARCHAR(64) NOT NULL DEFAULT '',
quality_status VARCHAR(32) NOT NULL DEFAULT 'OK',
quality_reason VARCHAR(64) NOT NULL DEFAULT '',
consecutive_good_samples INT NOT NULL DEFAULT 1,
latest_sample_received_at DATETIME(3) NULL,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (protocol, vin, source_key),
KEY idx_location_source_vehicle_fresh (vin, protocol, received_at),
KEY idx_location_source_quality (quality_status, updated_at)
)`
const vehicleLocationSourcePolicyTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_location_source_policy (
vin VARCHAR(32) NOT NULL,
protocol VARCHAR(32) NOT NULL,
source_key VARCHAR(256) NOT NULL,
enabled TINYINT(1) NOT NULL DEFAULT 1,
priority INT NOT NULL DEFAULT 100,
remark VARCHAR(255) NOT NULL DEFAULT '',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (vin, protocol, source_key),
KEY idx_location_policy_enabled (vin, protocol, enabled, priority)
)`
const jt808AcceptedLocationCondition = `VALUES(event_time) IS NOT NULL
AND (vehicle_realtime_location_source.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location_source.event_time)
AND (vehicle_realtime_location_source.event_time IS NULL
OR ST_Distance_Sphere(
POINT(vehicle_realtime_location_source.longitude, vehicle_realtime_location_source.latitude),
POINT(VALUES(longitude), VALUES(latitude))
) <= GREATEST(500, GREATEST(1, TIMESTAMPDIFF(SECOND, vehicle_realtime_location_source.event_time, VALUES(event_time))) * 70))`
var upsertJT808RealtimeLocationSourceSQL = `
INSERT INTO vehicle_realtime_location_source
(source_key, phone, device_id, source_code, source_kind, source_endpoint,
protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time,
soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, latest_sample_received_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
quality_status = IF(` + jt808AcceptedLocationCondition + `, 'OK', 'REJECTED'),
quality_reason = IF(quality_status = 'OK', '', 'impossible_jump'),
consecutive_good_samples = IF(quality_status = 'OK', IF(VALUES(event_id) <> event_id, consecutive_good_samples + 1, consecutive_good_samples), 0),
latest_sample_received_at = VALUES(latest_sample_received_at),
phone = IF(VALUES(phone) <> '', VALUES(phone), phone),
device_id = IF(VALUES(device_id) <> '', VALUES(device_id), device_id),
source_code = IF(VALUES(source_code) <> '', VALUES(source_code), source_code),
source_kind = IF(VALUES(source_kind) <> 'UNKNOWN', VALUES(source_kind), source_kind),
source_endpoint = IF(VALUES(source_endpoint) <> '', VALUES(source_endpoint), source_endpoint),
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
event_time = IF(quality_status = 'OK', VALUES(event_time), event_time),
latitude = IF(quality_status = 'OK', VALUES(latitude), latitude),
longitude = IF(quality_status = 'OK', VALUES(longitude), longitude),
speed_kmh = IF(quality_status = 'OK', COALESCE(VALUES(speed_kmh), speed_kmh), speed_kmh),
total_mileage_km = IF(quality_status = 'OK', COALESCE(VALUES(total_mileage_km), total_mileage_km), total_mileage_km),
total_mileage_event_time = IF(quality_status = 'OK' AND VALUES(total_mileage_km) IS NOT NULL, COALESCE(VALUES(total_mileage_event_time), total_mileage_event_time), total_mileage_event_time),
soc_percent = IF(quality_status = 'OK', COALESCE(VALUES(soc_percent), soc_percent), soc_percent),
altitude_m = IF(quality_status = 'OK', COALESCE(VALUES(altitude_m), altitude_m), altitude_m),
direction_deg = IF(quality_status = 'OK', COALESCE(VALUES(direction_deg), direction_deg), direction_deg),
alarm_flag = IF(quality_status = 'OK', COALESCE(VALUES(alarm_flag), alarm_flag), alarm_flag),
status_flag = IF(quality_status = 'OK', COALESCE(VALUES(status_flag), status_flag), status_flag),
received_at = IF(quality_status = 'OK', VALUES(received_at), received_at),
event_id = IF(quality_status = 'OK', VALUES(event_id), event_id)
`
const electJT808RealtimeLocationSQL = `
INSERT INTO vehicle_realtime_location
(protocol, vin, plate, source_key, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time,
soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, location_conflict, location_conflict_distance_m,
received_at, event_id)
SELECT s.protocol, s.vin, s.plate, s.source_key, s.event_time, s.latitude, s.longitude, s.speed_kmh,
s.total_mileage_km, s.total_mileage_event_time, s.soc_percent, s.altitude_m, s.direction_deg,
s.alarm_flag, s.status_flag,
EXISTS(SELECT 1 FROM vehicle_realtime_location_source other
WHERE other.vin = s.vin AND other.protocol = s.protocol AND other.source_key <> s.source_key
AND other.quality_status = 'OK' AND other.received_at >= DATE_SUB(NOW(), INTERVAL 2 MINUTE)
AND ST_Distance_Sphere(POINT(s.longitude, s.latitude), POINT(other.longitude, other.latitude)) > 200),
(SELECT MAX(ST_Distance_Sphere(POINT(s.longitude, s.latitude), POINT(other.longitude, other.latitude)))
FROM vehicle_realtime_location_source other
WHERE other.vin = s.vin AND other.protocol = s.protocol AND other.source_key <> s.source_key
AND other.quality_status = 'OK' AND other.received_at >= DATE_SUB(NOW(), INTERVAL 2 MINUTE)),
s.received_at, s.event_id
FROM vehicle_realtime_location_source s
LEFT JOIN vehicle_location_source_policy policy
ON policy.vin = s.vin AND policy.protocol = s.protocol AND policy.source_key = s.source_key
LEFT JOIN vehicle_realtime_location cur
ON cur.vin = s.vin AND cur.protocol = s.protocol
WHERE s.vin = ? AND s.protocol = 'JT808' AND s.quality_status = 'OK' AND COALESCE(policy.enabled, 1) = 1
ORDER BY
CASE WHEN s.received_at >= DATE_SUB(NOW(), INTERVAL 2 MINUTE) THEN 0 ELSE 1 END ASC,
(COALESCE(policy.priority, CASE s.source_kind WHEN 'DIRECT' THEN 20 WHEN 'PLATFORM' THEN 30 ELSE 40 END)
+ CASE WHEN cur.source_key = s.source_key AND cur.received_at >= DATE_SUB(NOW(), INTERVAL 2 MINUTE) THEN -5 ELSE 0 END
+ CASE WHEN cur.source_key <> '' AND cur.source_key <> s.source_key AND s.consecutive_good_samples < 3 THEN 1000 ELSE 0 END) ASC,
s.received_at DESC, s.source_key ASC
LIMIT 1
ON DUPLICATE KEY UPDATE
plate = IF(VALUES(plate) <> '', VALUES(plate), vehicle_realtime_location.plate), source_key = VALUES(source_key),
event_time = VALUES(event_time), latitude = VALUES(latitude), longitude = VALUES(longitude),
speed_kmh = VALUES(speed_kmh), total_mileage_km = VALUES(total_mileage_km),
total_mileage_event_time = VALUES(total_mileage_event_time), soc_percent = VALUES(soc_percent),
altitude_m = VALUES(altitude_m), direction_deg = VALUES(direction_deg), alarm_flag = VALUES(alarm_flag),
status_flag = VALUES(status_flag), location_conflict = VALUES(location_conflict),
location_conflict_distance_m = VALUES(location_conflict_distance_m), received_at = VALUES(received_at),
event_id = VALUES(event_id), updated_at = CURRENT_TIMESTAMP
`
var cleanupInvalidRealtimeTotalMileageSQL = []string{
`UPDATE vehicle_realtime_location
SET total_mileage_km = NULL,