478 lines
13 KiB
Go
478 lines
13 KiB
Go
package realtime
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
|
)
|
|
|
|
type SnapshotExecer interface {
|
|
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
|
}
|
|
|
|
type PlateResolver interface {
|
|
PlateByVIN(context.Context, string) (string, error)
|
|
}
|
|
|
|
type SnapshotWriter struct {
|
|
exec SnapshotExecer
|
|
plateResolver PlateResolver
|
|
}
|
|
|
|
func NewSnapshotWriter(exec SnapshotExecer) *SnapshotWriter {
|
|
return NewSnapshotWriterWithPlateResolver(exec, nil)
|
|
}
|
|
|
|
func NewSnapshotWriterWithPlateResolver(exec SnapshotExecer, plateResolver PlateResolver) *SnapshotWriter {
|
|
if exec == nil {
|
|
panic("snapshot execer must not be nil")
|
|
}
|
|
return &SnapshotWriter{exec: exec, plateResolver: plateResolver}
|
|
}
|
|
|
|
func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error {
|
|
if _, err := w.exec.ExecContext(ctx, realtimeSnapshotTableSQL); err != nil {
|
|
return err
|
|
}
|
|
if _, err := w.exec.ExecContext(ctx, realtimeLocationTableSQL); err != nil {
|
|
return err
|
|
}
|
|
return w.dropObsoleteColumns(ctx)
|
|
}
|
|
|
|
func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope) error {
|
|
vin := strings.TrimSpace(env.VIN)
|
|
if vin == "" {
|
|
return nil
|
|
}
|
|
if !hasRealtimePayload(env) {
|
|
return nil
|
|
}
|
|
plate, err := w.plateForEnvelope(ctx, env)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
eventTime := nullableTime(env.EventTimeMS)
|
|
receivedAt := nullableTime(env.ReceivedAtMS)
|
|
if _, err = w.exec.ExecContext(ctx, upsertRealtimeSnapshotSQL,
|
|
string(env.Protocol),
|
|
vin,
|
|
plate,
|
|
strings.TrimSpace(env.MessageID),
|
|
env.Sequence,
|
|
strings.TrimSpace(env.SourceEndpoint),
|
|
eventTime,
|
|
receivedAt,
|
|
env.StableEventID(),
|
|
); err != nil {
|
|
return err
|
|
}
|
|
location, ok := realtimeLocationFromEnvelope(env, vin, plate)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
_, err = w.exec.ExecContext(ctx, upsertRealtimeLocationSQL,
|
|
location.Protocol,
|
|
location.VIN,
|
|
location.Plate,
|
|
location.MessageID,
|
|
location.Sequence,
|
|
location.SourceEndpoint,
|
|
location.EventTime,
|
|
location.Latitude,
|
|
location.Longitude,
|
|
location.SpeedKMH,
|
|
location.TotalMileageKM,
|
|
location.SOCPercent,
|
|
location.AltitudeM,
|
|
location.DirectionDeg,
|
|
location.AlarmFlag,
|
|
location.StatusFlag,
|
|
location.ReceivedAt,
|
|
location.EventID,
|
|
)
|
|
return err
|
|
}
|
|
|
|
func (w *SnapshotWriter) plateForEnvelope(ctx context.Context, env envelope.FrameEnvelope) (string, error) {
|
|
if plate := strings.TrimSpace(env.Plate); plate != "" {
|
|
return plate, nil
|
|
}
|
|
if w.plateResolver == nil {
|
|
return "", nil
|
|
}
|
|
vin := strings.TrimSpace(env.VIN)
|
|
if vin == "" {
|
|
return "", nil
|
|
}
|
|
plate, err := w.plateResolver.PlateByVIN(ctx, vin)
|
|
if err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return "", nil
|
|
}
|
|
return "", err
|
|
}
|
|
return strings.TrimSpace(plate), nil
|
|
}
|
|
|
|
func (w *SnapshotWriter) dropObsoleteColumns(ctx context.Context) error {
|
|
queryer, ok := w.exec.(Queryer)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
if err := w.ensureRealtimeVINIndexes(ctx, queryer); err != nil {
|
|
return err
|
|
}
|
|
for _, column := range []struct {
|
|
table string
|
|
name string
|
|
}{
|
|
{table: "vehicle_realtime_snapshot", name: "parsed_json"},
|
|
{table: "vehicle_realtime_snapshot", name: "fields_json"},
|
|
{table: "vehicle_realtime_snapshot", name: "vehicle_key"},
|
|
{table: "vehicle_realtime_snapshot", name: "phone"},
|
|
{table: "vehicle_realtime_snapshot", name: "device_id"},
|
|
{table: "vehicle_realtime_location", name: "fields_json"},
|
|
{table: "vehicle_realtime_location", name: "vehicle_key"},
|
|
{table: "vehicle_realtime_location", name: "phone"},
|
|
{table: "vehicle_realtime_location", name: "device_id"},
|
|
} {
|
|
exists, err := columnExists(ctx, queryer, column.table, column.name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !exists {
|
|
continue
|
|
}
|
|
if _, err := w.exec.ExecContext(ctx, "ALTER TABLE "+column.table+" DROP COLUMN "+column.name); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (w *SnapshotWriter) ensureRealtimeVINIndexes(ctx context.Context, queryer Queryer) error {
|
|
for _, item := range []struct {
|
|
table string
|
|
oldIndex string
|
|
newIndex string
|
|
}{
|
|
{table: "vehicle_realtime_snapshot", oldIndex: "uk_realtime_snapshot_vehicle", newIndex: "uk_realtime_snapshot_vin"},
|
|
{table: "vehicle_realtime_location", oldIndex: "uk_realtime_location_vehicle", newIndex: "uk_realtime_location_vin"},
|
|
} {
|
|
if err := w.normalizeVINRows(ctx, item.table); err != nil {
|
|
return err
|
|
}
|
|
exists, err := indexExists(ctx, queryer, item.table, item.newIndex)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !exists {
|
|
if _, err := w.exec.ExecContext(ctx, "ALTER TABLE "+item.table+" ADD UNIQUE KEY "+item.newIndex+" (protocol, vin)"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
exists, err = indexExists(ctx, queryer, item.table, item.oldIndex)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if exists {
|
|
if _, err := w.exec.ExecContext(ctx, "ALTER TABLE "+item.table+" DROP INDEX "+item.oldIndex); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (w *SnapshotWriter) normalizeVINRows(ctx context.Context, table string) error {
|
|
if _, err := w.exec.ExecContext(ctx, "DELETE FROM "+table+" WHERE vin IS NULL OR vin = ''"); err != nil {
|
|
return err
|
|
}
|
|
_, err := w.exec.ExecContext(ctx, `DELETE newer_duplicate
|
|
FROM `+table+` newer_duplicate
|
|
JOIN `+table+` latest
|
|
ON newer_duplicate.protocol = latest.protocol
|
|
AND newer_duplicate.vin = latest.vin
|
|
AND (
|
|
newer_duplicate.updated_at < latest.updated_at
|
|
OR (newer_duplicate.updated_at = latest.updated_at AND newer_duplicate.id < latest.id)
|
|
)`)
|
|
return err
|
|
}
|
|
|
|
func columnExists(ctx context.Context, queryer Queryer, table string, column string) (bool, error) {
|
|
var count int
|
|
err := queryer.QueryRowContext(ctx, `
|
|
SELECT COUNT(*)
|
|
FROM information_schema.columns
|
|
WHERE table_schema = DATABASE()
|
|
AND table_name = ?
|
|
AND column_name = ?`, table, column).Scan(&count)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return count > 0, nil
|
|
}
|
|
|
|
func indexExists(ctx context.Context, queryer Queryer, table string, index string) (bool, error) {
|
|
var count int
|
|
err := queryer.QueryRowContext(ctx, `
|
|
SELECT COUNT(*)
|
|
FROM information_schema.statistics
|
|
WHERE table_schema = DATABASE()
|
|
AND table_name = ?
|
|
AND index_name = ?`, table, index).Scan(&count)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return count > 0, nil
|
|
}
|
|
|
|
type realtimeLocationRow struct {
|
|
Protocol string
|
|
VIN string
|
|
Plate string
|
|
MessageID string
|
|
Sequence uint16
|
|
SourceEndpoint string
|
|
EventTime any
|
|
Latitude float64
|
|
Longitude float64
|
|
SpeedKMH any
|
|
TotalMileageKM any
|
|
SOCPercent any
|
|
AltitudeM any
|
|
DirectionDeg any
|
|
AlarmFlag any
|
|
StatusFlag any
|
|
ReceivedAt any
|
|
EventID string
|
|
}
|
|
|
|
func realtimeLocationFromEnvelope(env envelope.FrameEnvelope, vin string, plate string) (realtimeLocationRow, bool) {
|
|
latitude, okLat := numberField(env.Fields, envelope.FieldLatitude)
|
|
longitude, okLon := numberField(env.Fields, envelope.FieldLongitude)
|
|
if !okLat || !okLon {
|
|
return realtimeLocationRow{}, false
|
|
}
|
|
return realtimeLocationRow{
|
|
Protocol: string(env.Protocol),
|
|
VIN: vin,
|
|
Plate: plate,
|
|
MessageID: strings.TrimSpace(env.MessageID),
|
|
Sequence: env.Sequence,
|
|
SourceEndpoint: strings.TrimSpace(env.SourceEndpoint),
|
|
EventTime: nullableTime(env.EventTimeMS),
|
|
Latitude: latitude,
|
|
Longitude: longitude,
|
|
SpeedKMH: nullableNumberField(env.Fields, envelope.FieldSpeedKMH),
|
|
TotalMileageKM: nullableNumberField(env.Fields, envelope.FieldTotalMileageKM),
|
|
SOCPercent: nullableNumberField(env.Fields, envelope.FieldSOCPercent),
|
|
AltitudeM: nullableNumberField(env.Fields, "altitude_m"),
|
|
DirectionDeg: nullableNumberField(env.Fields, "direction_deg"),
|
|
AlarmFlag: nullableNumberField(env.Fields, "alarm_flag"),
|
|
StatusFlag: nullableNumberField(env.Fields, "status_flag"),
|
|
ReceivedAt: nullableTime(env.ReceivedAtMS),
|
|
EventID: env.StableEventID(),
|
|
}, true
|
|
}
|
|
|
|
func nullableNumberField(fields map[string]any, key string) any {
|
|
value, ok := numberField(fields, key)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
func numberField(fields map[string]any, key string) (float64, bool) {
|
|
value, ok := fields[key]
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
switch typed := value.(type) {
|
|
case float64:
|
|
return typed, true
|
|
case float32:
|
|
return float64(typed), true
|
|
case int:
|
|
return float64(typed), true
|
|
case int8:
|
|
return float64(typed), true
|
|
case int16:
|
|
return float64(typed), true
|
|
case int32:
|
|
return float64(typed), true
|
|
case int64:
|
|
return float64(typed), true
|
|
case uint:
|
|
return float64(typed), true
|
|
case uint8:
|
|
return float64(typed), true
|
|
case uint16:
|
|
return float64(typed), true
|
|
case uint32:
|
|
return float64(typed), true
|
|
case uint64:
|
|
return float64(typed), true
|
|
default:
|
|
return 0, false
|
|
}
|
|
}
|
|
|
|
type BindingPlateResolver struct {
|
|
queryer Queryer
|
|
table string
|
|
}
|
|
|
|
type Queryer interface {
|
|
QueryRowContext(context.Context, string, ...any) *sql.Row
|
|
}
|
|
|
|
func NewBindingPlateResolver(queryer Queryer, table string) *BindingPlateResolver {
|
|
if queryer == nil {
|
|
panic("plate binding queryer must not be nil")
|
|
}
|
|
table = strings.TrimSpace(table)
|
|
if table == "" || !safeIdentifier(table) {
|
|
table = "vehicle_identity_binding"
|
|
}
|
|
return &BindingPlateResolver{queryer: queryer, table: table}
|
|
}
|
|
|
|
func (r *BindingPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) {
|
|
vin = strings.TrimSpace(vin)
|
|
if vin == "" {
|
|
return "", sql.ErrNoRows
|
|
}
|
|
query := "SELECT plate FROM " + r.table + " WHERE vin = ? AND plate IS NOT NULL AND plate <> '' ORDER BY updated_at DESC LIMIT 1"
|
|
var plate string
|
|
err := r.queryer.QueryRowContext(ctx, query, vin).Scan(&plate)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return strings.TrimSpace(plate), nil
|
|
}
|
|
|
|
func safeIdentifier(value string) bool {
|
|
if value == "" {
|
|
return false
|
|
}
|
|
for _, r := range value {
|
|
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
|
continue
|
|
}
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func nullableTime(ms int64) any {
|
|
if ms <= 0 {
|
|
return nil
|
|
}
|
|
return time.UnixMilli(ms)
|
|
}
|
|
|
|
func hasRealtimePayload(env envelope.FrameEnvelope) bool {
|
|
if len(env.Fields) > 0 {
|
|
return true
|
|
}
|
|
if _, ok := env.Parsed["data_units"]; ok {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot (
|
|
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
|
protocol VARCHAR(32) NOT NULL,
|
|
vin VARCHAR(32) NOT NULL DEFAULT '',
|
|
plate VARCHAR(32) NOT NULL DEFAULT '',
|
|
message_id VARCHAR(32) NOT NULL DEFAULT '',
|
|
sequence_id INT NOT NULL DEFAULT 0,
|
|
source_endpoint VARCHAR(128) NOT NULL DEFAULT '',
|
|
event_time DATETIME(3) NULL,
|
|
received_at DATETIME(3) NULL,
|
|
event_id VARCHAR(64) NOT NULL DEFAULT '',
|
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
UNIQUE KEY uk_realtime_snapshot_vin (protocol, vin),
|
|
KEY idx_vin (vin),
|
|
KEY idx_protocol_updated (protocol, updated_at)
|
|
)`
|
|
|
|
const upsertRealtimeSnapshotSQL = `
|
|
INSERT INTO vehicle_realtime_snapshot
|
|
(protocol, vin, plate, message_id, sequence_id, source_endpoint, event_time, received_at, event_id)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON DUPLICATE KEY UPDATE
|
|
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
|
|
message_id = VALUES(message_id),
|
|
sequence_id = VALUES(sequence_id),
|
|
source_endpoint = VALUES(source_endpoint),
|
|
event_time = VALUES(event_time),
|
|
received_at = VALUES(received_at),
|
|
event_id = VALUES(event_id),
|
|
updated_at = CURRENT_TIMESTAMP
|
|
`
|
|
|
|
const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location (
|
|
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
|
protocol VARCHAR(32) NOT NULL,
|
|
vin VARCHAR(32) NOT NULL DEFAULT '',
|
|
plate VARCHAR(32) NOT NULL DEFAULT '',
|
|
message_id VARCHAR(32) NOT NULL DEFAULT '',
|
|
sequence_id INT NOT NULL DEFAULT 0,
|
|
source_endpoint VARCHAR(128) NOT NULL DEFAULT '',
|
|
event_time DATETIME(3) NULL,
|
|
latitude DECIMAL(12,6) NOT NULL,
|
|
longitude DECIMAL(12,6) NOT NULL,
|
|
speed_kmh DECIMAL(10,3) NULL,
|
|
total_mileage_km DECIMAL(18,3) NULL,
|
|
soc_percent DECIMAL(6,2) NULL,
|
|
altitude_m DECIMAL(10,3) NULL,
|
|
direction_deg DECIMAL(10,3) NULL,
|
|
alarm_flag BIGINT NULL,
|
|
status_flag BIGINT NULL,
|
|
received_at DATETIME(3) NULL,
|
|
event_id VARCHAR(64) NOT NULL DEFAULT '',
|
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
UNIQUE KEY uk_realtime_location_vin (protocol, vin),
|
|
KEY idx_vin (vin),
|
|
KEY idx_protocol_updated (protocol, updated_at),
|
|
KEY idx_location (longitude, latitude)
|
|
)`
|
|
|
|
const upsertRealtimeLocationSQL = `
|
|
INSERT INTO vehicle_realtime_location
|
|
(protocol, vin, plate, message_id, sequence_id, source_endpoint, event_time,
|
|
latitude, longitude, speed_kmh, total_mileage_km,
|
|
soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON DUPLICATE KEY UPDATE
|
|
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
|
|
message_id = VALUES(message_id),
|
|
sequence_id = VALUES(sequence_id),
|
|
source_endpoint = VALUES(source_endpoint),
|
|
event_time = VALUES(event_time),
|
|
latitude = VALUES(latitude),
|
|
longitude = VALUES(longitude),
|
|
speed_kmh = VALUES(speed_kmh),
|
|
total_mileage_km = VALUES(total_mileage_km),
|
|
soc_percent = VALUES(soc_percent),
|
|
altitude_m = VALUES(altitude_m),
|
|
direction_deg = VALUES(direction_deg),
|
|
alarm_flag = VALUES(alarm_flag),
|
|
status_flag = VALUES(status_flag),
|
|
received_at = VALUES(received_at),
|
|
event_id = VALUES(event_id),
|
|
updated_at = CURRENT_TIMESTAMP
|
|
`
|