package realtime import ( "context" "database/sql" "errors" "strings" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" ) type SnapshotExecer interface { ExecContext(context.Context, string, ...any) (sql.Result, error) } type PlateResolver interface { PlateByVIN(context.Context, string) (string, error) } type SnapshotWriter struct { exec SnapshotExecer plateResolver PlateResolver } func NewSnapshotWriter(exec SnapshotExecer) *SnapshotWriter { return NewSnapshotWriterWithPlateResolver(exec, nil) } func NewSnapshotWriterWithPlateResolver(exec SnapshotExecer, plateResolver PlateResolver) *SnapshotWriter { if exec == nil { panic("snapshot execer must not be nil") } return &SnapshotWriter{exec: exec, plateResolver: plateResolver} } func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error { if _, err := w.exec.ExecContext(ctx, realtimeSnapshotTableSQL); err != nil { return err } if _, err := w.exec.ExecContext(ctx, realtimeLocationTableSQL); err != nil { return err } return w.dropObsoleteColumns(ctx) } func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope) error { vin := strings.TrimSpace(env.VIN) if vin == "" { return nil } if !hasRealtimePayload(env) { return nil } plate, err := w.plateForEnvelope(ctx, env) if err != nil { return err } eventTime := nullableTime(env.EventTimeMS) receivedAt := nullableTime(env.ReceivedAtMS) if _, err = w.exec.ExecContext(ctx, upsertRealtimeSnapshotSQL, string(env.Protocol), vin, plate, strings.TrimSpace(env.MessageID), env.Sequence, strings.TrimSpace(env.SourceEndpoint), eventTime, receivedAt, env.StableEventID(), ); err != nil { return err } location, ok := realtimeLocationFromEnvelope(env, vin, plate) if !ok { return nil } _, err = w.exec.ExecContext(ctx, upsertRealtimeLocationSQL, location.Protocol, location.VIN, location.Plate, location.MessageID, location.Sequence, location.SourceEndpoint, location.EventTime, location.Latitude, location.Longitude, location.SpeedKMH, location.TotalMileageKM, location.SOCPercent, location.AltitudeM, location.DirectionDeg, location.AlarmFlag, location.StatusFlag, location.ReceivedAt, location.EventID, ) return err } func (w *SnapshotWriter) plateForEnvelope(ctx context.Context, env envelope.FrameEnvelope) (string, error) { if plate := strings.TrimSpace(env.Plate); plate != "" { return plate, nil } if w.plateResolver == nil { return "", nil } vin := strings.TrimSpace(env.VIN) if vin == "" { return "", nil } plate, err := w.plateResolver.PlateByVIN(ctx, vin) if err != nil { if errors.Is(err, sql.ErrNoRows) { return "", nil } return "", err } return strings.TrimSpace(plate), nil } func (w *SnapshotWriter) dropObsoleteColumns(ctx context.Context) error { queryer, ok := w.exec.(Queryer) if !ok { return nil } if err := w.ensureRealtimeVINIndexes(ctx, queryer); err != nil { return err } for _, column := range []struct { table string name string }{ {table: "vehicle_realtime_snapshot", name: "parsed_json"}, {table: "vehicle_realtime_snapshot", name: "fields_json"}, {table: "vehicle_realtime_snapshot", name: "vehicle_key"}, {table: "vehicle_realtime_snapshot", name: "phone"}, {table: "vehicle_realtime_snapshot", name: "device_id"}, {table: "vehicle_realtime_location", name: "fields_json"}, {table: "vehicle_realtime_location", name: "vehicle_key"}, {table: "vehicle_realtime_location", name: "phone"}, {table: "vehicle_realtime_location", name: "device_id"}, } { exists, err := columnExists(ctx, queryer, column.table, column.name) if err != nil { return err } if !exists { continue } if _, err := w.exec.ExecContext(ctx, "ALTER TABLE "+column.table+" DROP COLUMN "+column.name); err != nil { return err } } return nil } func (w *SnapshotWriter) ensureRealtimeVINIndexes(ctx context.Context, queryer Queryer) error { for _, item := range []struct { table string oldIndex string newIndex string }{ {table: "vehicle_realtime_snapshot", oldIndex: "uk_realtime_snapshot_vehicle", newIndex: "uk_realtime_snapshot_vin"}, {table: "vehicle_realtime_location", oldIndex: "uk_realtime_location_vehicle", newIndex: "uk_realtime_location_vin"}, } { if err := w.normalizeVINRows(ctx, item.table); err != nil { return err } exists, err := indexExists(ctx, queryer, item.table, item.newIndex) if err != nil { return err } if !exists { if _, err := w.exec.ExecContext(ctx, "ALTER TABLE "+item.table+" ADD UNIQUE KEY "+item.newIndex+" (protocol, vin)"); err != nil { return err } } exists, err = indexExists(ctx, queryer, item.table, item.oldIndex) if err != nil { return err } if exists { if _, err := w.exec.ExecContext(ctx, "ALTER TABLE "+item.table+" DROP INDEX "+item.oldIndex); err != nil { return err } } } return nil } func (w *SnapshotWriter) normalizeVINRows(ctx context.Context, table string) error { if _, err := w.exec.ExecContext(ctx, "DELETE FROM "+table+" WHERE vin IS NULL OR vin = ''"); err != nil { return err } _, err := w.exec.ExecContext(ctx, `DELETE newer_duplicate FROM `+table+` newer_duplicate JOIN `+table+` latest ON newer_duplicate.protocol = latest.protocol AND newer_duplicate.vin = latest.vin AND ( newer_duplicate.updated_at < latest.updated_at OR (newer_duplicate.updated_at = latest.updated_at AND newer_duplicate.id < latest.id) )`) return err } func columnExists(ctx context.Context, queryer Queryer, table string, column string) (bool, error) { var count int err := queryer.QueryRowContext(ctx, ` SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ?`, table, column).Scan(&count) if err != nil { return false, err } return count > 0, nil } func indexExists(ctx context.Context, queryer Queryer, table string, index string) (bool, error) { var count int err := queryer.QueryRowContext(ctx, ` SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? AND index_name = ?`, table, index).Scan(&count) if err != nil { return false, err } return count > 0, nil } type realtimeLocationRow struct { Protocol string VIN string Plate string MessageID string Sequence uint16 SourceEndpoint string EventTime any Latitude float64 Longitude float64 SpeedKMH any TotalMileageKM any SOCPercent any AltitudeM any DirectionDeg any AlarmFlag any StatusFlag any ReceivedAt any EventID string } func realtimeLocationFromEnvelope(env envelope.FrameEnvelope, vin string, plate string) (realtimeLocationRow, bool) { latitude, okLat := numberField(env.Fields, envelope.FieldLatitude) longitude, okLon := numberField(env.Fields, envelope.FieldLongitude) if !okLat || !okLon { return realtimeLocationRow{}, false } return realtimeLocationRow{ Protocol: string(env.Protocol), VIN: vin, Plate: plate, MessageID: strings.TrimSpace(env.MessageID), Sequence: env.Sequence, SourceEndpoint: strings.TrimSpace(env.SourceEndpoint), EventTime: nullableTime(env.EventTimeMS), Latitude: latitude, Longitude: longitude, SpeedKMH: nullableNumberField(env.Fields, envelope.FieldSpeedKMH), TotalMileageKM: nullableNumberField(env.Fields, envelope.FieldTotalMileageKM), SOCPercent: nullableNumberField(env.Fields, envelope.FieldSOCPercent), AltitudeM: nullableNumberField(env.Fields, "altitude_m"), DirectionDeg: nullableNumberField(env.Fields, "direction_deg"), AlarmFlag: nullableNumberField(env.Fields, "alarm_flag"), StatusFlag: nullableNumberField(env.Fields, "status_flag"), ReceivedAt: nullableTime(env.ReceivedAtMS), EventID: env.StableEventID(), }, true } func nullableNumberField(fields map[string]any, key string) any { value, ok := numberField(fields, key) if !ok { return nil } return value } func numberField(fields map[string]any, key string) (float64, bool) { value, ok := fields[key] if !ok { return 0, false } switch typed := value.(type) { case float64: return typed, true case float32: return float64(typed), true case int: return float64(typed), true case int8: return float64(typed), true case int16: return float64(typed), true case int32: return float64(typed), true case int64: return float64(typed), true case uint: return float64(typed), true case uint8: return float64(typed), true case uint16: return float64(typed), true case uint32: return float64(typed), true case uint64: return float64(typed), true default: return 0, false } } type BindingPlateResolver struct { queryer Queryer table string } type Queryer interface { QueryRowContext(context.Context, string, ...any) *sql.Row } func NewBindingPlateResolver(queryer Queryer, table string) *BindingPlateResolver { if queryer == nil { panic("plate binding queryer must not be nil") } table = strings.TrimSpace(table) if table == "" || !safeIdentifier(table) { table = "vehicle_identity_binding" } return &BindingPlateResolver{queryer: queryer, table: table} } func (r *BindingPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) { vin = strings.TrimSpace(vin) if vin == "" { return "", sql.ErrNoRows } query := "SELECT plate FROM " + r.table + " WHERE vin = ? AND plate IS NOT NULL AND plate <> '' ORDER BY updated_at DESC LIMIT 1" var plate string err := r.queryer.QueryRowContext(ctx, query, vin).Scan(&plate) if err != nil { return "", err } return strings.TrimSpace(plate), nil } func safeIdentifier(value string) bool { if value == "" { return false } for _, r := range value { if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' { continue } return false } return true } func nullableTime(ms int64) any { if ms <= 0 { return nil } return time.UnixMilli(ms) } func hasRealtimePayload(env envelope.FrameEnvelope) bool { if len(env.Fields) > 0 { return true } if _, ok := env.Parsed["data_units"]; ok { return true } return false } const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot ( id BIGINT PRIMARY KEY AUTO_INCREMENT, protocol VARCHAR(32) NOT NULL, vin VARCHAR(32) NOT NULL DEFAULT '', plate VARCHAR(32) NOT NULL DEFAULT '', message_id VARCHAR(32) NOT NULL DEFAULT '', sequence_id INT NOT NULL DEFAULT 0, source_endpoint VARCHAR(128) NOT NULL DEFAULT '', event_time DATETIME(3) NULL, received_at DATETIME(3) NULL, event_id VARCHAR(64) NOT NULL DEFAULT '', created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, UNIQUE KEY uk_realtime_snapshot_vin (protocol, vin), KEY idx_vin (vin), KEY idx_protocol_updated (protocol, updated_at) )` const upsertRealtimeSnapshotSQL = ` INSERT INTO vehicle_realtime_snapshot (protocol, vin, plate, message_id, sequence_id, source_endpoint, event_time, received_at, event_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE plate = IF(VALUES(plate) <> '', VALUES(plate), plate), message_id = VALUES(message_id), sequence_id = VALUES(sequence_id), source_endpoint = VALUES(source_endpoint), event_time = VALUES(event_time), received_at = VALUES(received_at), event_id = VALUES(event_id), updated_at = CURRENT_TIMESTAMP ` const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location ( id BIGINT PRIMARY KEY AUTO_INCREMENT, protocol VARCHAR(32) NOT NULL, vin VARCHAR(32) NOT NULL DEFAULT '', plate VARCHAR(32) NOT NULL DEFAULT '', message_id VARCHAR(32) NOT NULL DEFAULT '', sequence_id INT NOT NULL DEFAULT 0, source_endpoint VARCHAR(128) NOT NULL DEFAULT '', event_time DATETIME(3) NULL, latitude DECIMAL(12,6) NOT NULL, longitude DECIMAL(12,6) NOT NULL, speed_kmh DECIMAL(10,3) NULL, total_mileage_km DECIMAL(18,3) NULL, soc_percent DECIMAL(6,2) NULL, altitude_m DECIMAL(10,3) NULL, direction_deg DECIMAL(10,3) NULL, alarm_flag BIGINT NULL, status_flag BIGINT NULL, received_at DATETIME(3) NULL, event_id VARCHAR(64) NOT NULL DEFAULT '', created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, UNIQUE KEY uk_realtime_location_vin (protocol, vin), KEY idx_vin (vin), KEY idx_protocol_updated (protocol, updated_at), KEY idx_location (longitude, latitude) )` const upsertRealtimeLocationSQL = ` INSERT INTO vehicle_realtime_location (protocol, vin, plate, message_id, sequence_id, source_endpoint, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE plate = IF(VALUES(plate) <> '', VALUES(plate), plate), message_id = VALUES(message_id), sequence_id = VALUES(sequence_id), source_endpoint = VALUES(source_endpoint), event_time = VALUES(event_time), latitude = VALUES(latitude), longitude = VALUES(longitude), speed_kmh = VALUES(speed_kmh), total_mileage_km = VALUES(total_mileage_km), soc_percent = VALUES(soc_percent), altitude_m = VALUES(altitude_m), direction_deg = VALUES(direction_deg), alarm_flag = VALUES(alarm_flag), status_flag = VALUES(status_flag), received_at = VALUES(received_at), event_id = VALUES(event_id), updated_at = CURRENT_TIMESTAMP `