Files
lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime/snapshot_writer.go

585 lines
19 KiB
Go

package realtime
import (
"context"
"database/sql"
"encoding/json"
"errors"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
type SnapshotExecer interface {
ExecContext(context.Context, string, ...any) (sql.Result, error)
}
type PlateResolver interface {
PlateByVIN(context.Context, string) (string, error)
}
type CachedPlateResolver struct {
delegate PlateResolver
ttl time.Duration
now func() time.Time
mu sync.Mutex
entries map[string]cachedPlateEntry
}
type cachedPlateEntry struct {
plate string
notFound bool
expiresAt time.Time
}
func NewCachedPlateResolver(delegate PlateResolver, ttl time.Duration) *CachedPlateResolver {
if delegate == nil {
panic("cached plate resolver delegate must not be nil")
}
if ttl <= 0 {
ttl = 10 * time.Minute
}
return &CachedPlateResolver{
delegate: delegate,
ttl: ttl,
now: time.Now,
entries: map[string]cachedPlateEntry{},
}
}
func (r *CachedPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) {
vin = strings.TrimSpace(vin)
if vin == "" {
return "", sql.ErrNoRows
}
now := r.now()
r.mu.Lock()
entry, ok := r.entries[vin]
if ok && now.Before(entry.expiresAt) {
r.mu.Unlock()
if entry.notFound {
return "", sql.ErrNoRows
}
return entry.plate, nil
}
r.mu.Unlock()
plate, err := r.delegate.PlateByVIN(ctx, vin)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return "", err
}
r.mu.Lock()
r.entries[vin] = cachedPlateEntry{
plate: strings.TrimSpace(plate),
notFound: errors.Is(err, sql.ErrNoRows),
expiresAt: now.Add(r.ttl),
}
r.mu.Unlock()
if err != nil {
return "", err
}
return strings.TrimSpace(plate), nil
}
type SnapshotWriter struct {
exec SnapshotExecer
plateResolver PlateResolver
}
func NewSnapshotWriter(exec SnapshotExecer) *SnapshotWriter {
return NewSnapshotWriterWithPlateResolver(exec, nil)
}
func NewSnapshotWriterWithPlateResolver(exec SnapshotExecer, plateResolver PlateResolver) *SnapshotWriter {
if exec == nil {
panic("snapshot execer must not be nil")
}
return &SnapshotWriter{exec: exec, plateResolver: plateResolver}
}
func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error {
if _, err := w.exec.ExecContext(ctx, realtimeSnapshotTableSQL); err != nil {
return err
}
for _, statement := range realtimeSnapshotCompatibilitySQL {
if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateColumnError(err) {
return err
}
}
if _, err := w.exec.ExecContext(ctx, realtimeLocationTableSQL); err != nil {
return err
}
return nil
}
func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope) error {
if !envelope.IsRealtimeTelemetryFrame(env) {
return nil
}
vin := strings.TrimSpace(env.VIN)
if vin == "" {
return nil
}
if !isRealtimeSnapshotEvent(env) {
return nil
}
plate, err := w.plateForEnvelope(ctx, env)
if err != nil {
return err
}
eventTime := nullableTime(env.EventTimeMS)
receivedAt := nullableTime(env.ReceivedAtMS)
platformName := platformNameFromEnvelope(env)
parsed, err := w.snapshotFieldsForEnvelope(ctx, env, vin)
if err != nil {
return err
}
if _, err = w.exec.ExecContext(ctx, upsertRealtimeSnapshotSQL,
string(env.Protocol),
vin,
plate,
platformName,
env.SourceEndpoint,
marshalParsedJSON(parsed),
eventTime,
receivedAt,
env.StableEventID(),
); err != nil {
return err
}
location, ok := realtimeLocationFromEnvelope(env, vin, plate)
if !ok {
return nil
}
_, err = w.exec.ExecContext(ctx, upsertRealtimeLocationSQL,
location.Protocol,
location.VIN,
location.Plate,
location.EventTime,
location.Latitude,
location.Longitude,
location.SpeedKMH,
location.TotalMileageKM,
location.SOCPercent,
location.AltitudeM,
location.DirectionDeg,
location.AlarmFlag,
location.StatusFlag,
location.ReceivedAt,
location.EventID,
)
return err
}
func (w *SnapshotWriter) snapshotFieldsForEnvelope(ctx context.Context, env envelope.FrameEnvelope, vin string) (map[string]any, error) {
if len(env.Parsed) == 0 {
return nil, nil
}
parsed := cloneMap(env.Parsed)
incoming := realtimeSnapshotFlatFields(env, parsed)
if queryer, ok := w.exec.(Queryer); ok {
existing, err := realtimeSnapshotParsedJSON(ctx, queryer, env.Protocol, vin)
if err != nil {
return nil, err
}
if len(existing) > 0 {
if isStructuredSnapshotParsed(env.Protocol, existing) {
parsed = mergeParsedForProtocol(env.Protocol, existing, env.Parsed)
mergedEnv := env
mergedEnv.ParsedFields = nil
mergedEnv.ParsedFieldTypes = nil
return realtimeSnapshotFlatFields(mergedEnv, parsed), nil
}
return mergeRealtimeSnapshotFields(existing, incoming), nil
}
}
return incoming, nil
}
func realtimeSnapshotFlatFields(env envelope.FrameEnvelope, parsed map[string]any) map[string]any {
if len(env.ParsedFields) > 0 {
fields := cloneMap(env.ParsedFields)
if len(fields) > 0 {
return fields
}
}
rows := realtimeKVFields(env, parsed)
if len(rows) == 0 {
return nil
}
fields := make(map[string]any, len(rows))
for _, row := range rows {
key := realtimeKVFieldPath(row.Domain, row.Field)
if key == "" {
continue
}
fields[key] = row.Value
}
return fields
}
func mergeRealtimeSnapshotFields(existing map[string]any, incoming map[string]any) map[string]any {
if len(existing) == 0 {
return cloneMap(incoming)
}
merged := cloneMap(existing)
for key, value := range incoming {
merged[key] = value
}
return merged
}
func isStructuredSnapshotParsed(protocol envelope.Protocol, parsed map[string]any) bool {
if len(parsed) == 0 {
return false
}
if protocol == envelope.ProtocolGB32960 {
if _, ok := parsed["data_units"]; ok {
return true
}
}
mapping := realtimeMapping(protocol)
for key := range mapping.TopLevelName {
if _, ok := parsed[key]; ok {
return true
}
}
return false
}
func platformNameFromEnvelope(env envelope.FrameEnvelope) string {
if env.Fields != nil {
if value := strings.TrimSpace(stringValue(env.Fields["platform_account"])); value != "" {
return value
}
}
if env.Parsed != nil {
if value := strings.TrimSpace(stringValue(env.Parsed["platform_name"])); value != "" {
return value
}
if login, ok := env.Parsed["platform_login"].(map[string]any); ok {
return strings.TrimSpace(stringValue(login["username"]))
}
}
return ""
}
func stringValue(value any) string {
switch typed := value.(type) {
case string:
return typed
default:
return ""
}
}
func realtimeSnapshotParsedJSON(ctx context.Context, queryer Queryer, protocol envelope.Protocol, vin string) (map[string]any, error) {
var raw sql.NullString
err := queryer.QueryRowContext(ctx, selectRealtimeSnapshotParsedJSONSQL, string(protocol), vin).Scan(&raw)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return nil, err
}
if !raw.Valid || strings.TrimSpace(raw.String) == "" {
return nil, nil
}
var parsed map[string]any
if err := json.Unmarshal([]byte(raw.String), &parsed); err != nil {
return nil, nil
}
return parsed, nil
}
func marshalParsedJSON(parsed map[string]any) any {
if len(parsed) == 0 {
return nil
}
data, err := json.Marshal(parsed)
if err != nil {
return nil
}
return string(data)
}
func (w *SnapshotWriter) plateForEnvelope(ctx context.Context, env envelope.FrameEnvelope) (string, error) {
if plate := strings.TrimSpace(env.Plate); plate != "" {
return plate, nil
}
if w.plateResolver == nil {
return "", nil
}
vin := strings.TrimSpace(env.VIN)
if vin == "" {
return "", nil
}
plate, err := w.plateResolver.PlateByVIN(ctx, vin)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return "", nil
}
return "", err
}
return strings.TrimSpace(plate), nil
}
type realtimeLocationRow struct {
Protocol string
VIN string
Plate string
EventTime any
Latitude float64
Longitude float64
SpeedKMH any
TotalMileageKM any
SOCPercent any
AltitudeM any
DirectionDeg any
AlarmFlag any
StatusFlag any
ReceivedAt any
EventID string
}
func realtimeLocationFromEnvelope(env envelope.FrameEnvelope, vin string, plate string) (realtimeLocationRow, bool) {
latitude, okLat := numberField(env.Fields, envelope.FieldLatitude)
longitude, okLon := numberField(env.Fields, envelope.FieldLongitude)
if !okLat || !okLon {
return realtimeLocationRow{}, false
}
return realtimeLocationRow{
Protocol: string(env.Protocol),
VIN: vin,
Plate: plate,
EventTime: nullableTime(env.EventTimeMS),
Latitude: latitude,
Longitude: longitude,
SpeedKMH: nullableNumberField(env.Fields, envelope.FieldSpeedKMH),
TotalMileageKM: nullableNumberField(env.Fields, envelope.FieldTotalMileageKM),
SOCPercent: nullableNumberField(env.Fields, envelope.FieldSOCPercent),
AltitudeM: nullableNumberField(env.Fields, "altitude_m"),
DirectionDeg: nullableNumberField(env.Fields, "direction_deg"),
AlarmFlag: nullableNumberField(env.Fields, "alarm_flag"),
StatusFlag: nullableNumberField(env.Fields, "status_flag"),
ReceivedAt: nullableTime(env.ReceivedAtMS),
EventID: env.StableEventID(),
}, true
}
func nullableNumberField(fields map[string]any, key string) any {
value, ok := numberField(fields, key)
if !ok {
return nil
}
return value
}
func numberField(fields map[string]any, key string) (float64, bool) {
value, ok := fields[key]
if !ok {
return 0, false
}
switch typed := value.(type) {
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int8:
return float64(typed), true
case int16:
return float64(typed), true
case int32:
return float64(typed), true
case int64:
return float64(typed), true
case uint:
return float64(typed), true
case uint8:
return float64(typed), true
case uint16:
return float64(typed), true
case uint32:
return float64(typed), true
case uint64:
return float64(typed), true
default:
return 0, false
}
}
type BindingPlateResolver struct {
queryer Queryer
table string
}
type Queryer interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func NewBindingPlateResolver(queryer Queryer, table string) *BindingPlateResolver {
if queryer == nil {
panic("plate binding queryer must not be nil")
}
table = strings.TrimSpace(table)
if table == "" || !safeIdentifier(table) {
table = "vehicle_identity_binding"
}
return &BindingPlateResolver{queryer: queryer, table: table}
}
func (r *BindingPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) {
vin = strings.TrimSpace(vin)
if vin == "" {
return "", sql.ErrNoRows
}
query := "SELECT plate FROM " + r.table + " WHERE vin = ? AND plate IS NOT NULL AND plate <> ''"
var plate string
err := r.queryer.QueryRowContext(ctx, query, vin).Scan(&plate)
if err != nil {
return "", err
}
return strings.TrimSpace(plate), nil
}
func safeIdentifier(value string) bool {
if value == "" {
return false
}
for _, r := range value {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
continue
}
return false
}
return true
}
func nullableTime(ms int64) any {
if ms <= 0 {
return nil
}
return time.UnixMilli(ms)
}
func isRealtimeSnapshotEvent(env envelope.FrameEnvelope) bool {
switch env.Protocol {
case envelope.ProtocolGB32960:
return env.MessageID == "0x02" && hasGB32960RealtimeDataUnits(env)
case envelope.ProtocolJT808:
return env.MessageID == "0x0200" && hasLocationFields(env)
case envelope.ProtocolYutongMQTT:
return len(env.Fields) > 0
default:
return false
}
}
func hasGB32960RealtimeDataUnits(env envelope.FrameEnvelope) bool {
if units, ok := env.Parsed["data_units"].([]any); ok && len(units) > 0 {
return true
}
if units, ok := env.Parsed["data_units"].([]map[string]any); ok && len(units) > 0 {
return true
}
return false
}
func hasLocationFields(env envelope.FrameEnvelope) bool {
_, okLat := numberField(env.Fields, envelope.FieldLatitude)
_, okLon := numberField(env.Fields, envelope.FieldLongitude)
return okLat && okLon
}
const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot (
protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '',
plate VARCHAR(32) NOT NULL DEFAULT '',
platform_name VARCHAR(64) NOT NULL DEFAULT '',
peer VARCHAR(128) NOT NULL DEFAULT '',
parsed_json LONGTEXT NULL,
event_time DATETIME(3) NULL,
received_at DATETIME(3) NULL,
event_id VARCHAR(64) NOT NULL DEFAULT '',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (protocol, vin),
KEY idx_vin (vin),
KEY idx_protocol_updated (protocol, updated_at)
)`
var realtimeSnapshotCompatibilitySQL = []string{
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN platform_name VARCHAR(64) NOT NULL DEFAULT '' AFTER plate",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN peer VARCHAR(128) NOT NULL DEFAULT '' AFTER platform_name",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN parsed_json LONGTEXT NULL AFTER peer",
}
func isDuplicateColumnError(err error) bool {
text := strings.ToLower(err.Error())
return strings.Contains(text, "duplicate column") || strings.Contains(text, "1060")
}
const upsertRealtimeSnapshotSQL = `
INSERT INTO vehicle_realtime_snapshot
(protocol, vin, plate, platform_name, peer, parsed_json, event_time, received_at, event_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
platform_name = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(platform_name), platform_name),
peer = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(peer), peer),
parsed_json = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(parsed_json), parsed_json),
event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_time), event_time),
received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(received_at), received_at),
event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_id), event_id),
updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), CURRENT_TIMESTAMP, updated_at)
`
const selectRealtimeSnapshotParsedJSONSQL = `SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = ? AND vin = ?`
const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location (
protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '',
plate VARCHAR(32) NOT NULL DEFAULT '',
event_time DATETIME(3) NULL,
latitude DECIMAL(12,6) NOT NULL,
longitude DECIMAL(12,6) NOT NULL,
speed_kmh DECIMAL(10,3) NULL,
total_mileage_km DECIMAL(18,3) NULL,
soc_percent DECIMAL(6,2) NULL,
altitude_m DECIMAL(10,3) NULL,
direction_deg DECIMAL(10,3) NULL,
alarm_flag BIGINT NULL,
status_flag BIGINT NULL,
received_at DATETIME(3) NULL,
event_id VARCHAR(64) NOT NULL DEFAULT '',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (protocol, vin),
KEY idx_vin (vin),
KEY idx_protocol_updated (protocol, updated_at)
)`
const upsertRealtimeLocationSQL = `
INSERT INTO vehicle_realtime_location
(protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km,
soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(event_time), event_time),
latitude = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(latitude), latitude),
longitude = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(longitude), longitude),
speed_kmh = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(speed_kmh), speed_kmh), speed_kmh),
total_mileage_km = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(total_mileage_km), total_mileage_km), total_mileage_km),
soc_percent = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(soc_percent), soc_percent), soc_percent),
altitude_m = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(altitude_m), altitude_m), altitude_m),
direction_deg = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(direction_deg), direction_deg), direction_deg),
alarm_flag = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(alarm_flag), alarm_flag), alarm_flag),
status_flag = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(status_flag), status_flag), status_flag),
received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(received_at), received_at),
event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(event_id), event_id),
updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), CURRENT_TIMESTAMP, updated_at)
`