package realtime import ( "context" "database/sql" "encoding/json" "errors" "strings" "sync" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" ) type SnapshotExecer interface { ExecContext(context.Context, string, ...any) (sql.Result, error) } type PlateResolver interface { PlateByVIN(context.Context, string) (string, error) } type CachedPlateResolver struct { delegate PlateResolver ttl time.Duration now func() time.Time mu sync.Mutex entries map[string]cachedPlateEntry } type cachedPlateEntry struct { plate string notFound bool expiresAt time.Time } func NewCachedPlateResolver(delegate PlateResolver, ttl time.Duration) *CachedPlateResolver { if delegate == nil { panic("cached plate resolver delegate must not be nil") } if ttl <= 0 { ttl = 10 * time.Minute } return &CachedPlateResolver{ delegate: delegate, ttl: ttl, now: time.Now, entries: map[string]cachedPlateEntry{}, } } func (r *CachedPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) { vin = strings.TrimSpace(vin) if vin == "" { return "", sql.ErrNoRows } now := r.now() r.mu.Lock() entry, ok := r.entries[vin] if ok && now.Before(entry.expiresAt) { r.mu.Unlock() if entry.notFound { return "", sql.ErrNoRows } return entry.plate, nil } r.mu.Unlock() plate, err := r.delegate.PlateByVIN(ctx, vin) if err != nil && !errors.Is(err, sql.ErrNoRows) { return "", err } r.mu.Lock() r.entries[vin] = cachedPlateEntry{ plate: strings.TrimSpace(plate), notFound: errors.Is(err, sql.ErrNoRows), expiresAt: now.Add(r.ttl), } r.mu.Unlock() if err != nil { return "", err } return strings.TrimSpace(plate), nil } type SnapshotWriter struct { exec SnapshotExecer plateResolver PlateResolver } func NewSnapshotWriter(exec SnapshotExecer) *SnapshotWriter { return NewSnapshotWriterWithPlateResolver(exec, nil) } func NewSnapshotWriterWithPlateResolver(exec SnapshotExecer, plateResolver PlateResolver) *SnapshotWriter { if exec == nil { panic("snapshot execer must not be nil") } return &SnapshotWriter{exec: exec, plateResolver: plateResolver} } func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error { if _, err := w.exec.ExecContext(ctx, realtimeSnapshotTableSQL); err != nil { return err } for _, statement := range realtimeSnapshotCompatibilitySQL { if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateColumnError(err) { return err } } if _, err := w.exec.ExecContext(ctx, realtimeLocationTableSQL); err != nil { return err } if _, err := w.exec.ExecContext(ctx, realtimeKVTableSQL); err != nil { return err } return nil } func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope) error { vin := strings.TrimSpace(env.VIN) if vin == "" { return nil } if !isRealtimeSnapshotEvent(env) { return nil } plate, err := w.plateForEnvelope(ctx, env) if err != nil { return err } eventTime := nullableTime(env.EventTimeMS) receivedAt := nullableTime(env.ReceivedAtMS) platformName := platformNameFromEnvelope(env) parsed, err := w.parsedForEnvelope(ctx, env, vin) if err != nil { return err } if _, err = w.exec.ExecContext(ctx, upsertRealtimeSnapshotSQL, string(env.Protocol), vin, plate, platformName, env.SourceEndpoint, marshalParsedJSON(parsed), eventTime, receivedAt, env.StableEventID(), ); err != nil { return err } if err := w.upsertKV(ctx, env, parsed); err != nil { return err } location, ok := realtimeLocationFromEnvelope(env, vin, plate) if !ok { return nil } _, err = w.exec.ExecContext(ctx, upsertRealtimeLocationSQL, location.Protocol, location.VIN, location.Plate, location.EventTime, location.Latitude, location.Longitude, location.SpeedKMH, location.TotalMileageKM, location.SOCPercent, location.AltitudeM, location.DirectionDeg, location.AlarmFlag, location.StatusFlag, location.ReceivedAt, location.EventID, ) return err } func (w *SnapshotWriter) parsedForEnvelope(ctx context.Context, env envelope.FrameEnvelope, vin string) (map[string]any, error) { if len(env.Parsed) == 0 { return nil, nil } parsed := cloneMap(env.Parsed) if queryer, ok := w.exec.(Queryer); ok { existing, err := realtimeSnapshotParsedJSON(ctx, queryer, env.Protocol, vin) if err != nil { return nil, err } if len(existing) > 0 { parsed = mergeParsedForProtocol(env.Protocol, existing, env.Parsed) } } return parsed, nil } func (w *SnapshotWriter) upsertKV(ctx context.Context, env envelope.FrameEnvelope, parsed map[string]any) error { rows := realtimeKVFields(env, parsed) if len(rows) == 0 { return nil } placeholders := make([]string, 0, len(rows)) args := make([]any, 0, len(rows)*9) for _, row := range rows { placeholders = append(placeholders, "(?, ?, ?, ?, ?, ?, ?, ?, ?)") args = append(args, string(row.Protocol), row.VIN, row.Domain, row.Field, row.Value, row.ValueType, nullableTime(row.EventTimeMS), nullableTime(row.ReceivedAtMS), row.EventID, ) } _, err := w.exec.ExecContext(ctx, upsertRealtimeKVSQLPrefix+strings.Join(placeholders, ",")+upsertRealtimeKVSQLSuffix, args...) return err } func platformNameFromEnvelope(env envelope.FrameEnvelope) string { if env.Fields != nil { if value := strings.TrimSpace(stringValue(env.Fields["platform_account"])); value != "" { return value } } if env.Parsed != nil { if value := strings.TrimSpace(stringValue(env.Parsed["platform_name"])); value != "" { return value } if login, ok := env.Parsed["platform_login"].(map[string]any); ok { return strings.TrimSpace(stringValue(login["username"])) } } return "" } func stringValue(value any) string { switch typed := value.(type) { case string: return typed default: return "" } } func realtimeSnapshotParsedJSON(ctx context.Context, queryer Queryer, protocol envelope.Protocol, vin string) (map[string]any, error) { var raw sql.NullString err := queryer.QueryRowContext(ctx, selectRealtimeSnapshotParsedJSONSQL, string(protocol), vin).Scan(&raw) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } return nil, err } if !raw.Valid || strings.TrimSpace(raw.String) == "" { return nil, nil } var parsed map[string]any if err := json.Unmarshal([]byte(raw.String), &parsed); err != nil { return nil, nil } return parsed, nil } func marshalParsedJSON(parsed map[string]any) any { if len(parsed) == 0 { return nil } data, err := json.Marshal(parsed) if err != nil { return nil } return string(data) } func (w *SnapshotWriter) plateForEnvelope(ctx context.Context, env envelope.FrameEnvelope) (string, error) { if plate := strings.TrimSpace(env.Plate); plate != "" { return plate, nil } if w.plateResolver == nil { return "", nil } vin := strings.TrimSpace(env.VIN) if vin == "" { return "", nil } plate, err := w.plateResolver.PlateByVIN(ctx, vin) if err != nil { if errors.Is(err, sql.ErrNoRows) { return "", nil } return "", err } return strings.TrimSpace(plate), nil } type realtimeLocationRow struct { Protocol string VIN string Plate string EventTime any Latitude float64 Longitude float64 SpeedKMH any TotalMileageKM any SOCPercent any AltitudeM any DirectionDeg any AlarmFlag any StatusFlag any ReceivedAt any EventID string } func realtimeLocationFromEnvelope(env envelope.FrameEnvelope, vin string, plate string) (realtimeLocationRow, bool) { latitude, okLat := numberField(env.Fields, envelope.FieldLatitude) longitude, okLon := numberField(env.Fields, envelope.FieldLongitude) if !okLat || !okLon { return realtimeLocationRow{}, false } return realtimeLocationRow{ Protocol: string(env.Protocol), VIN: vin, Plate: plate, EventTime: nullableTime(env.EventTimeMS), Latitude: latitude, Longitude: longitude, SpeedKMH: nullableNumberField(env.Fields, envelope.FieldSpeedKMH), TotalMileageKM: nullableNumberField(env.Fields, envelope.FieldTotalMileageKM), SOCPercent: nullableNumberField(env.Fields, envelope.FieldSOCPercent), AltitudeM: nullableNumberField(env.Fields, "altitude_m"), DirectionDeg: nullableNumberField(env.Fields, "direction_deg"), AlarmFlag: nullableNumberField(env.Fields, "alarm_flag"), StatusFlag: nullableNumberField(env.Fields, "status_flag"), ReceivedAt: nullableTime(env.ReceivedAtMS), EventID: env.StableEventID(), }, true } func nullableNumberField(fields map[string]any, key string) any { value, ok := numberField(fields, key) if !ok { return nil } return value } func numberField(fields map[string]any, key string) (float64, bool) { value, ok := fields[key] if !ok { return 0, false } switch typed := value.(type) { case float64: return typed, true case float32: return float64(typed), true case int: return float64(typed), true case int8: return float64(typed), true case int16: return float64(typed), true case int32: return float64(typed), true case int64: return float64(typed), true case uint: return float64(typed), true case uint8: return float64(typed), true case uint16: return float64(typed), true case uint32: return float64(typed), true case uint64: return float64(typed), true default: return 0, false } } type BindingPlateResolver struct { queryer Queryer table string } type Queryer interface { QueryRowContext(context.Context, string, ...any) *sql.Row } func NewBindingPlateResolver(queryer Queryer, table string) *BindingPlateResolver { if queryer == nil { panic("plate binding queryer must not be nil") } table = strings.TrimSpace(table) if table == "" || !safeIdentifier(table) { table = "vehicle_identity_binding" } return &BindingPlateResolver{queryer: queryer, table: table} } func (r *BindingPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) { vin = strings.TrimSpace(vin) if vin == "" { return "", sql.ErrNoRows } query := "SELECT plate FROM " + r.table + " WHERE vin = ? AND plate IS NOT NULL AND plate <> ''" var plate string err := r.queryer.QueryRowContext(ctx, query, vin).Scan(&plate) if err != nil { return "", err } return strings.TrimSpace(plate), nil } func safeIdentifier(value string) bool { if value == "" { return false } for _, r := range value { if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' { continue } return false } return true } func nullableTime(ms int64) any { if ms <= 0 { return nil } return time.UnixMilli(ms) } func isRealtimeSnapshotEvent(env envelope.FrameEnvelope) bool { switch env.Protocol { case envelope.ProtocolGB32960: return env.MessageID == "0x02" && hasGB32960RealtimeDataUnits(env) case envelope.ProtocolJT808: return env.MessageID == "0x0200" && hasLocationFields(env) case envelope.ProtocolYutongMQTT: return len(env.Fields) > 0 default: return false } } func hasGB32960RealtimeDataUnits(env envelope.FrameEnvelope) bool { if units, ok := env.Parsed["data_units"].([]any); ok && len(units) > 0 { return true } if units, ok := env.Parsed["data_units"].([]map[string]any); ok && len(units) > 0 { return true } return false } func hasLocationFields(env envelope.FrameEnvelope) bool { _, okLat := numberField(env.Fields, envelope.FieldLatitude) _, okLon := numberField(env.Fields, envelope.FieldLongitude) return okLat && okLon } const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot ( protocol VARCHAR(32) NOT NULL, vin VARCHAR(32) NOT NULL DEFAULT '', plate VARCHAR(32) NOT NULL DEFAULT '', platform_name VARCHAR(64) NOT NULL DEFAULT '', peer VARCHAR(128) NOT NULL DEFAULT '', parsed_json LONGTEXT NULL, event_time DATETIME(3) NULL, received_at DATETIME(3) NULL, event_id VARCHAR(64) NOT NULL DEFAULT '', updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (protocol, vin), KEY idx_vin (vin), KEY idx_protocol_updated (protocol, updated_at) )` var realtimeSnapshotCompatibilitySQL = []string{ "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN platform_name VARCHAR(64) NOT NULL DEFAULT '' AFTER plate", "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN peer VARCHAR(128) NOT NULL DEFAULT '' AFTER platform_name", "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN parsed_json LONGTEXT NULL AFTER peer", } func isDuplicateColumnError(err error) bool { text := strings.ToLower(err.Error()) return strings.Contains(text, "duplicate column") || strings.Contains(text, "1060") } const upsertRealtimeSnapshotSQL = ` INSERT INTO vehicle_realtime_snapshot (protocol, vin, plate, platform_name, peer, parsed_json, event_time, received_at, event_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE plate = IF(VALUES(plate) <> '', VALUES(plate), plate), platform_name = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(platform_name), platform_name), peer = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(peer), peer), parsed_json = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(parsed_json), parsed_json), event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_time), event_time), received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(received_at), received_at), event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_id), event_id), updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), CURRENT_TIMESTAMP, updated_at) ` const selectRealtimeSnapshotParsedJSONSQL = `SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = ? AND vin = ?` const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location ( protocol VARCHAR(32) NOT NULL, vin VARCHAR(32) NOT NULL DEFAULT '', plate VARCHAR(32) NOT NULL DEFAULT '', event_time DATETIME(3) NULL, latitude DECIMAL(12,6) NOT NULL, longitude DECIMAL(12,6) NOT NULL, speed_kmh DECIMAL(10,3) NULL, total_mileage_km DECIMAL(18,3) NULL, soc_percent DECIMAL(6,2) NULL, altitude_m DECIMAL(10,3) NULL, direction_deg DECIMAL(10,3) NULL, alarm_flag BIGINT NULL, status_flag BIGINT NULL, received_at DATETIME(3) NULL, event_id VARCHAR(64) NOT NULL DEFAULT '', updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (protocol, vin), KEY idx_vin (vin), KEY idx_protocol_updated (protocol, updated_at) )` const realtimeKVTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_kv ( protocol VARCHAR(32) NOT NULL, vin VARCHAR(32) NOT NULL, domain_name VARCHAR(64) NOT NULL, field_name VARCHAR(160) NOT NULL, field_value TEXT NULL, value_type VARCHAR(16) NOT NULL DEFAULT '', event_time DATETIME(3) NULL, received_at DATETIME(3) NULL, event_id VARCHAR(64) NOT NULL DEFAULT '', updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (protocol, vin, domain_name, field_name), KEY idx_vin_domain (vin, domain_name), KEY idx_protocol_domain (protocol, domain_name) )` const upsertRealtimeKVSQLPrefix = ` INSERT INTO vehicle_realtime_kv (protocol, vin, domain_name, field_name, field_value, value_type, event_time, received_at, event_id) VALUES ` const upsertRealtimeKVSQLSuffix = ` ON DUPLICATE KEY UPDATE field_value = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(field_value), field_value), value_type = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(value_type), value_type), event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(event_time), event_time), received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(received_at), received_at), event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(event_id), event_id), updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), CURRENT_TIMESTAMP, updated_at) ` const upsertRealtimeLocationSQL = ` INSERT INTO vehicle_realtime_location (protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE plate = IF(VALUES(plate) <> '', VALUES(plate), plate), event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(event_time), event_time), latitude = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(latitude), latitude), longitude = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(longitude), longitude), speed_kmh = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(speed_kmh), speed_kmh), speed_kmh), total_mileage_km = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(total_mileage_km), total_mileage_km), total_mileage_km), soc_percent = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(soc_percent), soc_percent), soc_percent), altitude_m = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(altitude_m), altitude_m), altitude_m), direction_deg = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(direction_deg), direction_deg), direction_deg), alarm_flag = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(alarm_flag), alarm_flag), alarm_flag), status_flag = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(status_flag), status_flag), status_flag), received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(received_at), received_at), event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(event_id), event_id), updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), CURRENT_TIMESTAMP, updated_at) `