feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
773
go/vehicle-gateway/internal/identity/mapping_import.go
Normal file
773
go/vehicle-gateway/internal/identity/mapping_import.go
Normal file
@@ -0,0 +1,773 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
IdentifierTypeJT808Phone = "JT808_PHONE"
|
||||
IdentifierTypePlate = "PLATE"
|
||||
)
|
||||
|
||||
type MappingRecord struct {
|
||||
File string `json:"file"`
|
||||
Sheet string `json:"sheet"`
|
||||
Row int `json:"row"`
|
||||
SourceCode string `json:"source_code"`
|
||||
SourceName string `json:"source_name"`
|
||||
Protocol string `json:"protocol"`
|
||||
IdentifierType string `json:"identifier_type"`
|
||||
IdentifierValue string `json:"identifier_value"`
|
||||
RawValue string `json:"raw_value,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
OEM string `json:"oem,omitempty"`
|
||||
}
|
||||
|
||||
type MappingScanReport struct {
|
||||
Files int `json:"files"`
|
||||
Sheets int `json:"sheets"`
|
||||
Rows int `json:"rows"`
|
||||
Records int `json:"records"`
|
||||
Skipped int `json:"skipped"`
|
||||
UnsupportedFiles int `json:"unsupported_files,omitempty"`
|
||||
Sources []MappingSourceScanReport `json:"sources,omitempty"`
|
||||
UnsupportedItems []MappingUnsupportedFileReport `json:"unsupported_items,omitempty"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
type MappingSourceScanReport struct {
|
||||
SourceCode string `json:"source_code"`
|
||||
SourceName string `json:"source_name"`
|
||||
Files int `json:"files"`
|
||||
Sheets int `json:"sheets"`
|
||||
Rows int `json:"rows"`
|
||||
Records int `json:"records"`
|
||||
PhoneRecords int `json:"phone_records"`
|
||||
PlateRecords int `json:"plate_records"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
|
||||
type MappingUnsupportedFileReport struct {
|
||||
File string `json:"file"`
|
||||
Ext string `json:"ext"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type MappingImportOptions struct {
|
||||
Apply bool
|
||||
LegacyTable string
|
||||
ReportItemLimit int
|
||||
}
|
||||
|
||||
type MappingImportReport struct {
|
||||
Scan MappingScanReport `json:"scan"`
|
||||
Records int `json:"records"`
|
||||
Deduplicated int `json:"deduplicated"`
|
||||
Resolved int `json:"resolved"`
|
||||
Unresolved int `json:"unresolved"`
|
||||
Conflicts int `json:"conflicts"`
|
||||
WouldInsert int `json:"would_insert,omitempty"`
|
||||
WouldUpdate int `json:"would_update,omitempty"`
|
||||
Inserted int `json:"inserted,omitempty"`
|
||||
Updated int `json:"updated,omitempty"`
|
||||
SourceResults []MappingSourceImportStat `json:"source_results,omitempty"`
|
||||
UnresolvedItems []MappingRecord `json:"unresolved_items,omitempty"`
|
||||
ConflictItems []MappingConflict `json:"conflict_items,omitempty"`
|
||||
}
|
||||
|
||||
type MappingSourceImportStat struct {
|
||||
SourceCode string `json:"source_code"`
|
||||
SourceName string `json:"source_name"`
|
||||
Records int `json:"records"`
|
||||
Deduplicated int `json:"deduplicated"`
|
||||
Resolved int `json:"resolved"`
|
||||
Unresolved int `json:"unresolved"`
|
||||
Conflicts int `json:"conflicts"`
|
||||
WouldInsert int `json:"would_insert,omitempty"`
|
||||
WouldUpdate int `json:"would_update,omitempty"`
|
||||
Inserted int `json:"inserted,omitempty"`
|
||||
Updated int `json:"updated,omitempty"`
|
||||
}
|
||||
|
||||
type MappingConflict struct {
|
||||
Record MappingRecord `json:"record"`
|
||||
ExistingVIN string `json:"existing_vin,omitempty"`
|
||||
NewVIN string `json:"new_vin,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type resolvedMappingRecord struct {
|
||||
MappingRecord
|
||||
VIN string
|
||||
}
|
||||
|
||||
type mappingStore interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}
|
||||
|
||||
var digitPattern = regexp.MustCompile(`\D+`)
|
||||
|
||||
func EnsureVehicleIdentifierSchema(ctx context.Context, db *sql.DB) error {
|
||||
if db == nil {
|
||||
return errors.New("identity db must not be nil")
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, vehicleTableSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := db.ExecContext(ctx, vehicleIdentifierTableSQL)
|
||||
return err
|
||||
}
|
||||
|
||||
func ReadMappingDirectory(root string) ([]MappingRecord, MappingScanReport, error) {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
return nil, MappingScanReport{}, errors.New("mapping input directory is empty")
|
||||
}
|
||||
var records []MappingRecord
|
||||
report := MappingScanReport{}
|
||||
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
report.Errors = append(report.Errors, err.Error())
|
||||
return nil
|
||||
}
|
||||
if entry == nil || entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := entry.Name()
|
||||
if strings.HasPrefix(name, "~$") || strings.HasPrefix(name, "._") {
|
||||
return nil
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(name))
|
||||
if !isSupportedMappingWorkbookExt(ext) {
|
||||
if isUnsupportedMappingWorkbookExt(ext) {
|
||||
report.UnsupportedFiles++
|
||||
report.UnsupportedItems = append(report.UnsupportedItems, MappingUnsupportedFileReport{
|
||||
File: path,
|
||||
Ext: ext,
|
||||
Reason: "convert legacy workbook to .xlsx before import",
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fileRecords, fileReport, err := readMappingWorkbook(root, path)
|
||||
report.Files++
|
||||
report.Sheets += fileReport.Sheets
|
||||
report.Rows += fileReport.Rows
|
||||
report.Records += fileReport.Records
|
||||
report.Skipped += fileReport.Skipped
|
||||
report.Errors = append(report.Errors, fileReport.Errors...)
|
||||
for _, source := range fileReport.Sources {
|
||||
mergeMappingSourceScan(&report, source)
|
||||
}
|
||||
if err != nil {
|
||||
report.Errors = append(report.Errors, fmt.Sprintf("%s: %v", path, err))
|
||||
return nil
|
||||
}
|
||||
records = append(records, fileRecords...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return records, report, err
|
||||
}
|
||||
sortMappingSourceScans(report.Sources)
|
||||
return records, report, nil
|
||||
}
|
||||
|
||||
func readMappingWorkbook(root string, path string) ([]MappingRecord, MappingScanReport, error) {
|
||||
workbook, err := excelize.OpenFile(path)
|
||||
if err != nil {
|
||||
return nil, MappingScanReport{}, err
|
||||
}
|
||||
defer func() { _ = workbook.Close() }()
|
||||
|
||||
sourceCode, sourceName := mappingSource(root, path)
|
||||
sourceReport := MappingSourceScanReport{
|
||||
SourceCode: sourceCode,
|
||||
SourceName: sourceName,
|
||||
Files: 1,
|
||||
}
|
||||
var records []MappingRecord
|
||||
report := MappingScanReport{}
|
||||
for _, sheet := range workbook.GetSheetList() {
|
||||
rows, err := workbook.GetRows(sheet)
|
||||
if err != nil {
|
||||
report.Errors = append(report.Errors, fmt.Sprintf("%s/%s: %v", path, sheet, err))
|
||||
continue
|
||||
}
|
||||
report.Sheets++
|
||||
sourceReport.Sheets++
|
||||
report.Rows += len(rows)
|
||||
sourceReport.Rows += len(rows)
|
||||
if len(rows) == 0 {
|
||||
continue
|
||||
}
|
||||
header, dataStart := mappingHeader(rows)
|
||||
for rowIndex := dataStart; rowIndex < len(rows); rowIndex++ {
|
||||
row := rows[rowIndex]
|
||||
plate := normalizePlate(cellByHeader(row, header, "plate"))
|
||||
rawPhone := cellByHeader(row, header, "phone")
|
||||
phone := normalizeMappingPhone(rawPhone)
|
||||
if len(header) == 0 {
|
||||
plate = normalizePlate(cell(row, 0))
|
||||
rawPhone = cell(row, 1)
|
||||
phone = normalizeMappingPhone(rawPhone)
|
||||
}
|
||||
if plate == "" && phone == "" {
|
||||
report.Skipped++
|
||||
sourceReport.Skipped++
|
||||
continue
|
||||
}
|
||||
if phone != "" {
|
||||
records = append(records, MappingRecord{
|
||||
File: path,
|
||||
Sheet: sheet,
|
||||
Row: rowIndex + 1,
|
||||
SourceCode: sourceCode,
|
||||
SourceName: sourceName,
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: phone,
|
||||
RawValue: strings.TrimSpace(rawPhone),
|
||||
Plate: plate,
|
||||
OEM: sourceName,
|
||||
})
|
||||
sourceReport.PhoneRecords++
|
||||
sourceReport.Records++
|
||||
}
|
||||
if plate != "" {
|
||||
records = append(records, MappingRecord{
|
||||
File: path,
|
||||
Sheet: sheet,
|
||||
Row: rowIndex + 1,
|
||||
SourceCode: sourceCode,
|
||||
SourceName: sourceName,
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypePlate,
|
||||
IdentifierValue: plate,
|
||||
RawValue: plate,
|
||||
Plate: plate,
|
||||
OEM: sourceName,
|
||||
})
|
||||
sourceReport.PlateRecords++
|
||||
sourceReport.Records++
|
||||
}
|
||||
}
|
||||
}
|
||||
report.Records = len(records)
|
||||
report.Sources = []MappingSourceScanReport{sourceReport}
|
||||
return records, report, nil
|
||||
}
|
||||
|
||||
func mergeMappingSourceScan(report *MappingScanReport, source MappingSourceScanReport) {
|
||||
if report == nil || strings.TrimSpace(source.SourceCode) == "" {
|
||||
return
|
||||
}
|
||||
for index := range report.Sources {
|
||||
if report.Sources[index].SourceCode != source.SourceCode {
|
||||
continue
|
||||
}
|
||||
report.Sources[index].Files += source.Files
|
||||
report.Sources[index].Sheets += source.Sheets
|
||||
report.Sources[index].Rows += source.Rows
|
||||
report.Sources[index].Records += source.Records
|
||||
report.Sources[index].PhoneRecords += source.PhoneRecords
|
||||
report.Sources[index].PlateRecords += source.PlateRecords
|
||||
report.Sources[index].Skipped += source.Skipped
|
||||
if report.Sources[index].SourceName == "" {
|
||||
report.Sources[index].SourceName = source.SourceName
|
||||
}
|
||||
return
|
||||
}
|
||||
report.Sources = append(report.Sources, source)
|
||||
}
|
||||
|
||||
func sortMappingSourceScans(sources []MappingSourceScanReport) {
|
||||
sort.SliceStable(sources, func(i, j int) bool {
|
||||
return sources[i].SourceCode < sources[j].SourceCode
|
||||
})
|
||||
}
|
||||
|
||||
func isSupportedMappingWorkbookExt(ext string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(ext)) {
|
||||
case ".xlsx", ".xlsm":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isUnsupportedMappingWorkbookExt(ext string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(ext)) {
|
||||
case ".xls", ".xlsb":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func ImportMappingRecords(ctx context.Context, db *sql.DB, records []MappingRecord, scan MappingScanReport, opts MappingImportOptions) (MappingImportReport, error) {
|
||||
if db == nil {
|
||||
return MappingImportReport{}, errors.New("identity db must not be nil")
|
||||
}
|
||||
legacyTable := strings.TrimSpace(opts.LegacyTable)
|
||||
if legacyTable == "" || !safeIdentifier(legacyTable) {
|
||||
legacyTable = "vehicle_identity_binding"
|
||||
}
|
||||
report := MappingImportReport{
|
||||
Scan: scan,
|
||||
Records: len(records),
|
||||
}
|
||||
sourceStats := map[string]*MappingSourceImportStat{}
|
||||
for _, record := range records {
|
||||
sourceImportStat(sourceStats, record).Records++
|
||||
}
|
||||
reportLimit := opts.ReportItemLimit
|
||||
if reportLimit == 0 {
|
||||
reportLimit = 50
|
||||
}
|
||||
deduped, conflicts := dedupeMappingRecords(records)
|
||||
report.Deduplicated = len(deduped)
|
||||
report.Conflicts += len(conflicts)
|
||||
for _, conflict := range conflicts {
|
||||
sourceImportStat(sourceStats, conflict.Record).Conflicts++
|
||||
appendConflictItem(&report, conflict, reportLimit)
|
||||
}
|
||||
|
||||
store := mappingStore(db)
|
||||
var tx *sql.Tx
|
||||
if opts.Apply {
|
||||
var err error
|
||||
tx, err = db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
store = tx
|
||||
defer func() {
|
||||
if tx != nil {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for _, record := range deduped {
|
||||
sourceStat := sourceImportStat(sourceStats, record)
|
||||
sourceStat.Deduplicated++
|
||||
vin, err := resolveMappingVIN(ctx, store, legacyTable, record)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if vin == "" {
|
||||
report.Unresolved++
|
||||
sourceStat.Unresolved++
|
||||
appendUnresolvedItem(&report, record, reportLimit)
|
||||
continue
|
||||
}
|
||||
resolved := resolvedMappingRecord{MappingRecord: record, VIN: vin}
|
||||
existingVIN, exists, err := existingIdentifierVIN(ctx, store, resolved)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if exists && !strings.EqualFold(existingVIN, vin) {
|
||||
report.Conflicts++
|
||||
sourceStat.Conflicts++
|
||||
appendConflictItem(&report, MappingConflict{
|
||||
Record: record,
|
||||
ExistingVIN: existingVIN,
|
||||
NewVIN: vin,
|
||||
Reason: "identifier already points to another vin",
|
||||
}, reportLimit)
|
||||
continue
|
||||
}
|
||||
report.Resolved++
|
||||
sourceStat.Resolved++
|
||||
if exists {
|
||||
if opts.Apply {
|
||||
if err := updateVehicleIdentifier(ctx, store, resolved); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.Updated++
|
||||
sourceStat.Updated++
|
||||
} else {
|
||||
report.WouldUpdate++
|
||||
sourceStat.WouldUpdate++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if opts.Apply {
|
||||
if err := upsertVehicle(ctx, store, resolved); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := insertVehicleIdentifier(ctx, store, resolved); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.Inserted++
|
||||
sourceStat.Inserted++
|
||||
} else {
|
||||
report.WouldInsert++
|
||||
sourceStat.WouldInsert++
|
||||
}
|
||||
}
|
||||
if tx != nil {
|
||||
if err := tx.Commit(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
tx = nil
|
||||
}
|
||||
report.SourceResults = sortedMappingSourceImportStats(sourceStats)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func sourceImportStat(stats map[string]*MappingSourceImportStat, record MappingRecord) *MappingSourceImportStat {
|
||||
sourceCode := strings.TrimSpace(record.SourceCode)
|
||||
if sourceCode == "" {
|
||||
sourceCode = "unknown"
|
||||
}
|
||||
stat := stats[sourceCode]
|
||||
if stat != nil {
|
||||
if stat.SourceName == "" {
|
||||
stat.SourceName = strings.TrimSpace(record.SourceName)
|
||||
}
|
||||
return stat
|
||||
}
|
||||
stat = &MappingSourceImportStat{
|
||||
SourceCode: sourceCode,
|
||||
SourceName: strings.TrimSpace(record.SourceName),
|
||||
}
|
||||
stats[sourceCode] = stat
|
||||
return stat
|
||||
}
|
||||
|
||||
func sortedMappingSourceImportStats(stats map[string]*MappingSourceImportStat) []MappingSourceImportStat {
|
||||
if len(stats) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(stats))
|
||||
for key := range stats {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]MappingSourceImportStat, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
out = append(out, *stats[key])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendUnresolvedItem(report *MappingImportReport, record MappingRecord, limit int) {
|
||||
if report == nil {
|
||||
return
|
||||
}
|
||||
if limit >= 0 && len(report.UnresolvedItems) >= limit {
|
||||
return
|
||||
}
|
||||
report.UnresolvedItems = append(report.UnresolvedItems, record)
|
||||
}
|
||||
|
||||
func appendConflictItem(report *MappingImportReport, conflict MappingConflict, limit int) {
|
||||
if report == nil {
|
||||
return
|
||||
}
|
||||
if limit >= 0 && len(report.ConflictItems) >= limit {
|
||||
return
|
||||
}
|
||||
report.ConflictItems = append(report.ConflictItems, conflict)
|
||||
}
|
||||
|
||||
func dedupeMappingRecords(records []MappingRecord) ([]MappingRecord, []MappingConflict) {
|
||||
seen := map[string]MappingRecord{}
|
||||
indexByKey := map[string]int{}
|
||||
var out []MappingRecord
|
||||
var conflicts []MappingConflict
|
||||
for _, record := range records {
|
||||
record.IdentifierValue = normalizeIdentifierValue(record.IdentifierType, record.IdentifierValue)
|
||||
record.Plate = normalizePlate(record.Plate)
|
||||
if record.Protocol == "" {
|
||||
record.Protocol = "JT808"
|
||||
}
|
||||
if record.IdentifierValue == "" || record.IdentifierType == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.Join([]string{record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue}, "\x00")
|
||||
existing, ok := seen[key]
|
||||
if !ok {
|
||||
seen[key] = record
|
||||
indexByKey[key] = len(out)
|
||||
out = append(out, record)
|
||||
continue
|
||||
}
|
||||
if existing.Plate != "" && record.Plate != "" && existing.Plate != record.Plate {
|
||||
conflicts = append(conflicts, MappingConflict{
|
||||
Record: record,
|
||||
Reason: fmt.Sprintf("same source identifier maps to multiple plates: %s/%s", existing.Plate, record.Plate),
|
||||
})
|
||||
continue
|
||||
}
|
||||
merged := mergeMappingRecord(existing, record)
|
||||
seen[key] = merged
|
||||
if index, ok := indexByKey[key]; ok && index >= 0 && index < len(out) {
|
||||
out[index] = merged
|
||||
}
|
||||
}
|
||||
return out, conflicts
|
||||
}
|
||||
|
||||
func mergeMappingRecord(existing MappingRecord, incoming MappingRecord) MappingRecord {
|
||||
merged := existing
|
||||
if merged.File == "" {
|
||||
merged.File = incoming.File
|
||||
}
|
||||
if merged.Sheet == "" {
|
||||
merged.Sheet = incoming.Sheet
|
||||
}
|
||||
if merged.Row == 0 {
|
||||
merged.Row = incoming.Row
|
||||
}
|
||||
if merged.SourceName == "" {
|
||||
merged.SourceName = incoming.SourceName
|
||||
}
|
||||
if merged.Protocol == "" {
|
||||
merged.Protocol = incoming.Protocol
|
||||
}
|
||||
if merged.IdentifierType == "" {
|
||||
merged.IdentifierType = incoming.IdentifierType
|
||||
}
|
||||
if merged.IdentifierValue == "" {
|
||||
merged.IdentifierValue = incoming.IdentifierValue
|
||||
}
|
||||
if merged.RawValue == "" {
|
||||
merged.RawValue = incoming.RawValue
|
||||
}
|
||||
if merged.Plate == "" {
|
||||
merged.Plate = incoming.Plate
|
||||
}
|
||||
if merged.OEM == "" {
|
||||
merged.OEM = incoming.OEM
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func resolveMappingVIN(ctx context.Context, db mappingStore, legacyTable string, record MappingRecord) (string, error) {
|
||||
if record.Plate != "" {
|
||||
vin, err := lookupLegacyVIN(ctx, db, legacyTable, "plate", record.Plate)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return "", err
|
||||
}
|
||||
if vin != "" {
|
||||
return vin, nil
|
||||
}
|
||||
}
|
||||
if record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue != "" {
|
||||
vin, err := lookupLegacyVIN(ctx, db, legacyTable, "phone", record.IdentifierValue)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return "", err
|
||||
}
|
||||
return vin, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func lookupLegacyVIN(ctx context.Context, db mappingStore, table string, column string, value string) (string, error) {
|
||||
if !safeIdentifier(table) || !safeIdentifier(column) {
|
||||
return "", sql.ErrNoRows
|
||||
}
|
||||
query := "SELECT vin FROM " + table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' LIMIT 1"
|
||||
var vin string
|
||||
err := db.QueryRowContext(ctx, query, value).Scan(&vin)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(vin), nil
|
||||
}
|
||||
|
||||
func existingIdentifierVIN(ctx context.Context, db mappingStore, record resolvedMappingRecord) (string, bool, error) {
|
||||
var vin string
|
||||
err := db.QueryRowContext(ctx, `SELECT vin FROM vehicle_identifier
|
||||
WHERE protocol = ? AND source_code = ? AND identifier_type = ? AND identifier_value = ?`,
|
||||
record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue).Scan(&vin)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return strings.TrimSpace(vin), true, nil
|
||||
}
|
||||
|
||||
func upsertVehicle(ctx context.Context, db mappingStore, record resolvedMappingRecord) error {
|
||||
_, err := db.ExecContext(ctx, `INSERT INTO vehicle (vin, plate, oem, enabled)
|
||||
VALUES (?, ?, ?, 1)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
|
||||
oem = IF(VALUES(oem) <> '', VALUES(oem), oem),
|
||||
enabled = 1`,
|
||||
record.VIN, record.Plate, record.OEM)
|
||||
return err
|
||||
}
|
||||
|
||||
func insertVehicleIdentifier(ctx context.Context, db mappingStore, record resolvedMappingRecord) error {
|
||||
_, err := db.ExecContext(ctx, `INSERT INTO vehicle_identifier
|
||||
(protocol, source_code, identifier_type, identifier_value, vin, plate, oem, raw_value, enabled, latest_import_file)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`,
|
||||
record.Protocol,
|
||||
record.SourceCode,
|
||||
record.IdentifierType,
|
||||
record.IdentifierValue,
|
||||
record.VIN,
|
||||
record.Plate,
|
||||
record.OEM,
|
||||
record.RawValue,
|
||||
record.File,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func updateVehicleIdentifier(ctx context.Context, db mappingStore, record resolvedMappingRecord) error {
|
||||
_, err := db.ExecContext(ctx, `UPDATE vehicle_identifier
|
||||
SET plate = IF(? <> '', ?, plate),
|
||||
oem = IF(? <> '', ?, oem),
|
||||
raw_value = IF(? <> '', ?, raw_value),
|
||||
latest_import_file = ?,
|
||||
enabled = 1
|
||||
WHERE protocol = ? AND source_code = ? AND identifier_type = ? AND identifier_value = ?`,
|
||||
record.Plate, record.Plate,
|
||||
record.OEM, record.OEM,
|
||||
record.RawValue, record.RawValue,
|
||||
record.File,
|
||||
record.Protocol,
|
||||
record.SourceCode,
|
||||
record.IdentifierType,
|
||||
record.IdentifierValue,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func mappingSource(root string, path string) (string, string) {
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
rel = filepath.Base(path)
|
||||
}
|
||||
parts := strings.Split(filepath.ToSlash(rel), "/")
|
||||
name := strings.TrimSpace(parts[0])
|
||||
if name == "" || strings.EqualFold(name, ".") {
|
||||
name = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
||||
}
|
||||
code := sourceCode(name)
|
||||
return code, name
|
||||
}
|
||||
|
||||
func sourceCode(name string) string {
|
||||
switch strings.TrimSpace(strings.ToLower(name)) {
|
||||
case "g7s":
|
||||
return "g7s"
|
||||
case "信达":
|
||||
return "xinda"
|
||||
case "广安北斗", "广安车联":
|
||||
return "guangan_beidou"
|
||||
case "东方北斗":
|
||||
return "dongfang_beidou"
|
||||
case "赛格":
|
||||
return "saige"
|
||||
default:
|
||||
return normalizeASCIIKey(name)
|
||||
}
|
||||
}
|
||||
|
||||
func mappingHeader(rows [][]string) (map[string]int, int) {
|
||||
for index, row := range rows {
|
||||
header := map[string]int{}
|
||||
for columnIndex, value := range row {
|
||||
key := normalizeHeader(value)
|
||||
switch key {
|
||||
case "车牌", "车牌号", "车牌号码":
|
||||
header["plate"] = columnIndex
|
||||
case "sim", "sim卡号", "手机号", "终端手机号", "终端id", "终端标识":
|
||||
header["phone"] = columnIndex
|
||||
}
|
||||
}
|
||||
if len(header) > 0 {
|
||||
return header, index + 1
|
||||
}
|
||||
if index >= 5 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
func cellByHeader(row []string, header map[string]int, key string) string {
|
||||
if len(header) == 0 {
|
||||
return ""
|
||||
}
|
||||
index, ok := header[key]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return cell(row, index)
|
||||
}
|
||||
|
||||
func cell(row []string, index int) string {
|
||||
if index < 0 || index >= len(row) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(row[index])
|
||||
}
|
||||
|
||||
func normalizeHeader(value string) string {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
value = strings.ReplaceAll(value, " ", "")
|
||||
value = strings.ReplaceAll(value, "\t", "")
|
||||
value = strings.ReplaceAll(value, "（", "(")
|
||||
value = strings.ReplaceAll(value, "）", ")")
|
||||
return value
|
||||
}
|
||||
|
||||
func normalizePlate(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
value = strings.ReplaceAll(value, " ", "")
|
||||
value = strings.ReplaceAll(value, "\t", "")
|
||||
return value
|
||||
}
|
||||
|
||||
func normalizeMappingPhone(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.ContainsAny(value, ".eE") {
|
||||
if parsed, err := strconv.ParseFloat(value, 64); err == nil && parsed > 0 {
|
||||
return normalizePhone(strconv.FormatFloat(parsed, 'f', 0, 64))
|
||||
}
|
||||
}
|
||||
digits := digitPattern.ReplaceAllString(value, "")
|
||||
return normalizePhone(digits)
|
||||
}
|
||||
|
||||
func normalizeASCIIKey(value string) string {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
var b strings.Builder
|
||||
lastUnderscore := false
|
||||
for _, r := range value {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
lastUnderscore = false
|
||||
continue
|
||||
}
|
||||
if !lastUnderscore {
|
||||
b.WriteByte('_')
|
||||
lastUnderscore = true
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), "_")
|
||||
}
|
||||
345
go/vehicle-gateway/internal/identity/mapping_import_test.go
Normal file
345
go/vehicle-gateway/internal/identity/mapping_import_test.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
func TestReadMappingDirectoryExtractsPhoneAndPlate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeWorkbook(t, filepath.Join(dir, "G7s", "宇速全量.xlsx"), [][]string{
|
||||
{"车牌号", "sim卡号", "设备号"},
|
||||
{"粤AG18312", "013307795425", "DEV001"},
|
||||
})
|
||||
writeWorkbook(t, filepath.Join(dir, "东方北斗", "无标题0703.xlsx"), [][]string{
|
||||
{"沪A01559F", "64341233712"},
|
||||
})
|
||||
if err := os.MkdirAll(filepath.Join(dir, "信达"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "信达", "旧格式.xls"), []byte("legacy xls"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "信达", "说明.txt"), []byte("ignored"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
|
||||
records, report, err := ReadMappingDirectory(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMappingDirectory() error = %v", err)
|
||||
}
|
||||
if report.Files != 2 {
|
||||
t.Fatalf("files = %d, report=%#v", report.Files, report)
|
||||
}
|
||||
if report.UnsupportedFiles != 1 || len(report.UnsupportedItems) != 1 {
|
||||
t.Fatalf("unsupported files = %d, items=%#v", report.UnsupportedFiles, report.UnsupportedItems)
|
||||
}
|
||||
if got := report.UnsupportedItems[0]; got.Ext != ".xls" || got.Reason == "" {
|
||||
t.Fatalf("unsupported item = %#v", got)
|
||||
}
|
||||
sources := map[string]MappingSourceScanReport{}
|
||||
for _, source := range report.Sources {
|
||||
sources[source.SourceCode] = source
|
||||
}
|
||||
if got := sources["g7s"]; got.Files != 1 || got.Sheets != 1 || got.PhoneRecords != 1 || got.PlateRecords != 1 || got.Records != 2 {
|
||||
t.Fatalf("g7s source report = %#v", got)
|
||||
}
|
||||
if got := sources["dongfang_beidou"]; got.Files != 1 || got.Sheets != 1 || got.PhoneRecords != 1 || got.PlateRecords != 1 || got.Records != 2 {
|
||||
t.Fatalf("dongfang source report = %#v", got)
|
||||
}
|
||||
var phoneSeen bool
|
||||
var plateSeen bool
|
||||
var headerlessSeen bool
|
||||
for _, record := range records {
|
||||
if record.SourceCode == "g7s" && record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue == "13307795425" && record.Plate == "粤AG18312" {
|
||||
phoneSeen = true
|
||||
}
|
||||
if record.SourceCode == "g7s" && record.IdentifierType == IdentifierTypePlate && record.IdentifierValue == "粤AG18312" {
|
||||
plateSeen = true
|
||||
}
|
||||
if record.SourceCode == "dongfang_beidou" && record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue == "64341233712" && record.Plate == "沪A01559F" {
|
||||
headerlessSeen = true
|
||||
}
|
||||
}
|
||||
if !phoneSeen || !plateSeen || !headerlessSeen {
|
||||
t.Fatalf("records missing expected mappings: %#v", records)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsDryRunResolvesVINFromLegacyPlate(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "013307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportMappingRecords() error = %v", err)
|
||||
}
|
||||
if report.Resolved != 1 || report.WouldInsert != 1 || report.Unresolved != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if len(report.SourceResults) != 1 {
|
||||
t.Fatalf("source results = %#v, want one source", report.SourceResults)
|
||||
}
|
||||
if got := report.SourceResults[0]; got.SourceCode != "g7s" || got.Records != 1 || got.Deduplicated != 1 || got.Resolved != 1 || got.WouldInsert != 1 {
|
||||
t.Fatalf("source result = %#v", got)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsReportsSourceResults(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤B00000").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("14400000000").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
{
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
Plate: "粤B99999",
|
||||
OEM: "G7s",
|
||||
},
|
||||
{
|
||||
SourceCode: "xinda",
|
||||
SourceName: "信达",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "14400000000",
|
||||
Plate: "粤B00000",
|
||||
OEM: "信达",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportMappingRecords() error = %v", err)
|
||||
}
|
||||
if report.Records != 3 || report.Deduplicated != 2 || report.Resolved != 1 || report.Unresolved != 1 || report.Conflicts != 1 || report.WouldInsert != 1 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
got := map[string]MappingSourceImportStat{}
|
||||
for _, source := range report.SourceResults {
|
||||
got[source.SourceCode] = source
|
||||
}
|
||||
if source := got["g7s"]; source.Records != 2 || source.Deduplicated != 1 || source.Resolved != 1 || source.Conflicts != 1 || source.WouldInsert != 1 {
|
||||
t.Fatalf("g7s source result = %#v", source)
|
||||
}
|
||||
if source := got["xinda"]; source.Records != 1 || source.Deduplicated != 1 || source.Unresolved != 1 {
|
||||
t.Fatalf("xinda source result = %#v", source)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsMergesDuplicateIdentifierDetails(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
File: "G7s/no-plate.xlsx",
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
OEM: "G7s",
|
||||
},
|
||||
{
|
||||
File: "G7s/with-plate.xlsx",
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportMappingRecords() error = %v", err)
|
||||
}
|
||||
if report.Records != 2 || report.Deduplicated != 1 || report.Resolved != 1 || report.WouldInsert != 1 || report.Unresolved != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsApplyCommitsSingleTransaction(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
mock.ExpectExec("INSERT INTO vehicle").
|
||||
WithArgs("LB9A32A22P0LS1230", "粤AG18312", "G7s").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("INSERT INTO vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425", "LB9A32A22P0LS1230", "粤AG18312", "G7s", "13307795425", "G7s/example.xlsx").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
File: "G7s/example.xlsx",
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
RawValue: "13307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{
|
||||
Apply: true,
|
||||
LegacyTable: "vehicle_identity_binding",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportMappingRecords() error = %v", err)
|
||||
}
|
||||
if report.Inserted != 1 || report.Resolved != 1 || report.WouldInsert != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsApplyRollsBackOnWriteError(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
errWrite := errors.New("insert vehicle failed")
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
mock.ExpectExec("INSERT INTO vehicle").
|
||||
WithArgs("LB9A32A22P0LS1230", "粤AG18312", "G7s").
|
||||
WillReturnError(errWrite)
|
||||
mock.ExpectRollback()
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
RawValue: "13307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{
|
||||
Apply: true,
|
||||
LegacyTable: "vehicle_identity_binding",
|
||||
})
|
||||
if !errors.Is(err, errWrite) {
|
||||
t.Fatalf("ImportMappingRecords() error = %v, want %v", err, errWrite)
|
||||
}
|
||||
if report.Inserted != 0 || report.Resolved != 1 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMappingPhoneHandlesExcelNumericFormats(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"013307795425": "13307795425",
|
||||
"13307795425.0": "13307795425",
|
||||
"1.3307795425E10": "13307795425",
|
||||
}
|
||||
for input, want := range tests {
|
||||
if got := normalizeMappingPhone(input); got != want {
|
||||
t.Fatalf("normalizeMappingPhone(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeWorkbook(t *testing.T, path string, rows [][]string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
workbook := excelize.NewFile()
|
||||
sheet := "Sheet1"
|
||||
for rowIndex, row := range rows {
|
||||
for columnIndex, value := range row {
|
||||
cellName, err := excelize.CoordinatesToCellName(columnIndex+1, rowIndex+1)
|
||||
if err != nil {
|
||||
t.Fatalf("CoordinatesToCellName() error = %v", err)
|
||||
}
|
||||
if err := workbook.SetCellValue(sheet, cellName, value); err != nil {
|
||||
t.Fatalf("SetCellValue() error = %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := workbook.SaveAs(path); err != nil {
|
||||
t.Fatalf("SaveAs() error = %v", err)
|
||||
}
|
||||
if err := workbook.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
}
|
||||
357
go/vehicle-gateway/internal/identity/registration_writer.go
Normal file
357
go/vehicle-gateway/internal/identity/registration_writer.go
Normal file
@@ -0,0 +1,357 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const (
|
||||
JT808RegisterMessageID = "0x0100"
|
||||
JT808AuthMessageID = "0x0102"
|
||||
JT808LocationMessageID = "0x0200"
|
||||
)
|
||||
|
||||
// JT808RegistrationFact is the durable identity projection carried by a raw
|
||||
// JT808 envelope. SeenAt uses gateway receive time so replay cannot move a
|
||||
// terminal to a future date because its device clock was wrong.
|
||||
type JT808RegistrationFact struct {
|
||||
Phone string
|
||||
DeviceID string
|
||||
Plate string
|
||||
VIN string
|
||||
Province string
|
||||
City string
|
||||
Manufacturer string
|
||||
DeviceType string
|
||||
PlateColor string
|
||||
AuthToken string
|
||||
AuthIMEI string
|
||||
AuthSoftwareVersion string
|
||||
SourceEndpoint string
|
||||
SourceIP string
|
||||
FirstRegisteredAt *time.Time
|
||||
LatestRegisteredAt *time.Time
|
||||
LatestAuthenticated *time.Time
|
||||
SeenAt time.Time
|
||||
}
|
||||
|
||||
// JT808RegistrationProjector throttles ordinary location touches in memory.
|
||||
// Registration and authentication frames are never throttled.
|
||||
type JT808RegistrationProjector struct {
|
||||
location *time.Location
|
||||
touchInterval time.Duration
|
||||
retention time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
lastTouches map[string]time.Time
|
||||
nextCleanup time.Time
|
||||
}
|
||||
|
||||
func NewJT808RegistrationProjector(location *time.Location, touchInterval time.Duration) *JT808RegistrationProjector {
|
||||
if location == nil {
|
||||
location = time.Local
|
||||
}
|
||||
if touchInterval <= 0 {
|
||||
touchInterval = 10 * time.Minute
|
||||
}
|
||||
return &JT808RegistrationProjector{
|
||||
location: location,
|
||||
touchInterval: touchInterval,
|
||||
retention: 24 * time.Hour,
|
||||
lastTouches: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
// ProjectBatch returns at most one merged fact per phone. Call MarkPersisted
|
||||
// only after the database transaction succeeds; otherwise replay must remain
|
||||
// eligible immediately.
|
||||
func (p *JT808RegistrationProjector) ProjectBatch(envelopes []envelope.FrameEnvelope) []JT808RegistrationFact {
|
||||
if p == nil || len(envelopes) == 0 {
|
||||
return nil
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
byPhone := make(map[string]JT808RegistrationFact)
|
||||
order := make([]string, 0, len(envelopes))
|
||||
for _, env := range envelopes {
|
||||
fact, ok := p.projectLocked(env)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if current, exists := byPhone[fact.Phone]; exists {
|
||||
byPhone[fact.Phone] = mergeJT808RegistrationFact(current, fact)
|
||||
continue
|
||||
}
|
||||
byPhone[fact.Phone] = fact
|
||||
order = append(order, fact.Phone)
|
||||
}
|
||||
result := make([]JT808RegistrationFact, 0, len(order))
|
||||
for _, phone := range order {
|
||||
result = append(result, byPhone[phone])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (p *JT808RegistrationProjector) MarkPersisted(facts []JT808RegistrationFact) {
|
||||
if p == nil || len(facts) == 0 {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for _, fact := range facts {
|
||||
phone := normalizePhone(fact.Phone)
|
||||
if phone == "" || fact.SeenAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if current := p.lastTouches[phone]; fact.SeenAt.After(current) {
|
||||
p.lastTouches[phone] = fact.SeenAt
|
||||
}
|
||||
}
|
||||
p.cleanupLocked(time.Now())
|
||||
}
|
||||
|
||||
func (p *JT808RegistrationProjector) projectLocked(env envelope.FrameEnvelope) (JT808RegistrationFact, bool) {
|
||||
if env.Protocol != envelope.ProtocolJT808 || env.ParseStatus == envelope.ParseBadFrame {
|
||||
return JT808RegistrationFact{}, false
|
||||
}
|
||||
messageID := strings.TrimSpace(env.MessageID)
|
||||
if messageID != JT808RegisterMessageID && messageID != JT808AuthMessageID && messageID != JT808LocationMessageID {
|
||||
return JT808RegistrationFact{}, false
|
||||
}
|
||||
phone := normalizePhone(env.Phone)
|
||||
if phone == "" {
|
||||
return JT808RegistrationFact{}, false
|
||||
}
|
||||
seenAt := p.receivedAt(env)
|
||||
if seenAt.IsZero() {
|
||||
return JT808RegistrationFact{}, false
|
||||
}
|
||||
if messageID == JT808LocationMessageID {
|
||||
if last := p.lastTouches[phone]; !last.IsZero() && seenAt.Before(last.Add(p.touchInterval)) {
|
||||
return JT808RegistrationFact{}, false
|
||||
}
|
||||
}
|
||||
|
||||
registration := mapValue(env.Parsed, "registration")
|
||||
authentication := mapValue(env.Parsed, "authentication")
|
||||
authenticationAccepted := messageID != JT808AuthMessageID ||
|
||||
!env.AuthenticationEnforced || env.AuthenticationStatus == "accepted"
|
||||
vin := strings.TrimSpace(env.VIN)
|
||||
if vin == "" {
|
||||
vin = "unknown"
|
||||
}
|
||||
fact := JT808RegistrationFact{
|
||||
Phone: phone,
|
||||
DeviceID: firstNonEmpty(env.DeviceID, textValue(registration, "device_id"), parsedFieldText(env.ParsedFields, "jt808.registration.device_id")),
|
||||
Plate: firstNonEmpty(env.Plate, textValue(registration, "plate"), parsedFieldText(env.ParsedFields, "jt808.registration.plate")),
|
||||
VIN: vin,
|
||||
Province: firstNonEmpty(textValue(registration, "province"), parsedFieldText(env.ParsedFields, "jt808.registration.province")),
|
||||
City: firstNonEmpty(textValue(registration, "city"), parsedFieldText(env.ParsedFields, "jt808.registration.city")),
|
||||
Manufacturer: firstNonEmpty(textValue(registration, "manufacturer"), parsedFieldText(env.ParsedFields, "jt808.registration.manufacturer")),
|
||||
DeviceType: firstNonEmpty(textValue(registration, "device_type"), parsedFieldText(env.ParsedFields, "jt808.registration.device_type")),
|
||||
PlateColor: firstNonEmpty(textValue(registration, "plate_color"), parsedFieldText(env.ParsedFields, "jt808.registration.plate_color")),
|
||||
AuthToken: firstNonEmpty(textValue(authentication, "token"), parsedFieldText(env.ParsedFields, "jt808.authentication.token")),
|
||||
AuthIMEI: firstNonEmpty(textValue(authentication, "imei"), parsedFieldText(env.ParsedFields, "jt808.authentication.imei")),
|
||||
AuthSoftwareVersion: firstNonEmpty(textValue(authentication, "software_version"), parsedFieldText(env.ParsedFields, "jt808.authentication.software_version")),
|
||||
SourceEndpoint: strings.TrimSpace(env.SourceEndpoint),
|
||||
SourceIP: normalizeEndpointIP(env.SourceEndpoint),
|
||||
SeenAt: seenAt,
|
||||
}
|
||||
if !authenticationAccepted {
|
||||
fact.AuthToken = ""
|
||||
fact.AuthIMEI = ""
|
||||
fact.AuthSoftwareVersion = ""
|
||||
}
|
||||
if messageID == JT808RegisterMessageID {
|
||||
fact.FirstRegisteredAt = timePointer(seenAt)
|
||||
fact.LatestRegisteredAt = timePointer(seenAt)
|
||||
}
|
||||
if messageID == JT808AuthMessageID && authenticationAccepted {
|
||||
fact.LatestAuthenticated = timePointer(seenAt)
|
||||
}
|
||||
return fact, true
|
||||
}
|
||||
|
||||
func parsedFieldText(fields map[string]any, key string) string {
|
||||
value, ok := fields[key]
|
||||
if !ok || value == nil {
|
||||
return ""
|
||||
}
|
||||
text := strings.TrimSpace(fmt.Sprint(value))
|
||||
if text == "<nil>" {
|
||||
return ""
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func (p *JT808RegistrationProjector) receivedAt(env envelope.FrameEnvelope) time.Time {
|
||||
milliseconds := env.ReceivedAtMS
|
||||
if milliseconds <= 0 {
|
||||
milliseconds = env.EventTimeMS
|
||||
}
|
||||
if milliseconds <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.UnixMilli(milliseconds).In(p.location).Truncate(time.Second)
|
||||
}
|
||||
|
||||
func (p *JT808RegistrationProjector) cleanupLocked(now time.Time) {
|
||||
if !p.nextCleanup.IsZero() && now.Before(p.nextCleanup) {
|
||||
return
|
||||
}
|
||||
p.nextCleanup = now.Add(time.Hour)
|
||||
cutoff := now.Add(-p.retention)
|
||||
for phone, touchedAt := range p.lastTouches {
|
||||
if touchedAt.Before(cutoff) {
|
||||
delete(p.lastTouches, phone)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mergeJT808RegistrationFact(current JT808RegistrationFact, candidate JT808RegistrationFact) JT808RegistrationFact {
|
||||
if candidate.SeenAt.After(current.SeenAt) || candidate.SeenAt.Equal(current.SeenAt) {
|
||||
current.DeviceID = firstNonEmpty(candidate.DeviceID, current.DeviceID)
|
||||
current.Plate = firstNonEmpty(candidate.Plate, current.Plate)
|
||||
current.VIN = preferKnownVIN(candidate.VIN, current.VIN)
|
||||
current.Province = firstNonEmpty(candidate.Province, current.Province)
|
||||
current.City = firstNonEmpty(candidate.City, current.City)
|
||||
current.Manufacturer = firstNonEmpty(candidate.Manufacturer, current.Manufacturer)
|
||||
current.DeviceType = firstNonEmpty(candidate.DeviceType, current.DeviceType)
|
||||
current.PlateColor = firstNonEmpty(candidate.PlateColor, current.PlateColor)
|
||||
current.AuthToken = firstNonEmpty(candidate.AuthToken, current.AuthToken)
|
||||
current.AuthIMEI = firstNonEmpty(candidate.AuthIMEI, current.AuthIMEI)
|
||||
current.AuthSoftwareVersion = firstNonEmpty(candidate.AuthSoftwareVersion, current.AuthSoftwareVersion)
|
||||
current.SourceEndpoint = firstNonEmpty(candidate.SourceEndpoint, current.SourceEndpoint)
|
||||
current.SourceIP = firstNonEmpty(candidate.SourceIP, current.SourceIP)
|
||||
current.SeenAt = candidate.SeenAt
|
||||
} else {
|
||||
current.DeviceID = firstNonEmpty(current.DeviceID, candidate.DeviceID)
|
||||
current.Plate = firstNonEmpty(current.Plate, candidate.Plate)
|
||||
current.VIN = preferKnownVIN(current.VIN, candidate.VIN)
|
||||
current.Province = firstNonEmpty(current.Province, candidate.Province)
|
||||
current.City = firstNonEmpty(current.City, candidate.City)
|
||||
current.Manufacturer = firstNonEmpty(current.Manufacturer, candidate.Manufacturer)
|
||||
current.DeviceType = firstNonEmpty(current.DeviceType, candidate.DeviceType)
|
||||
current.PlateColor = firstNonEmpty(current.PlateColor, candidate.PlateColor)
|
||||
current.AuthToken = firstNonEmpty(current.AuthToken, candidate.AuthToken)
|
||||
current.AuthIMEI = firstNonEmpty(current.AuthIMEI, candidate.AuthIMEI)
|
||||
current.AuthSoftwareVersion = firstNonEmpty(current.AuthSoftwareVersion, candidate.AuthSoftwareVersion)
|
||||
}
|
||||
current.FirstRegisteredAt = earlierTimePointer(current.FirstRegisteredAt, candidate.FirstRegisteredAt)
|
||||
current.LatestRegisteredAt = laterTimePointer(current.LatestRegisteredAt, candidate.LatestRegisteredAt)
|
||||
current.LatestAuthenticated = laterTimePointer(current.LatestAuthenticated, candidate.LatestAuthenticated)
|
||||
return current
|
||||
}
|
||||
|
||||
func timePointer(value time.Time) *time.Time {
|
||||
copy := value
|
||||
return ©
|
||||
}
|
||||
|
||||
func earlierTimePointer(left *time.Time, right *time.Time) *time.Time {
|
||||
if left == nil {
|
||||
return right
|
||||
}
|
||||
if right == nil || left.Before(*right) || left.Equal(*right) {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
|
||||
func laterTimePointer(left *time.Time, right *time.Time) *time.Time {
|
||||
if left == nil {
|
||||
return right
|
||||
}
|
||||
if right == nil || left.After(*right) || left.Equal(*right) {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
|
||||
type JT808RegistrationStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func NewJT808RegistrationStore(db *sql.DB) *JT808RegistrationStore {
|
||||
if db == nil {
|
||||
panic("jt808 registration db must not be nil")
|
||||
}
|
||||
return &JT808RegistrationStore{db: db}
|
||||
}
|
||||
|
||||
func EnsureJT808RegistrationSchema(ctx context.Context, db *sql.DB) error {
|
||||
if db == nil {
|
||||
return fmt.Errorf("jt808 registration db is nil")
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, jt808RegistrationTableSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, statement := range jt808RegistrationAlterSQL {
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil && !isIgnoredJT808RegistrationAlterError(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err := db.ExecContext(ctx, jt808RegistrationSourceIPBackfillSQL)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *JT808RegistrationStore) UpsertBatch(ctx context.Context, facts []JT808RegistrationFact) error {
|
||||
if s == nil || s.db == nil || len(facts) == 0 {
|
||||
return nil
|
||||
}
|
||||
const columns = `phone, device_id, plate, vin, province, city, manufacturer, device_type, plate_color,
|
||||
auth_token, auth_imei, auth_software_version, source_endpoint, source_ip,
|
||||
first_registered_at, latest_registered_at, latest_authenticated_at, latest_seen_at`
|
||||
values := make([]string, 0, len(facts))
|
||||
args := make([]any, 0, len(facts)*18)
|
||||
for _, fact := range facts {
|
||||
phone := normalizePhone(fact.Phone)
|
||||
if phone == "" || fact.SeenAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
vin := strings.TrimSpace(fact.VIN)
|
||||
if vin == "" {
|
||||
vin = "unknown"
|
||||
}
|
||||
values = append(values, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
args = append(args,
|
||||
phone, strings.TrimSpace(fact.DeviceID), strings.TrimSpace(fact.Plate), vin,
|
||||
strings.TrimSpace(fact.Province), strings.TrimSpace(fact.City), strings.TrimSpace(fact.Manufacturer),
|
||||
strings.TrimSpace(fact.DeviceType), strings.TrimSpace(fact.PlateColor), strings.TrimSpace(fact.AuthToken),
|
||||
strings.TrimSpace(fact.AuthIMEI), strings.TrimSpace(fact.AuthSoftwareVersion),
|
||||
strings.TrimSpace(fact.SourceEndpoint), strings.TrimSpace(fact.SourceIP),
|
||||
fact.FirstRegisteredAt, fact.LatestRegisteredAt, fact.LatestAuthenticated, fact.SeenAt,
|
||||
)
|
||||
}
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
query := `INSERT INTO jt808_registration (` + columns + `) VALUES ` + strings.Join(values, ",") + `
|
||||
ON DUPLICATE KEY UPDATE
|
||||
device_id = IF(VALUES(device_id) <> '' AND (device_id IS NULL OR device_id = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(device_id), device_id),
|
||||
plate = IF(VALUES(plate) <> '' AND (plate IS NULL OR plate = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(plate), plate),
|
||||
vin = IF(VALUES(vin) <> '' AND VALUES(vin) <> 'unknown' AND (vin IS NULL OR vin = '' OR vin = 'unknown' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(vin), vin),
|
||||
province = IF(VALUES(province) <> '' AND (province IS NULL OR province = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(province), province),
|
||||
city = IF(VALUES(city) <> '' AND (city IS NULL OR city = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(city), city),
|
||||
manufacturer = IF(VALUES(manufacturer) <> '' AND (manufacturer IS NULL OR manufacturer = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(manufacturer), manufacturer),
|
||||
device_type = IF(VALUES(device_type) <> '' AND (device_type IS NULL OR device_type = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(device_type), device_type),
|
||||
plate_color = IF(VALUES(plate_color) <> '' AND (plate_color IS NULL OR plate_color = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(plate_color), plate_color),
|
||||
auth_token = IF(VALUES(auth_token) <> '' AND (auth_token IS NULL OR auth_token = '' OR latest_authenticated_at IS NULL OR VALUES(latest_authenticated_at) >= latest_authenticated_at), VALUES(auth_token), auth_token),
|
||||
auth_imei = IF(VALUES(auth_imei) <> '' AND (auth_imei IS NULL OR auth_imei = '' OR latest_authenticated_at IS NULL OR VALUES(latest_authenticated_at) >= latest_authenticated_at), VALUES(auth_imei), auth_imei),
|
||||
auth_software_version = IF(VALUES(auth_software_version) <> '' AND (auth_software_version IS NULL OR auth_software_version = '' OR latest_authenticated_at IS NULL OR VALUES(latest_authenticated_at) >= latest_authenticated_at), VALUES(auth_software_version), auth_software_version),
|
||||
source_endpoint = IF(VALUES(source_endpoint) <> '' AND (source_endpoint IS NULL OR source_endpoint = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(source_endpoint), source_endpoint),
|
||||
source_ip = IF(VALUES(source_ip) <> '' AND (source_ip IS NULL OR source_ip = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(source_ip), source_ip),
|
||||
first_registered_at = CASE WHEN VALUES(first_registered_at) IS NULL THEN first_registered_at WHEN first_registered_at IS NULL THEN VALUES(first_registered_at) ELSE LEAST(first_registered_at, VALUES(first_registered_at)) END,
|
||||
latest_registered_at = CASE WHEN VALUES(latest_registered_at) IS NULL THEN latest_registered_at WHEN latest_registered_at IS NULL THEN VALUES(latest_registered_at) ELSE GREATEST(latest_registered_at, VALUES(latest_registered_at)) END,
|
||||
latest_authenticated_at = CASE WHEN VALUES(latest_authenticated_at) IS NULL THEN latest_authenticated_at WHEN latest_authenticated_at IS NULL THEN VALUES(latest_authenticated_at) ELSE GREATEST(latest_authenticated_at, VALUES(latest_authenticated_at)) END,
|
||||
latest_seen_at = GREATEST(latest_seen_at, VALUES(latest_seen_at))`
|
||||
_, err := s.db.ExecContext(ctx, query, args...)
|
||||
return err
|
||||
}
|
||||
221
go/vehicle-gateway/internal/identity/registration_writer_test.go
Normal file
221
go/vehicle-gateway/internal/identity/registration_writer_test.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestJT808RegistrationProjectorProjectsRegistrationUsingReceiveTime(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*60*60)
|
||||
projector := NewJT808RegistrationProjector(loc, 10*time.Minute)
|
||||
receivedAt := time.Date(2026, 7, 13, 17, 20, 30, 987000000, loc)
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: JT808RegisterMessageID,
|
||||
Phone: "0013307795425",
|
||||
VIN: "LTESTVIN000000001",
|
||||
DeviceID: "DEV-1",
|
||||
Plate: "粤A00001",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
ReceivedAtMS: receivedAt.UnixMilli(),
|
||||
EventTimeMS: receivedAt.Add(24 * time.Hour).UnixMilli(),
|
||||
ParseStatus: envelope.ParseOK,
|
||||
ParsedFields: map[string]any{
|
||||
"jt808.registration.province": "44",
|
||||
"jt808.registration.city": "1",
|
||||
"jt808.registration.manufacturer": "YUTNG",
|
||||
"jt808.registration.device_type": "TBOX-1",
|
||||
"jt808.registration.plate_color": "2",
|
||||
},
|
||||
}
|
||||
|
||||
facts := projector.ProjectBatch([]envelope.FrameEnvelope{env})
|
||||
if len(facts) != 1 {
|
||||
t.Fatalf("facts = %d, want 1", len(facts))
|
||||
}
|
||||
fact := facts[0]
|
||||
if fact.Phone != "13307795425" || fact.VIN != env.VIN || fact.Plate != env.Plate {
|
||||
t.Fatalf("identity fact = %+v", fact)
|
||||
}
|
||||
wantTime := receivedAt.Truncate(time.Second)
|
||||
if !fact.SeenAt.Equal(wantTime) || fact.FirstRegisteredAt == nil || !fact.FirstRegisteredAt.Equal(wantTime) {
|
||||
t.Fatalf("fact times = seen %s first %#v, want %s", fact.SeenAt, fact.FirstRegisteredAt, wantTime)
|
||||
}
|
||||
if fact.SourceIP != "115.231.168.135" || fact.Manufacturer != "YUTNG" || fact.PlateColor != "2" {
|
||||
t.Fatalf("registration details = %+v", fact)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJT808RegistrationProjectorThrottlesLocationOnlyAfterPersist(t *testing.T) {
|
||||
loc := time.UTC
|
||||
projector := NewJT808RegistrationProjector(loc, 10*time.Minute)
|
||||
base := time.Date(2026, 7, 13, 8, 0, 0, 0, loc)
|
||||
location := func(at time.Time) envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: JT808LocationMessageID,
|
||||
Phone: "013307795425",
|
||||
VIN: "LTESTVIN000000001",
|
||||
ReceivedAtMS: at.UnixMilli(),
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
}
|
||||
|
||||
first := projector.ProjectBatch([]envelope.FrameEnvelope{location(base)})
|
||||
if len(first) != 1 {
|
||||
t.Fatalf("first facts = %d, want 1", len(first))
|
||||
}
|
||||
// A failed database attempt must remain immediately replayable.
|
||||
if replay := projector.ProjectBatch([]envelope.FrameEnvelope{location(base)}); len(replay) != 1 {
|
||||
t.Fatalf("uncommitted replay facts = %d, want 1", len(replay))
|
||||
}
|
||||
projector.MarkPersisted(first)
|
||||
if throttled := projector.ProjectBatch([]envelope.FrameEnvelope{location(base.Add(9 * time.Minute))}); len(throttled) != 0 {
|
||||
t.Fatalf("throttled facts = %d, want 0", len(throttled))
|
||||
}
|
||||
if due := projector.ProjectBatch([]envelope.FrameEnvelope{location(base.Add(10 * time.Minute))}); len(due) != 1 {
|
||||
t.Fatalf("due facts = %d, want 1", len(due))
|
||||
}
|
||||
}
|
||||
|
||||
func TestJT808RegistrationProjectorMergesRegisterAndAuthForPhone(t *testing.T) {
|
||||
projector := NewJT808RegistrationProjector(time.UTC, 10*time.Minute)
|
||||
base := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC)
|
||||
register := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: JT808RegisterMessageID,
|
||||
Phone: "013307795425",
|
||||
VIN: "LTESTVIN000000001",
|
||||
Plate: "粤A00001",
|
||||
ReceivedAtMS: base.UnixMilli(),
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
auth := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: JT808AuthMessageID,
|
||||
Phone: "13307795425",
|
||||
ReceivedAtMS: base.Add(time.Second).UnixMilli(),
|
||||
ParseStatus: envelope.ParseOK,
|
||||
ParsedFields: map[string]any{
|
||||
"jt808.authentication.token": "g7gps",
|
||||
"jt808.authentication.imei": "123456789012345",
|
||||
},
|
||||
}
|
||||
|
||||
facts := projector.ProjectBatch([]envelope.FrameEnvelope{register, auth})
|
||||
if len(facts) != 1 {
|
||||
t.Fatalf("facts = %d, want 1", len(facts))
|
||||
}
|
||||
fact := facts[0]
|
||||
if fact.VIN != register.VIN || fact.Plate != register.Plate || fact.AuthToken != "g7gps" {
|
||||
t.Fatalf("merged fact = %+v", fact)
|
||||
}
|
||||
if fact.FirstRegisteredAt == nil || fact.LatestRegisteredAt == nil || fact.LatestAuthenticated == nil {
|
||||
t.Fatalf("merged timestamps missing: %+v", fact)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJT808RegistrationProjectorDoesNotTrustEnforcedRejectedAuth(t *testing.T) {
|
||||
projector := NewJT808RegistrationProjector(time.UTC, 10*time.Minute)
|
||||
seenAt := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC)
|
||||
facts := projector.ProjectBatch([]envelope.FrameEnvelope{{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: JT808AuthMessageID,
|
||||
Phone: "13307795425",
|
||||
ReceivedAtMS: seenAt.UnixMilli(),
|
||||
ParseStatus: envelope.ParseOK,
|
||||
AuthenticationEnforced: true,
|
||||
AuthenticationStatus: "rejected",
|
||||
Parsed: map[string]any{
|
||||
"authentication": map[string]any{
|
||||
"token": "untrusted-token",
|
||||
"imei": "untrusted-imei",
|
||||
"software_version": "untrusted-version",
|
||||
},
|
||||
},
|
||||
}})
|
||||
if len(facts) != 1 {
|
||||
t.Fatalf("facts = %d, want 1 audit touch", len(facts))
|
||||
}
|
||||
fact := facts[0]
|
||||
if fact.AuthToken != "" || fact.AuthIMEI != "" || fact.AuthSoftwareVersion != "" || fact.LatestAuthenticated != nil {
|
||||
t.Fatalf("rejected credential leaked into registration fact: %+v", fact)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJT808RegistrationStoreUsesIdempotentEventTimeUpsert(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
store := NewJT808RegistrationStore(db)
|
||||
seenAt := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC)
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO jt808_registration")).
|
||||
WithArgs(
|
||||
"13307795425", "DEV-1", "粤A00001", "LTESTVIN000000001", "", "", "YUTNG", "", "",
|
||||
"g7gps", "", "", "115.231.168.135:43625", "115.231.168.135",
|
||||
sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), seenAt,
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
err = store.UpsertBatch(context.Background(), []JT808RegistrationFact{{
|
||||
Phone: "013307795425",
|
||||
DeviceID: "DEV-1",
|
||||
Plate: "粤A00001",
|
||||
VIN: "LTESTVIN000000001",
|
||||
Manufacturer: "YUTNG",
|
||||
AuthToken: "g7gps",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
SourceIP: "115.231.168.135",
|
||||
FirstRegisteredAt: timePointer(seenAt),
|
||||
LatestRegisteredAt: timePointer(seenAt),
|
||||
LatestAuthenticated: timePointer(seenAt),
|
||||
SeenAt: seenAt,
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertBatch() error = %v", err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrationUpsertProtectsLatestValuesFromOldReplay(t *testing.T) {
|
||||
var query string
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherFunc(func(_ string, actual string) error {
|
||||
query = actual
|
||||
return nil
|
||||
})))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
store := NewJT808RegistrationStore(db)
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
if err := store.UpsertBatch(context.Background(), []JT808RegistrationFact{{
|
||||
Phone: "13307795425",
|
||||
VIN: "unknown",
|
||||
SeenAt: time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC),
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, required := range []string{
|
||||
"VALUES(latest_seen_at) >= latest_seen_at",
|
||||
"LEAST(first_registered_at, VALUES(first_registered_at))",
|
||||
"GREATEST(latest_authenticated_at, VALUES(latest_authenticated_at))",
|
||||
"latest_seen_at = GREATEST(latest_seen_at, VALUES(latest_seen_at))",
|
||||
} {
|
||||
if !regexp.MustCompile(regexp.QuoteMeta(required)).MatchString(query) {
|
||||
t.Fatalf("upsert SQL missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -57,9 +58,7 @@ func TestCandidateKeysNormalizesPhone(t *testing.T) {
|
||||
func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001"))
|
||||
expectVehicleIdentifierHit(mock, "JT808_PHONE", "13307795425", "LNBVIN00000000001")
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
@@ -74,7 +73,7 @@ func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "phone" {
|
||||
if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE" {
|
||||
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
@@ -82,9 +81,254 @@ func TestMySQLResolverFillsVINFromPhone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverUsesDataSourceCodeForJT808IdentifierLookup(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectSourceCodeHit(mock, "115.231.168.135", "xinda")
|
||||
expectVehicleIdentifierHitForSource(mock, "xinda", "JT808_PHONE", "13307795425", "LNBVIN00000000001")
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SourceCodeLookup: true,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNBVIN00000000001" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if env.SourceCode != "xinda" || env.PlatformName != "G7s" || env.SourceKind != "PLATFORM" {
|
||||
t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE.xinda" {
|
||||
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if identity["source_code"] != "xinda" || identity["platform_name"] != "G7s" || identity["source_kind"] != "PLATFORM" {
|
||||
t.Fatalf("identity source metadata = %#v", identity)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverVehicleIdentifierSourceOverridesDataSourceCode(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT source_code").
|
||||
WithArgs("JT808", "115.159.85.149").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"source_code", "platform_name", "source_kind"}).AddRow("dongfang_beidou", "G7易流", "PLATFORM"))
|
||||
expectVehicleIdentifierMissForSource(mock, "dongfang_beidou", "JT808_PHONE", "64341232682")
|
||||
expectVehicleIdentifierHitWithSource(mock, "JT808_PHONE", "64341232682", "LB9A32A23R0LS1045", "g7s", "G7s")
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SourceCodeLookup: true,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "064341232682",
|
||||
SourceEndpoint: "115.159.85.149:42823",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LB9A32A23R0LS1045" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if env.SourceCode != "g7s" || env.PlatformName != "G7s" || env.SourceKind != "PLATFORM" {
|
||||
t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE.g7s" {
|
||||
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if identity["source_code"] != "g7s" || identity["platform_name"] != "G7s" || identity["source_kind"] != "PLATFORM" {
|
||||
t.Fatalf("identity source metadata = %#v", identity)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverUsesStaleIdentifierCacheWhenLookupFails(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectVehicleIdentifierHit(mock, "JT808_PHONE", "13307795425", "LNBVIN00000000001")
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
LookupCacheTTL: time.Nanosecond,
|
||||
StaleLookupTTL: time.Hour,
|
||||
})
|
||||
first, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("first Resolve() error = %v", err)
|
||||
}
|
||||
if first.VIN != "LNBVIN00000000001" {
|
||||
t.Fatalf("first vin = %q", first.VIN)
|
||||
}
|
||||
|
||||
time.Sleep(time.Millisecond)
|
||||
mock.ExpectQuery("SELECT DISTINCT vin").
|
||||
WithArgs("JT808", "JT808_PHONE", "13307795425").
|
||||
WillReturnError(errors.New("mysql temporarily unavailable"))
|
||||
second, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("second Resolve() error = %v", err)
|
||||
}
|
||||
if second.VIN != "LNBVIN00000000001" {
|
||||
t.Fatalf("second vin = %q, want stale cached vin", second.VIN)
|
||||
}
|
||||
identity, ok := second.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE" {
|
||||
t.Fatalf("identity metadata = %#v", second.Parsed["identity"])
|
||||
}
|
||||
if identity["cache_status"] != "stale" {
|
||||
t.Fatalf("identity cache_status = %#v, want stale", identity["cache_status"])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeEndpointIPUsesSharedSourceEndpointKey(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"115.231.168.135:43625": "115.231.168.135",
|
||||
" 115.159.85.149:28316 ": "115.159.85.149",
|
||||
"mqtt://yutong/ytforward/shln/3": "mqtt",
|
||||
"MQTT://YUTONG/topic": "mqtt",
|
||||
}
|
||||
for input, want := range tests {
|
||||
if got := normalizeEndpointIP(input); got != want {
|
||||
t.Fatalf("normalizeEndpointIP(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverKeepsDirectSourceKindWithoutSourceCode(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT source_code").
|
||||
WithArgs("JT808", "39.144.3.22").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"source_code", "platform_name", "source_kind"}).AddRow("", "", "DIRECT"))
|
||||
expectVehicleIdentifierHit(mock, "JT808_PHONE", "13307765812", "LA9GG64L7PBAF4001")
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SourceCodeLookup: true,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307765812",
|
||||
SourceEndpoint: "39.144.3.22:60177",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LA9GG64L7PBAF4001" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if env.SourceCode != "" || env.SourceKind != "DIRECT" {
|
||||
t.Fatalf("source metadata = code:%q kind:%q", env.SourceCode, env.SourceKind)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source_kind"] != "DIRECT" {
|
||||
t.Fatalf("identity source metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverFallsBackToGlobalIdentifierWhenSourceCodeMisses(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectSourceCodeHit(mock, "115.231.168.135", "xinda")
|
||||
expectVehicleIdentifierMissForSource(mock, "xinda", "JT808_PHONE", "13307795425")
|
||||
expectVehicleIdentifierHit(mock, "JT808_PHONE", "13307795425", "LNBVIN00000000002")
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SourceCodeLookup: true,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNBVIN00000000002" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE" {
|
||||
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverUsesSingleGlobalIdentifierSourceCodeForJT808SourceMetadata(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectSourceCodeMiss(mock, "117.132.196.119")
|
||||
expectVehicleIdentifierHitWithSource(mock, "JT808_PHONE", "41456413943", "LNXNEGRR0SR321372", "xinda", "信达")
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SourceCodeLookup: true,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "41456413943",
|
||||
SourceEndpoint: "117.132.196.119:3275",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if env.VIN != "LNXNEGRR0SR321372" {
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
if env.SourceCode != "xinda" || env.PlatformName != "信达" || env.SourceKind != "PLATFORM" {
|
||||
t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind)
|
||||
}
|
||||
identity, ok := env.Parsed["identity"].(map[string]any)
|
||||
if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE.xinda" {
|
||||
t.Fatalf("identity metadata = %#v", env.Parsed["identity"])
|
||||
}
|
||||
if identity["source_code"] != "xinda" || identity["platform_name"] != "信达" || identity["source_kind"] != "PLATFORM" {
|
||||
t.Fatalf("identity source metadata = %#v", identity)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverLooksUpVINByUniqueKeyWithoutSort(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13307795425")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\? AND vin IS NOT NULL AND vin <> ''$").
|
||||
WithArgs("13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001"))
|
||||
@@ -109,6 +353,7 @@ func TestMySQLResolverLooksUpVINByUniqueKeyWithoutSort(t *testing.T) {
|
||||
func TestMySQLResolverDoesNotLookupBindingByDeviceID(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13307795425")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
@@ -133,9 +378,11 @@ func TestMySQLResolverDoesNotLookupBindingByDeviceID(t *testing.T) {
|
||||
func TestMySQLResolverTracksJT808RegistrationWithoutWritingBinding(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13079963379")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13079963379").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
expectVehicleIdentifierMiss(mock, "PLATE", "TEST123")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("TEST123").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LKLG7C4E3NA774736"))
|
||||
@@ -176,6 +423,7 @@ func TestMySQLResolverTracksJT808RegistrationWithoutWritingBinding(t *testing.T)
|
||||
func TestMySQLResolverTracksFirstJT808LocationThenThrottles(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13307795425")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
@@ -206,12 +454,14 @@ func TestMySQLResolverTracksFirstJT808LocationThenThrottles(t *testing.T) {
|
||||
func TestMySQLResolverUsesJT808RegistrationPlateForLocationVIN(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectVehicleIdentifierMiss(mock, "JT808_PHONE", "40692934322")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("40692934322").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin, device_id, plate FROM jt808_registration WHERE phone = \\?").
|
||||
WithArgs("40692934322").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "device_id", "plate"}).AddRow("unknown", "18285", "粤AG18285"))
|
||||
expectVehicleIdentifierMiss(mock, "PLATE", "粤AG18285")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18285").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNXNEGRR7SR318212"))
|
||||
@@ -248,9 +498,352 @@ func TestMySQLResolverUsesJT808RegistrationPlateForLocationVIN(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverRefreshesRegistrationCacheAfterRegisterFrame(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectVehicleIdentifierMiss(mock, "JT808_PHONE", "40692934322")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("40692934322").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectQuery("SELECT vin, device_id, plate FROM jt808_registration WHERE phone = \\?").
|
||||
WithArgs("40692934322").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
expectVehicleIdentifierMiss(mock, "PLATE", "粤AG18285")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18285").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNXNEGRR7SR318212"))
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
LocationTouchInterval: time.Hour,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
firstLocation, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "040692934322",
|
||||
SourceEndpoint: "115.231.168.135:47822",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("first location Resolve() error = %v", err)
|
||||
}
|
||||
if firstLocation.VIN != "" {
|
||||
t.Fatalf("first location vin = %q, want unresolved", firstLocation.VIN)
|
||||
}
|
||||
|
||||
registered, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0100",
|
||||
Phone: "040692934322",
|
||||
DeviceID: "18285",
|
||||
Plate: "粤AG18285",
|
||||
SourceEndpoint: "115.231.168.135:47822",
|
||||
Parsed: map[string]any{
|
||||
"registration": map[string]any{
|
||||
"device_id": "18285",
|
||||
"plate": "粤AG18285",
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("registration Resolve() error = %v", err)
|
||||
}
|
||||
if registered.VIN != "LNXNEGRR7SR318212" {
|
||||
t.Fatalf("registered vin = %q", registered.VIN)
|
||||
}
|
||||
|
||||
secondLocation, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "040692934322",
|
||||
SourceEndpoint: "115.231.168.135:47822",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("second location Resolve() error = %v", err)
|
||||
}
|
||||
if secondLocation.VIN != "LNXNEGRR7SR318212" {
|
||||
t.Fatalf("second location vin = %q, want cache-refreshed registration vin", secondLocation.VIN)
|
||||
}
|
||||
if secondLocation.DeviceID != "18285" || secondLocation.Plate != "粤AG18285" {
|
||||
t.Fatalf("second location identity not copied: device=%q plate=%q", secondLocation.DeviceID, secondLocation.Plate)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverDelegatesRegistrationPersistenceButKeepsLocalSession(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
var results []RegistrationWriteResult
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
DisableRegistrationWrites: true,
|
||||
OnRegistrationWriteResult: func(result RegistrationWriteResult) {
|
||||
results = append(results, result)
|
||||
},
|
||||
})
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: JT808RegisterMessageID,
|
||||
Phone: "013307795425",
|
||||
VIN: "LTESTVIN000000001",
|
||||
DeviceID: "DEV-1",
|
||||
Plate: "粤A00001",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
|
||||
resolved, err := resolver.Resolve(context.Background(), env)
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if resolved.VIN != env.VIN {
|
||||
t.Fatalf("resolved vin = %q, want %q", resolved.VIN, env.VIN)
|
||||
}
|
||||
entry, ok := resolver.registrationCache["13307795425"]
|
||||
if !ok || entry.vin != env.VIN || entry.plate != env.Plate || entry.deviceID != env.DeviceID {
|
||||
t.Fatalf("local session = %+v exists=%v", entry, ok)
|
||||
}
|
||||
if len(results) != 1 || results[0].Mode != "delegated" || results[0].Status != "ok" {
|
||||
t.Fatalf("registration write results = %#v", results)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("unexpected mysql access: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverRetriesRegistrationUpsertTransientFailure(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnError(errors.New("driver: bad connection: read tcp: connection reset by peer"))
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
LocationTouchInterval: time.Hour,
|
||||
RegistrationWriteAttempts: 2,
|
||||
RegistrationWriteRetryDelay: -1,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "040692934322",
|
||||
VIN: "LNXNEGRR7SR318212",
|
||||
SourceEndpoint: "115.231.168.135:47822",
|
||||
Parsed: map[string]any{},
|
||||
}
|
||||
if _, err := resolver.Resolve(context.Background(), env); err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want internal retry to recover", err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverAsyncRegistrationWritesDrainOnClose(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
results := make(chan RegistrationWriteResult, 2)
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
AsyncRegistrationWrites: true,
|
||||
RegistrationWriteQueueSize: 8,
|
||||
RegistrationWriteWorkers: 1,
|
||||
RegistrationWriteTimeout: time.Second,
|
||||
OnRegistrationWriteResult: func(result RegistrationWriteResult) {
|
||||
results <- result
|
||||
},
|
||||
LocationTouchInterval: time.Hour,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "040692934322",
|
||||
VIN: "LNXNEGRR7SR318212",
|
||||
SourceEndpoint: "115.231.168.135:47822",
|
||||
Parsed: map[string]any{},
|
||||
}
|
||||
if _, err := resolver.Resolve(context.Background(), env); err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if err := resolver.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
close(results)
|
||||
got := map[string]int{}
|
||||
for result := range results {
|
||||
got[result.Mode+":"+result.Status]++
|
||||
}
|
||||
if got["async_enqueue:ok"] != 1 || got["async_background:ok"] != 1 {
|
||||
t.Fatalf("registration write results = %#v, want enqueue/background ok", got)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverAsyncRegistrationWriteFailureMarksLocationRetry(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnError(errors.New("driver: bad connection"))
|
||||
errs := make(chan error, 1)
|
||||
results := make(chan RegistrationWriteResult, 2)
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
AsyncRegistrationWrites: true,
|
||||
RegistrationWriteQueueSize: 8,
|
||||
RegistrationWriteWorkers: 1,
|
||||
RegistrationWriteTimeout: time.Second,
|
||||
OnRegistrationWriteResult: func(result RegistrationWriteResult) {
|
||||
results <- result
|
||||
},
|
||||
OnRegistrationWriteError: func(err error) {
|
||||
errs <- err
|
||||
},
|
||||
LocationTouchInterval: time.Hour,
|
||||
LocationTouchRetryInterval: time.Hour,
|
||||
RegistrationWriteAttempts: 1,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "040692934322",
|
||||
VIN: "LNXNEGRR7SR318212",
|
||||
SourceEndpoint: "115.231.168.135:47822",
|
||||
Parsed: map[string]any{},
|
||||
}
|
||||
if _, err := resolver.Resolve(context.Background(), env); err != nil {
|
||||
t.Fatalf("Resolve() error = %v, async write failure should be reported out-of-band", err)
|
||||
}
|
||||
if err := resolver.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-errs:
|
||||
if err == nil {
|
||||
t.Fatal("async error callback received nil")
|
||||
}
|
||||
default:
|
||||
t.Fatal("async write failure was not reported")
|
||||
}
|
||||
stats := resolver.CacheStats()
|
||||
if stats.LocationTouchFailureEntries != 1 {
|
||||
t.Fatalf("location touch failure cache entries = %d, want 1", stats.LocationTouchFailureEntries)
|
||||
}
|
||||
close(results)
|
||||
got := map[string]int{}
|
||||
for result := range results {
|
||||
got[result.Mode+":"+result.Status]++
|
||||
}
|
||||
if got["async_enqueue:ok"] != 1 || got["async_background:error"] != 1 {
|
||||
t.Fatalf("registration write results = %#v, want enqueue ok/background error", got)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverBacksOffLocationTouchAfterExhaustedUpsert(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectExec("INSERT INTO jt808_registration").
|
||||
WillReturnError(errors.New("driver: bad connection"))
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
LocationTouchInterval: time.Hour,
|
||||
LocationTouchRetryInterval: time.Hour,
|
||||
RegistrationWriteAttempts: 1,
|
||||
LookupCacheTTL: time.Hour,
|
||||
})
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Phone: "040692934322",
|
||||
VIN: "LNXNEGRR7SR318212",
|
||||
SourceEndpoint: "115.231.168.135:47822",
|
||||
Parsed: map[string]any{},
|
||||
}
|
||||
if _, err := resolver.Resolve(context.Background(), env); err == nil {
|
||||
t.Fatal("first Resolve() error = nil, want exhausted transient registration upsert failure")
|
||||
}
|
||||
stats := resolver.CacheStats()
|
||||
if stats.LocationTouchFailureEntries != 1 {
|
||||
t.Fatalf("location touch failure cache entries = %d, want 1", stats.LocationTouchFailureEntries)
|
||||
}
|
||||
if _, err := resolver.Resolve(context.Background(), env); err != nil {
|
||||
t.Fatalf("second Resolve() error = %v, want short backoff to skip immediate retry", err)
|
||||
}
|
||||
stats = resolver.CacheStats()
|
||||
if stats.LocationTouchFailureEntries != 1 {
|
||||
t.Fatalf("location touch failure cache entries after backoff = %d, want 1", stats.LocationTouchFailureEntries)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTransientMySQLIdentityWriteError(t *testing.T) {
|
||||
for _, err := range []error{
|
||||
errors.New("dial tcp 127.0.0.1:3306: connection refused"),
|
||||
errors.New("read tcp: connection reset by peer"),
|
||||
errors.New("write tcp: broken pipe"),
|
||||
errors.New("driver: bad connection"),
|
||||
errors.New("invalid connection"),
|
||||
errors.New("i/o timeout"),
|
||||
errors.New("EOF"),
|
||||
errors.New("server is down"),
|
||||
errors.New("network is unreachable"),
|
||||
} {
|
||||
if !isTransientMySQLIdentityWriteError(err) {
|
||||
t.Fatalf("isTransientMySQLIdentityWriteError(%q) = false, want true", err.Error())
|
||||
}
|
||||
}
|
||||
for _, err := range []error{
|
||||
context.Canceled,
|
||||
context.DeadlineExceeded,
|
||||
errors.New("duplicate key conflict"),
|
||||
nil,
|
||||
} {
|
||||
if isTransientMySQLIdentityWriteError(err) {
|
||||
t.Fatalf("isTransientMySQLIdentityWriteError(%v) = true, want false", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutResolverAppliesDeadlineToDelegate(t *testing.T) {
|
||||
delegate := &deadlineCheckingResolver{}
|
||||
resolver := TimeoutResolver{Delegate: delegate, Timeout: 50 * time.Millisecond}
|
||||
|
||||
if _, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}); err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if delegate.deadline.IsZero() {
|
||||
t.Fatal("delegate did not receive a deadline")
|
||||
}
|
||||
if remaining := time.Until(delegate.deadline); remaining <= 0 || remaining > time.Second {
|
||||
t.Fatalf("deadline remaining = %s", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverCachesIdentityMissesForHighFrequencyFrames(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13307795425")
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("13307795425").
|
||||
WillReturnError(sql.ErrNoRows)
|
||||
@@ -274,6 +867,95 @@ func TestMySQLResolverCachesIdentityMissesForHighFrequencyFrames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverBoundsIdentityLookupCaches(t *testing.T) {
|
||||
db, _ := newMockDB(t)
|
||||
defer db.Close()
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
LookupCacheTTL: time.Hour,
|
||||
CacheCleanupInterval: time.Hour,
|
||||
MaxCacheEntries: 2,
|
||||
LocationTouchInterval: time.Hour,
|
||||
})
|
||||
now := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC)
|
||||
for i, key := range []string{"lookup-1", "lookup-2", "lookup-3"} {
|
||||
entryNow := now.Add(time.Duration(i) * time.Second)
|
||||
resolver.cacheLookup(key, lookupCacheEntry{
|
||||
vin: key,
|
||||
expiresAt: entryNow.Add(time.Hour),
|
||||
}, entryNow)
|
||||
resolver.cacheRegistration("phone-"+key, registrationCacheEntry{
|
||||
vin: key,
|
||||
expiresAt: entryNow.Add(time.Hour),
|
||||
}, entryNow)
|
||||
resolver.cacheSourceMetadata("source-"+key, sourceMetadata{SourceCode: key}, false, entryNow)
|
||||
}
|
||||
|
||||
stats := resolver.CacheStats()
|
||||
if stats.LookupEntries != 2 || stats.RegistrationEntries != 2 || stats.SourceCodeEntries != 2 || stats.MaxEntries != 2 {
|
||||
t.Fatalf("cache stats = %+v, want all identity caches capped at 2", stats)
|
||||
}
|
||||
resolver.lookupMu.Lock()
|
||||
_, hasOldLookup := resolver.lookupCache["lookup-1"]
|
||||
_, hasOldRegistration := resolver.registrationCache["phone-lookup-1"]
|
||||
_, hasOldSource := resolver.sourceCodeCache["source-lookup-1"]
|
||||
resolver.lookupMu.Unlock()
|
||||
if hasOldLookup || hasOldRegistration || hasOldSource {
|
||||
t.Fatalf("oldest cache entries should be evicted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLResolverBoundsLocationTouchCache(t *testing.T) {
|
||||
db, _ := newMockDB(t)
|
||||
defer db.Close()
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
LookupCacheTTL: time.Hour,
|
||||
CacheCleanupInterval: time.Hour,
|
||||
MaxCacheEntries: 2,
|
||||
LocationTouchInterval: time.Hour,
|
||||
})
|
||||
now := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC)
|
||||
resolver.touchMu.Lock()
|
||||
resolver.locationTouches["old-expired"] = now.Add(-2 * time.Hour)
|
||||
resolver.locationTouches["phone-1"] = now.Add(-2 * time.Minute)
|
||||
resolver.locationTouches["phone-2"] = now.Add(-time.Minute)
|
||||
resolver.locationTouches["phone-3"] = now
|
||||
resolver.locationTouchFailures["old-failure"] = now.Add(-time.Minute)
|
||||
resolver.locationTouchFailures["phone-2"] = now.Add(time.Minute)
|
||||
resolver.locationTouchFailures["phone-3"] = now.Add(2 * time.Minute)
|
||||
resolver.locationTouchFailures["phone-4"] = now.Add(3 * time.Minute)
|
||||
resolver.cleanupLocationTouchesLocked(now, false)
|
||||
resolver.touchMu.Unlock()
|
||||
|
||||
stats := resolver.CacheStats()
|
||||
if stats.LocationTouchEntries != 2 || stats.LocationTouchFailureEntries != 2 || stats.MaxEntries != 2 {
|
||||
t.Fatalf("cache stats = %+v, want location touch caches capped at 2", stats)
|
||||
}
|
||||
resolver.touchMu.Lock()
|
||||
_, hasExpired := resolver.locationTouches["old-expired"]
|
||||
_, hasOldest := resolver.locationTouches["phone-1"]
|
||||
_, hasExpiredFailure := resolver.locationTouchFailures["old-failure"]
|
||||
_, hasOldestFailure := resolver.locationTouchFailures["phone-2"]
|
||||
resolver.touchMu.Unlock()
|
||||
if hasExpired || hasOldest || hasExpiredFailure || hasOldestFailure {
|
||||
t.Fatalf("expired and oldest location touch entries should be evicted")
|
||||
}
|
||||
}
|
||||
|
||||
type deadlineCheckingResolver struct {
|
||||
mu sync.Mutex
|
||||
deadline time.Time
|
||||
}
|
||||
|
||||
func (r *deadlineCheckingResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) {
|
||||
deadline, _ := ctx.Deadline()
|
||||
r.mu.Lock()
|
||||
r.deadline = deadline
|
||||
r.mu.Unlock()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func TestMySQLResolverEnsuresMinimalIdentitySchema(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
@@ -285,8 +967,18 @@ func TestMySQLResolverEnsuresMinimalIdentitySchema(t *testing.T) {
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("ALTER TABLE vehicle_identity_binding DROP COLUMN device_id").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle \\(").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle_identifier").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("CREATE TABLE IF NOT EXISTS jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("ALTER TABLE jt808_registration ADD COLUMN source_ip").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("ALTER TABLE jt808_registration ADD KEY idx_jt808_registration_source_ip").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("UPDATE jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
if err := resolver.EnsureSchema(context.Background()); err != nil {
|
||||
@@ -308,8 +1000,18 @@ func TestMySQLResolverIgnoresExistingOEMColumn(t *testing.T) {
|
||||
WillReturnError(errors.New("Error 1091 (42000): Can't DROP 'uk_identity_device'; check that column/key exists"))
|
||||
mock.ExpectExec("ALTER TABLE vehicle_identity_binding DROP COLUMN device_id").
|
||||
WillReturnError(errors.New("Error 1091 (42000): Can't DROP 'device_id'; check that column/key exists"))
|
||||
mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle \\(").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle_identifier").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("CREATE TABLE IF NOT EXISTS jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("ALTER TABLE jt808_registration ADD COLUMN source_ip").
|
||||
WillReturnError(errors.New("Error 1060 (42S21): Duplicate column name 'source_ip'"))
|
||||
mock.ExpectExec("ALTER TABLE jt808_registration ADD KEY idx_jt808_registration_source_ip").
|
||||
WillReturnError(errors.New("Error 1061 (42000): Duplicate key name 'idx_jt808_registration_source_ip'"))
|
||||
mock.ExpectExec("UPDATE jt808_registration").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
|
||||
resolver := NewMySQLResolver(db, "vehicle_identity_binding")
|
||||
if err := resolver.EnsureSchema(context.Background()); err != nil {
|
||||
@@ -339,6 +1041,15 @@ func TestIdentitySchemaUsesBusinessKeysOnly(t *testing.T) {
|
||||
if !strings.Contains(registration, "phone VARCHAR(32) PRIMARY KEY") {
|
||||
t.Fatalf("registration table should key by phone:\n%s", registration)
|
||||
}
|
||||
if !strings.Contains(registration, "source_ip VARCHAR(64)") || !strings.Contains(registration, "idx_jt808_registration_source_ip") {
|
||||
t.Fatalf("registration table should keep indexed source_ip:\n%s", registration)
|
||||
}
|
||||
if !strings.Contains(vehicleIdentifierTableSQL, "PRIMARY KEY (protocol, source_code, identifier_type, identifier_value)") {
|
||||
t.Fatalf("vehicle identifier should use protocol/source/type/value as key:\n%s", vehicleIdentifierTableSQL)
|
||||
}
|
||||
if strings.Contains(vehicleIdentifierTableSQL, "AUTO_INCREMENT") {
|
||||
t.Fatalf("vehicle identifier should not use surrogate auto increment id:\n%s", vehicleIdentifierTableSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) {
|
||||
@@ -349,3 +1060,43 @@ func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) {
|
||||
}
|
||||
return db, mock
|
||||
}
|
||||
|
||||
func expectVehicleIdentifierMiss(mock sqlmock.Sqlmock, identifierType string, value string) {
|
||||
mock.ExpectQuery("SELECT DISTINCT vin").
|
||||
WithArgs("JT808", identifierType, value).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "source_code", "platform_name"}))
|
||||
}
|
||||
|
||||
func expectVehicleIdentifierHit(mock sqlmock.Sqlmock, identifierType string, value string, vin string) {
|
||||
expectVehicleIdentifierHitWithSource(mock, identifierType, value, vin, "", "")
|
||||
}
|
||||
|
||||
func expectVehicleIdentifierHitWithSource(mock sqlmock.Sqlmock, identifierType string, value string, vin string, sourceCode string, platformName string) {
|
||||
mock.ExpectQuery("SELECT DISTINCT vin").
|
||||
WithArgs("JT808", identifierType, value).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "source_code", "platform_name"}).AddRow(vin, sourceCode, platformName))
|
||||
}
|
||||
|
||||
func expectSourceCodeHit(mock sqlmock.Sqlmock, sourceIP string, sourceCode string) {
|
||||
mock.ExpectQuery("SELECT source_code").
|
||||
WithArgs("JT808", sourceIP).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"source_code", "platform_name", "source_kind"}).AddRow(sourceCode, "G7s", "PLATFORM"))
|
||||
}
|
||||
|
||||
func expectSourceCodeMiss(mock sqlmock.Sqlmock, sourceIP string) {
|
||||
mock.ExpectQuery("SELECT source_code").
|
||||
WithArgs("JT808", sourceIP).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"source_code", "platform_name", "source_kind"}))
|
||||
}
|
||||
|
||||
func expectVehicleIdentifierMissForSource(mock sqlmock.Sqlmock, sourceCode string, identifierType string, value string) {
|
||||
mock.ExpectQuery("SELECT DISTINCT vin").
|
||||
WithArgs("JT808", identifierType, value, sourceCode).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "source_code", "platform_name"}))
|
||||
}
|
||||
|
||||
func expectVehicleIdentifierHitForSource(mock sqlmock.Sqlmock, sourceCode string, identifierType string, value string, vin string) {
|
||||
mock.ExpectQuery("SELECT DISTINCT vin").
|
||||
WithArgs("JT808", identifierType, value, sourceCode).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "source_code", "platform_name"}).AddRow(vin, sourceCode, "G7s"))
|
||||
}
|
||||
|
||||
325
go/vehicle-gateway/internal/identity/snapshot.go
Normal file
325
go/vehicle-gateway/internal/identity/snapshot.go
Normal file
@@ -0,0 +1,325 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SnapshotRefreshResult struct {
|
||||
BindingEntries int
|
||||
IdentifierEntries int
|
||||
RegistrationEntries int
|
||||
SourceEntries int
|
||||
RefreshedAt time.Time
|
||||
}
|
||||
|
||||
// identitySnapshot is immutable after atomic publication, so frame handling
|
||||
// performs only local map lookups and never waits for MySQL or a refresh lock.
|
||||
type identitySnapshot struct {
|
||||
bindings map[string]string
|
||||
identifiers map[string]vehicleIdentifierMatch
|
||||
registrations map[string]registrationCacheEntry
|
||||
sources map[string]sourceMetadata
|
||||
refreshedAt time.Time
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) RefreshSnapshot(ctx context.Context) (SnapshotRefreshResult, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return SnapshotRefreshResult{}, fmt.Errorf("identity snapshot database is not configured")
|
||||
}
|
||||
r.snapshotRefreshMu.Lock()
|
||||
defer r.snapshotRefreshMu.Unlock()
|
||||
|
||||
next := &identitySnapshot{
|
||||
bindings: map[string]string{},
|
||||
identifiers: map[string]vehicleIdentifierMatch{},
|
||||
registrations: map[string]registrationCacheEntry{},
|
||||
sources: map[string]sourceMetadata{},
|
||||
}
|
||||
if err := r.loadBindingSnapshot(ctx, next); err != nil {
|
||||
return SnapshotRefreshResult{}, err
|
||||
}
|
||||
if err := r.loadIdentifierSnapshot(ctx, next); err != nil {
|
||||
return SnapshotRefreshResult{}, err
|
||||
}
|
||||
if err := r.loadRegistrationSnapshot(ctx, next); err != nil {
|
||||
return SnapshotRefreshResult{}, err
|
||||
}
|
||||
if err := r.loadSourceSnapshot(ctx, next); err != nil {
|
||||
return SnapshotRefreshResult{}, err
|
||||
}
|
||||
next.refreshedAt = time.Now()
|
||||
|
||||
r.snapshot.Store(next)
|
||||
return SnapshotRefreshResult{
|
||||
BindingEntries: len(next.bindings),
|
||||
IdentifierEntries: len(next.identifiers),
|
||||
RegistrationEntries: len(next.registrations),
|
||||
SourceEntries: len(next.sources),
|
||||
RefreshedAt: next.refreshedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) loadBindingSnapshot(ctx context.Context, target *identitySnapshot) error {
|
||||
rows, err := r.db.QueryContext(ctx, "SELECT vin, plate, phone FROM "+r.table+" WHERE vin IS NOT NULL AND TRIM(vin) <> ''")
|
||||
if err != nil {
|
||||
return fmt.Errorf("load identity binding snapshot: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
ambiguous := map[string]struct{}{}
|
||||
for rows.Next() {
|
||||
var vin, plate, phone sql.NullString
|
||||
if err := rows.Scan(&vin, &plate, &phone); err != nil {
|
||||
return fmt.Errorf("scan identity binding snapshot: %w", err)
|
||||
}
|
||||
vinValue := strings.TrimSpace(vin.String)
|
||||
if vinValue == "" {
|
||||
continue
|
||||
}
|
||||
addSnapshotBinding(target.bindings, ambiguous, bindingSnapshotKey("vin", vinValue), vinValue)
|
||||
if value := strings.TrimSpace(plate.String); value != "" {
|
||||
addSnapshotBinding(target.bindings, ambiguous, bindingSnapshotKey("plate", value), vinValue)
|
||||
}
|
||||
if value := normalizePhone(phone.String); value != "" {
|
||||
addSnapshotBinding(target.bindings, ambiguous, bindingSnapshotKey("phone", value), vinValue)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("iterate identity binding snapshot: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) loadIdentifierSnapshot(ctx context.Context, target *identitySnapshot) error {
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT protocol, source_code, identifier_type, identifier_value,
|
||||
vin, COALESCE(NULLIF(TRIM(oem), ''), source_code) AS platform_name
|
||||
FROM vehicle_identifier
|
||||
WHERE enabled = 1 AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
AND identifier_value IS NOT NULL AND TRIM(identifier_value) <> ''`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load vehicle identifier snapshot: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
ambiguous := map[string]struct{}{}
|
||||
for rows.Next() {
|
||||
var protocol, sourceCode, identifierType, identifierValue, vin, platformName sql.NullString
|
||||
if err := rows.Scan(&protocol, &sourceCode, &identifierType, &identifierValue, &vin, &platformName); err != nil {
|
||||
return fmt.Errorf("scan vehicle identifier snapshot: %w", err)
|
||||
}
|
||||
protocolValue := strings.TrimSpace(protocol.String)
|
||||
typeValue := strings.ToUpper(strings.TrimSpace(identifierType.String))
|
||||
value := normalizeIdentifierValue(typeValue, identifierValue.String)
|
||||
vinValue := strings.TrimSpace(vin.String)
|
||||
if protocolValue == "" || typeValue == "" || value == "" || vinValue == "" {
|
||||
continue
|
||||
}
|
||||
match := vehicleIdentifierMatch{
|
||||
VIN: vinValue,
|
||||
SourceCode: strings.TrimSpace(sourceCode.String),
|
||||
PlatformName: strings.TrimSpace(platformName.String),
|
||||
}
|
||||
scopedKey := vehicleIdentifierSnapshotKey(protocolValue, match.SourceCode, typeValue, value)
|
||||
addSnapshotIdentifier(target.identifiers, ambiguous, scopedKey, match)
|
||||
globalKey := vehicleIdentifierSnapshotKey(protocolValue, "", typeValue, value)
|
||||
addSnapshotIdentifier(target.identifiers, ambiguous, globalKey, match)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("iterate vehicle identifier snapshot: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) loadRegistrationSnapshot(ctx context.Context, target *identitySnapshot) error {
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT phone, vin, device_id, plate, auth_token
|
||||
FROM jt808_registration
|
||||
WHERE phone IS NOT NULL AND TRIM(phone) <> ''`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load jt808 registration snapshot: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var phone, vin, deviceID, plate, authToken sql.NullString
|
||||
if err := rows.Scan(&phone, &vin, &deviceID, &plate, &authToken); err != nil {
|
||||
return fmt.Errorf("scan jt808 registration snapshot: %w", err)
|
||||
}
|
||||
phoneValue := normalizePhone(phone.String)
|
||||
if phoneValue == "" {
|
||||
continue
|
||||
}
|
||||
target.registrations[phoneValue] = registrationCacheEntry{
|
||||
vin: strings.TrimSpace(vin.String),
|
||||
deviceID: strings.TrimSpace(deviceID.String),
|
||||
plate: strings.TrimSpace(plate.String),
|
||||
authToken: strings.TrimSpace(authToken.String),
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("iterate jt808 registration snapshot: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) loadSourceSnapshot(ctx context.Context, target *identitySnapshot) error {
|
||||
rows, err := r.db.QueryContext(ctx, `SELECT protocol, source_ip, source_code, platform_name, source_kind
|
||||
FROM vehicle_data_source
|
||||
WHERE enabled = 1 AND source_ip IS NOT NULL AND TRIM(source_ip) <> ''
|
||||
AND (
|
||||
(source_code IS NOT NULL AND TRIM(source_code) <> '')
|
||||
OR source_kind IN ('PLATFORM', 'DIRECT')
|
||||
OR (platform_name IS NOT NULL AND TRIM(platform_name) <> '')
|
||||
)`)
|
||||
if err != nil {
|
||||
if isOptionalSourceCodeLookupError(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("load vehicle data source snapshot: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var protocol, sourceIP, sourceCode, platformName, sourceKind sql.NullString
|
||||
if err := rows.Scan(&protocol, &sourceIP, &sourceCode, &platformName, &sourceKind); err != nil {
|
||||
return fmt.Errorf("scan vehicle data source snapshot: %w", err)
|
||||
}
|
||||
key := sourceSnapshotKey(protocol.String, sourceIP.String)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
target.sources[key] = sourceMetadata{
|
||||
SourceCode: strings.TrimSpace(sourceCode.String),
|
||||
PlatformName: strings.TrimSpace(platformName.String),
|
||||
SourceKind: strings.TrimSpace(sourceKind.String),
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("iterate vehicle data source snapshot: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addSnapshotBinding(values map[string]string, ambiguous map[string]struct{}, key string, vin string) {
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
if _, exists := ambiguous[key]; exists {
|
||||
return
|
||||
}
|
||||
if current, exists := values[key]; exists && !strings.EqualFold(current, vin) {
|
||||
delete(values, key)
|
||||
ambiguous[key] = struct{}{}
|
||||
return
|
||||
}
|
||||
values[key] = vin
|
||||
}
|
||||
|
||||
func addSnapshotIdentifier(values map[string]vehicleIdentifierMatch, ambiguous map[string]struct{}, key string, match vehicleIdentifierMatch) {
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
if _, exists := ambiguous[key]; exists {
|
||||
return
|
||||
}
|
||||
current, exists := values[key]
|
||||
if !exists {
|
||||
values[key] = match
|
||||
return
|
||||
}
|
||||
if !strings.EqualFold(current.VIN, match.VIN) {
|
||||
delete(values, key)
|
||||
ambiguous[key] = struct{}{}
|
||||
return
|
||||
}
|
||||
if !strings.EqualFold(current.SourceCode, match.SourceCode) {
|
||||
current.SourceCode = ""
|
||||
current.PlatformName = ""
|
||||
} else if current.PlatformName != match.PlatformName {
|
||||
current.PlatformName = ""
|
||||
}
|
||||
values[key] = current
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) snapshotBinding(column string, value string) (string, bool) {
|
||||
key := bindingSnapshotKey(column, value)
|
||||
if key == "" {
|
||||
return "", false
|
||||
}
|
||||
snapshot := r.snapshot.Load()
|
||||
if snapshot == nil {
|
||||
return "", false
|
||||
}
|
||||
vin, ok := snapshot.bindings[key]
|
||||
return vin, ok
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) snapshotIdentifier(protocolValue string, sourceCode string, identifierType string, value string) (vehicleIdentifierMatch, bool) {
|
||||
key := vehicleIdentifierSnapshotKey(protocolValue, sourceCode, identifierType, value)
|
||||
snapshot := r.snapshot.Load()
|
||||
if snapshot == nil {
|
||||
return vehicleIdentifierMatch{}, false
|
||||
}
|
||||
match, ok := snapshot.identifiers[key]
|
||||
return match, ok
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) snapshotRegistration(phone string) (registrationCacheEntry, bool) {
|
||||
phone = normalizePhone(phone)
|
||||
snapshot := r.snapshot.Load()
|
||||
if snapshot == nil {
|
||||
return registrationCacheEntry{}, false
|
||||
}
|
||||
entry, ok := snapshot.registrations[phone]
|
||||
return entry, ok
|
||||
}
|
||||
|
||||
func (r *MySQLResolver) snapshotSource(protocolValue string, sourceIP string) (sourceMetadata, bool) {
|
||||
key := sourceSnapshotKey(protocolValue, sourceIP)
|
||||
snapshot := r.snapshot.Load()
|
||||
if snapshot == nil {
|
||||
return sourceMetadata{}, false
|
||||
}
|
||||
metadata, ok := snapshot.sources[key]
|
||||
return metadata, ok
|
||||
}
|
||||
|
||||
// JT808AuthToken serves authentication from the same immutable snapshot used
|
||||
// by identity resolution. It deliberately never falls back to a per-frame SQL
|
||||
// query because authentication is on the protocol response hot path.
|
||||
func (r *MySQLResolver) JT808AuthToken(phone string) (string, bool) {
|
||||
entry, ok := r.snapshotRegistration(phone)
|
||||
token := strings.TrimSpace(entry.authToken)
|
||||
return token, ok && token != ""
|
||||
}
|
||||
|
||||
func bindingSnapshotKey(column string, value string) string {
|
||||
column = strings.ToLower(strings.TrimSpace(column))
|
||||
value = strings.TrimSpace(value)
|
||||
if column == "phone" {
|
||||
value = normalizePhone(value)
|
||||
}
|
||||
if value == "" || (column != "vin" && column != "plate" && column != "phone") {
|
||||
return ""
|
||||
}
|
||||
return column + "\x00" + value
|
||||
}
|
||||
|
||||
func vehicleIdentifierSnapshotKey(protocolValue string, sourceCode string, identifierType string, value string) string {
|
||||
protocolValue = strings.TrimSpace(protocolValue)
|
||||
sourceCode = strings.TrimSpace(sourceCode)
|
||||
identifierType = strings.ToUpper(strings.TrimSpace(identifierType))
|
||||
value = normalizeIdentifierValue(identifierType, value)
|
||||
if protocolValue == "" || identifierType == "" || value == "" {
|
||||
return ""
|
||||
}
|
||||
return "vehicle_identifier\x00" + protocolValue + "\x00" + sourceCode + "\x00" + identifierType + "\x00" + value
|
||||
}
|
||||
|
||||
func sourceSnapshotKey(protocolValue string, sourceIP string) string {
|
||||
protocolValue = strings.TrimSpace(protocolValue)
|
||||
sourceIP = normalizeEndpointIP(sourceIP)
|
||||
if protocolValue == "" || sourceIP == "" {
|
||||
return ""
|
||||
}
|
||||
return protocolValue + "\x00" + sourceIP
|
||||
}
|
||||
166
go/vehicle-gateway/internal/identity/snapshot_test.go
Normal file
166
go/vehicle-gateway/internal/identity/snapshot_test.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestRefreshSnapshotResolvesKnownJT808WithoutPerFrameQueries(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectIdentitySnapshot(mock)
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SnapshotOnlyLookups: true,
|
||||
})
|
||||
result, err := resolver.RefreshSnapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("RefreshSnapshot() error = %v", err)
|
||||
}
|
||||
if result.BindingEntries != 3 || result.IdentifierEntries != 2 || result.RegistrationEntries != 1 || result.SourceEntries != 1 {
|
||||
t.Fatalf("snapshot result = %+v", result)
|
||||
}
|
||||
|
||||
resolved, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if resolved.VIN != "LNBVIN00000000001" {
|
||||
t.Fatalf("vin = %q", resolved.VIN)
|
||||
}
|
||||
if resolved.SourceCode != "g7s" || resolved.PlatformName != "G7s" || resolved.SourceKind != "PLATFORM" {
|
||||
t.Fatalf("source metadata = code:%q platform:%q kind:%q", resolved.SourceCode, resolved.PlatformName, resolved.SourceKind)
|
||||
}
|
||||
identityMetadata, _ := resolved.Parsed["identity"].(map[string]any)
|
||||
if identityMetadata["cache_status"] != "snapshot" {
|
||||
t.Fatalf("identity metadata = %#v, want snapshot cache status", identityMetadata)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotOnlyResolverMissDoesNotQueryMySQL(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SnapshotOnlyLookups: true,
|
||||
})
|
||||
resolved, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307700000",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if resolved.VIN != "" {
|
||||
t.Fatalf("vin = %q, want unresolved", resolved.VIN)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("snapshot-only miss should not query mysql: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshSnapshotFailureKeepsLastKnownGoodSnapshot(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectIdentitySnapshot(mock)
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SnapshotOnlyLookups: true,
|
||||
})
|
||||
first, err := resolver.RefreshSnapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("first RefreshSnapshot() error = %v", err)
|
||||
}
|
||||
mock.ExpectQuery("SELECT vin, plate, phone").
|
||||
WillReturnError(errors.New("mysql unavailable"))
|
||||
if _, err := resolver.RefreshSnapshot(context.Background()); err == nil {
|
||||
t.Fatal("second RefreshSnapshot() error = nil, want failure")
|
||||
}
|
||||
|
||||
resolved, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
Phone: "013307795425",
|
||||
Parsed: map[string]any{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() after failed refresh error = %v", err)
|
||||
}
|
||||
if resolved.VIN != "LNBVIN00000000001" {
|
||||
t.Fatalf("vin after failed refresh = %q", resolved.VIN)
|
||||
}
|
||||
stats := resolver.CacheStats()
|
||||
if !stats.SnapshotReady || stats.SnapshotRefreshedAt.IsZero() || !stats.SnapshotRefreshedAt.Equal(first.RefreshedAt) {
|
||||
t.Fatalf("snapshot stats after failed refresh = %+v, first = %+v", stats, first)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func expectIdentitySnapshot(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery("SELECT vin, plate, phone").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "plate", "phone"}).
|
||||
AddRow("LNBVIN00000000001", "粤A00001", "13307795425"))
|
||||
mock.ExpectQuery("SELECT protocol, source_code, identifier_type, identifier_value").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"protocol", "source_code", "identifier_type", "identifier_value", "vin", "platform_name"}).
|
||||
AddRow("JT808", "g7s", "JT808_PHONE", "13307795425", "LNBVIN00000000001", "G7s"))
|
||||
mock.ExpectQuery("SELECT phone, vin, device_id, plate, auth_token").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"phone", "vin", "device_id", "plate", "auth_token"}).
|
||||
AddRow("13307795425", "LNBVIN00000000001", "DEVICE-1", "粤A00001", "device-code"))
|
||||
mock.ExpectQuery("SELECT protocol, source_ip, source_code, platform_name, source_kind").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"protocol", "source_ip", "source_code", "platform_name", "source_kind"}).
|
||||
AddRow("JT808", "115.231.168.135", "g7s", "G7s", "PLATFORM"))
|
||||
}
|
||||
|
||||
func TestSnapshotServesJT808AuthenticationTokenByNormalizedPhone(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectIdentitySnapshot(mock)
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SnapshotOnlyLookups: true,
|
||||
})
|
||||
if _, err := resolver.RefreshSnapshot(context.Background()); err != nil {
|
||||
t.Fatalf("RefreshSnapshot() error = %v", err)
|
||||
}
|
||||
token, ok := resolver.JT808AuthToken("0013307795425")
|
||||
if !ok || token != "device-code" {
|
||||
t.Fatalf("JT808AuthToken() = %q, %v", token, ok)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotResultRefreshedAtUsesCurrentTime(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
expectIdentitySnapshot(mock)
|
||||
|
||||
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
|
||||
SnapshotOnlyLookups: true,
|
||||
})
|
||||
before := time.Now()
|
||||
result, err := resolver.RefreshSnapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("RefreshSnapshot() error = %v", err)
|
||||
}
|
||||
if result.RefreshedAt.Before(before) || result.RefreshedAt.After(time.Now()) {
|
||||
t.Fatalf("refreshed_at = %v, want current time", result.RefreshedAt)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user