package identity import ( "context" "database/sql" "errors" "fmt" "io/fs" "path/filepath" "regexp" "sort" "strconv" "strings" "github.com/xuri/excelize/v2" ) const ( IdentifierTypeJT808Phone = "JT808_PHONE" IdentifierTypePlate = "PLATE" ) type MappingRecord struct { File string `json:"file"` Sheet string `json:"sheet"` Row int `json:"row"` SourceCode string `json:"source_code"` SourceName string `json:"source_name"` Protocol string `json:"protocol"` IdentifierType string `json:"identifier_type"` IdentifierValue string `json:"identifier_value"` RawValue string `json:"raw_value,omitempty"` Plate string `json:"plate,omitempty"` OEM string `json:"oem,omitempty"` } type MappingScanReport struct { Files int `json:"files"` Sheets int `json:"sheets"` Rows int `json:"rows"` Records int `json:"records"` Skipped int `json:"skipped"` UnsupportedFiles int `json:"unsupported_files,omitempty"` Sources []MappingSourceScanReport `json:"sources,omitempty"` UnsupportedItems []MappingUnsupportedFileReport `json:"unsupported_items,omitempty"` Errors []string `json:"errors,omitempty"` } type MappingSourceScanReport struct { SourceCode string `json:"source_code"` SourceName string `json:"source_name"` Files int `json:"files"` Sheets int `json:"sheets"` Rows int `json:"rows"` Records int `json:"records"` PhoneRecords int `json:"phone_records"` PlateRecords int `json:"plate_records"` Skipped int `json:"skipped"` } type MappingUnsupportedFileReport struct { File string `json:"file"` Ext string `json:"ext"` Reason string `json:"reason"` } type MappingImportOptions struct { Apply bool LegacyTable string ReportItemLimit int } type MappingImportReport struct { Scan MappingScanReport `json:"scan"` Records int `json:"records"` Deduplicated int `json:"deduplicated"` Resolved int `json:"resolved"` Unresolved int `json:"unresolved"` Conflicts int `json:"conflicts"` WouldInsert int `json:"would_insert,omitempty"` WouldUpdate int `json:"would_update,omitempty"` Inserted int `json:"inserted,omitempty"` Updated int `json:"updated,omitempty"` SourceResults []MappingSourceImportStat `json:"source_results,omitempty"` UnresolvedItems []MappingRecord `json:"unresolved_items,omitempty"` ConflictItems []MappingConflict `json:"conflict_items,omitempty"` } type MappingSourceImportStat struct { SourceCode string `json:"source_code"` SourceName string `json:"source_name"` Records int `json:"records"` Deduplicated int `json:"deduplicated"` Resolved int `json:"resolved"` Unresolved int `json:"unresolved"` Conflicts int `json:"conflicts"` WouldInsert int `json:"would_insert,omitempty"` WouldUpdate int `json:"would_update,omitempty"` Inserted int `json:"inserted,omitempty"` Updated int `json:"updated,omitempty"` } type MappingConflict struct { Record MappingRecord `json:"record"` ExistingVIN string `json:"existing_vin,omitempty"` NewVIN string `json:"new_vin,omitempty"` Reason string `json:"reason"` } type resolvedMappingRecord struct { MappingRecord VIN string } type mappingStore interface { ExecContext(context.Context, string, ...any) (sql.Result, error) QueryRowContext(context.Context, string, ...any) *sql.Row } var digitPattern = regexp.MustCompile(`\D+`) func EnsureVehicleIdentifierSchema(ctx context.Context, db *sql.DB) error { if db == nil { return errors.New("identity db must not be nil") } if _, err := db.ExecContext(ctx, vehicleTableSQL); err != nil { return err } _, err := db.ExecContext(ctx, vehicleIdentifierTableSQL) return err } func ReadMappingDirectory(root string) ([]MappingRecord, MappingScanReport, error) { root = strings.TrimSpace(root) if root == "" { return nil, MappingScanReport{}, errors.New("mapping input directory is empty") } var records []MappingRecord report := MappingScanReport{} err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { if err != nil { report.Errors = append(report.Errors, err.Error()) return nil } if entry == nil || entry.IsDir() { return nil } name := entry.Name() if strings.HasPrefix(name, "~$") || strings.HasPrefix(name, "._") { return nil } ext := strings.ToLower(filepath.Ext(name)) if !isSupportedMappingWorkbookExt(ext) { if isUnsupportedMappingWorkbookExt(ext) { report.UnsupportedFiles++ report.UnsupportedItems = append(report.UnsupportedItems, MappingUnsupportedFileReport{ File: path, Ext: ext, Reason: "convert legacy workbook to .xlsx before import", }) } return nil } fileRecords, fileReport, err := readMappingWorkbook(root, path) report.Files++ report.Sheets += fileReport.Sheets report.Rows += fileReport.Rows report.Records += fileReport.Records report.Skipped += fileReport.Skipped report.Errors = append(report.Errors, fileReport.Errors...) for _, source := range fileReport.Sources { mergeMappingSourceScan(&report, source) } if err != nil { report.Errors = append(report.Errors, fmt.Sprintf("%s: %v", path, err)) return nil } records = append(records, fileRecords...) return nil }) if err != nil { return records, report, err } sortMappingSourceScans(report.Sources) return records, report, nil } func readMappingWorkbook(root string, path string) ([]MappingRecord, MappingScanReport, error) { workbook, err := excelize.OpenFile(path) if err != nil { return nil, MappingScanReport{}, err } defer func() { _ = workbook.Close() }() sourceCode, sourceName := mappingSource(root, path) sourceReport := MappingSourceScanReport{ SourceCode: sourceCode, SourceName: sourceName, Files: 1, } var records []MappingRecord report := MappingScanReport{} for _, sheet := range workbook.GetSheetList() { rows, err := workbook.GetRows(sheet) if err != nil { report.Errors = append(report.Errors, fmt.Sprintf("%s/%s: %v", path, sheet, err)) continue } report.Sheets++ sourceReport.Sheets++ report.Rows += len(rows) sourceReport.Rows += len(rows) if len(rows) == 0 { continue } header, dataStart := mappingHeader(rows) for rowIndex := dataStart; rowIndex < len(rows); rowIndex++ { row := rows[rowIndex] plate := normalizePlate(cellByHeader(row, header, "plate")) rawPhone := cellByHeader(row, header, "phone") phone := normalizeMappingPhone(rawPhone) if len(header) == 0 { plate = normalizePlate(cell(row, 0)) rawPhone = cell(row, 1) phone = normalizeMappingPhone(rawPhone) } if plate == "" && phone == "" { report.Skipped++ sourceReport.Skipped++ continue } if phone != "" { records = append(records, MappingRecord{ File: path, Sheet: sheet, Row: rowIndex + 1, SourceCode: sourceCode, SourceName: sourceName, Protocol: "JT808", IdentifierType: IdentifierTypeJT808Phone, IdentifierValue: phone, RawValue: strings.TrimSpace(rawPhone), Plate: plate, OEM: sourceName, }) sourceReport.PhoneRecords++ sourceReport.Records++ } if plate != "" { records = append(records, MappingRecord{ File: path, Sheet: sheet, Row: rowIndex + 1, SourceCode: sourceCode, SourceName: sourceName, Protocol: "JT808", IdentifierType: IdentifierTypePlate, IdentifierValue: plate, RawValue: plate, Plate: plate, OEM: sourceName, }) sourceReport.PlateRecords++ sourceReport.Records++ } } } report.Records = len(records) report.Sources = []MappingSourceScanReport{sourceReport} return records, report, nil } func mergeMappingSourceScan(report *MappingScanReport, source MappingSourceScanReport) { if report == nil || strings.TrimSpace(source.SourceCode) == "" { return } for index := range report.Sources { if report.Sources[index].SourceCode != source.SourceCode { continue } report.Sources[index].Files += source.Files report.Sources[index].Sheets += source.Sheets report.Sources[index].Rows += source.Rows report.Sources[index].Records += source.Records report.Sources[index].PhoneRecords += source.PhoneRecords report.Sources[index].PlateRecords += source.PlateRecords report.Sources[index].Skipped += source.Skipped if report.Sources[index].SourceName == "" { report.Sources[index].SourceName = source.SourceName } return } report.Sources = append(report.Sources, source) } func sortMappingSourceScans(sources []MappingSourceScanReport) { sort.SliceStable(sources, func(i, j int) bool { return sources[i].SourceCode < sources[j].SourceCode }) } func isSupportedMappingWorkbookExt(ext string) bool { switch strings.ToLower(strings.TrimSpace(ext)) { case ".xlsx", ".xlsm": return true default: return false } } func isUnsupportedMappingWorkbookExt(ext string) bool { switch strings.ToLower(strings.TrimSpace(ext)) { case ".xls", ".xlsb": return true default: return false } } func ImportMappingRecords(ctx context.Context, db *sql.DB, records []MappingRecord, scan MappingScanReport, opts MappingImportOptions) (MappingImportReport, error) { if db == nil { return MappingImportReport{}, errors.New("identity db must not be nil") } legacyTable := strings.TrimSpace(opts.LegacyTable) if legacyTable == "" || !safeIdentifier(legacyTable) { legacyTable = "vehicle_identity_binding" } report := MappingImportReport{ Scan: scan, Records: len(records), } sourceStats := map[string]*MappingSourceImportStat{} for _, record := range records { sourceImportStat(sourceStats, record).Records++ } reportLimit := opts.ReportItemLimit if reportLimit == 0 { reportLimit = 50 } deduped, conflicts := dedupeMappingRecords(records) report.Deduplicated = len(deduped) report.Conflicts += len(conflicts) for _, conflict := range conflicts { sourceImportStat(sourceStats, conflict.Record).Conflicts++ appendConflictItem(&report, conflict, reportLimit) } store := mappingStore(db) var tx *sql.Tx if opts.Apply { var err error tx, err = db.BeginTx(ctx, nil) if err != nil { return report, err } store = tx defer func() { if tx != nil { _ = tx.Rollback() } }() } for _, record := range deduped { sourceStat := sourceImportStat(sourceStats, record) sourceStat.Deduplicated++ vin, err := resolveMappingVIN(ctx, store, legacyTable, record) if err != nil { return report, err } if vin == "" { report.Unresolved++ sourceStat.Unresolved++ appendUnresolvedItem(&report, record, reportLimit) continue } resolved := resolvedMappingRecord{MappingRecord: record, VIN: vin} existingVIN, exists, err := existingIdentifierVIN(ctx, store, resolved) if err != nil { return report, err } if exists && !strings.EqualFold(existingVIN, vin) { report.Conflicts++ sourceStat.Conflicts++ appendConflictItem(&report, MappingConflict{ Record: record, ExistingVIN: existingVIN, NewVIN: vin, Reason: "identifier already points to another vin", }, reportLimit) continue } report.Resolved++ sourceStat.Resolved++ if exists { if opts.Apply { if err := updateVehicleIdentifier(ctx, store, resolved); err != nil { return report, err } report.Updated++ sourceStat.Updated++ } else { report.WouldUpdate++ sourceStat.WouldUpdate++ } continue } if opts.Apply { if err := upsertVehicle(ctx, store, resolved); err != nil { return report, err } if err := insertVehicleIdentifier(ctx, store, resolved); err != nil { return report, err } report.Inserted++ sourceStat.Inserted++ } else { report.WouldInsert++ sourceStat.WouldInsert++ } } if tx != nil { if err := tx.Commit(); err != nil { return report, err } tx = nil } report.SourceResults = sortedMappingSourceImportStats(sourceStats) return report, nil } func sourceImportStat(stats map[string]*MappingSourceImportStat, record MappingRecord) *MappingSourceImportStat { sourceCode := strings.TrimSpace(record.SourceCode) if sourceCode == "" { sourceCode = "unknown" } stat := stats[sourceCode] if stat != nil { if stat.SourceName == "" { stat.SourceName = strings.TrimSpace(record.SourceName) } return stat } stat = &MappingSourceImportStat{ SourceCode: sourceCode, SourceName: strings.TrimSpace(record.SourceName), } stats[sourceCode] = stat return stat } func sortedMappingSourceImportStats(stats map[string]*MappingSourceImportStat) []MappingSourceImportStat { if len(stats) == 0 { return nil } keys := make([]string, 0, len(stats)) for key := range stats { keys = append(keys, key) } sort.Strings(keys) out := make([]MappingSourceImportStat, 0, len(keys)) for _, key := range keys { out = append(out, *stats[key]) } return out } func appendUnresolvedItem(report *MappingImportReport, record MappingRecord, limit int) { if report == nil { return } if limit >= 0 && len(report.UnresolvedItems) >= limit { return } report.UnresolvedItems = append(report.UnresolvedItems, record) } func appendConflictItem(report *MappingImportReport, conflict MappingConflict, limit int) { if report == nil { return } if limit >= 0 && len(report.ConflictItems) >= limit { return } report.ConflictItems = append(report.ConflictItems, conflict) } func dedupeMappingRecords(records []MappingRecord) ([]MappingRecord, []MappingConflict) { seen := map[string]MappingRecord{} indexByKey := map[string]int{} var out []MappingRecord var conflicts []MappingConflict for _, record := range records { record.IdentifierValue = normalizeIdentifierValue(record.IdentifierType, record.IdentifierValue) record.Plate = normalizePlate(record.Plate) if record.Protocol == "" { record.Protocol = "JT808" } if record.IdentifierValue == "" || record.IdentifierType == "" { continue } key := strings.Join([]string{record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue}, "\x00") existing, ok := seen[key] if !ok { seen[key] = record indexByKey[key] = len(out) out = append(out, record) continue } if existing.Plate != "" && record.Plate != "" && existing.Plate != record.Plate { conflicts = append(conflicts, MappingConflict{ Record: record, Reason: fmt.Sprintf("same source identifier maps to multiple plates: %s/%s", existing.Plate, record.Plate), }) continue } merged := mergeMappingRecord(existing, record) seen[key] = merged if index, ok := indexByKey[key]; ok && index >= 0 && index < len(out) { out[index] = merged } } return out, conflicts } func mergeMappingRecord(existing MappingRecord, incoming MappingRecord) MappingRecord { merged := existing if merged.File == "" { merged.File = incoming.File } if merged.Sheet == "" { merged.Sheet = incoming.Sheet } if merged.Row == 0 { merged.Row = incoming.Row } if merged.SourceName == "" { merged.SourceName = incoming.SourceName } if merged.Protocol == "" { merged.Protocol = incoming.Protocol } if merged.IdentifierType == "" { merged.IdentifierType = incoming.IdentifierType } if merged.IdentifierValue == "" { merged.IdentifierValue = incoming.IdentifierValue } if merged.RawValue == "" { merged.RawValue = incoming.RawValue } if merged.Plate == "" { merged.Plate = incoming.Plate } if merged.OEM == "" { merged.OEM = incoming.OEM } return merged } func resolveMappingVIN(ctx context.Context, db mappingStore, legacyTable string, record MappingRecord) (string, error) { if record.Plate != "" { vin, err := lookupLegacyVIN(ctx, db, legacyTable, "plate", record.Plate) if err != nil && !errors.Is(err, sql.ErrNoRows) { return "", err } if vin != "" { return vin, nil } } if record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue != "" { vin, err := lookupLegacyVIN(ctx, db, legacyTable, "phone", record.IdentifierValue) if err != nil && !errors.Is(err, sql.ErrNoRows) { return "", err } return vin, nil } return "", nil } func lookupLegacyVIN(ctx context.Context, db mappingStore, table string, column string, value string) (string, error) { if !safeIdentifier(table) || !safeIdentifier(column) { return "", sql.ErrNoRows } query := "SELECT vin FROM " + table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' LIMIT 1" var vin string err := db.QueryRowContext(ctx, query, value).Scan(&vin) if err != nil { return "", err } return strings.TrimSpace(vin), nil } func existingIdentifierVIN(ctx context.Context, db mappingStore, record resolvedMappingRecord) (string, bool, error) { var vin string err := db.QueryRowContext(ctx, `SELECT vin FROM vehicle_identifier WHERE protocol = ? AND source_code = ? AND identifier_type = ? AND identifier_value = ?`, record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue).Scan(&vin) if errors.Is(err, sql.ErrNoRows) { return "", false, nil } if err != nil { return "", false, err } return strings.TrimSpace(vin), true, nil } func upsertVehicle(ctx context.Context, db mappingStore, record resolvedMappingRecord) error { _, err := db.ExecContext(ctx, `INSERT INTO vehicle (vin, plate, oem, enabled) VALUES (?, ?, ?, 1) ON DUPLICATE KEY UPDATE plate = IF(VALUES(plate) <> '', VALUES(plate), plate), oem = IF(VALUES(oem) <> '', VALUES(oem), oem), enabled = 1`, record.VIN, record.Plate, record.OEM) return err } func insertVehicleIdentifier(ctx context.Context, db mappingStore, record resolvedMappingRecord) error { _, err := db.ExecContext(ctx, `INSERT INTO vehicle_identifier (protocol, source_code, identifier_type, identifier_value, vin, plate, oem, raw_value, enabled, latest_import_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`, record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue, record.VIN, record.Plate, record.OEM, record.RawValue, record.File, ) return err } func updateVehicleIdentifier(ctx context.Context, db mappingStore, record resolvedMappingRecord) error { _, err := db.ExecContext(ctx, `UPDATE vehicle_identifier SET plate = IF(? <> '', ?, plate), oem = IF(? <> '', ?, oem), raw_value = IF(? <> '', ?, raw_value), latest_import_file = ?, enabled = 1 WHERE protocol = ? AND source_code = ? AND identifier_type = ? AND identifier_value = ?`, record.Plate, record.Plate, record.OEM, record.OEM, record.RawValue, record.RawValue, record.File, record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue, ) return err } func mappingSource(root string, path string) (string, string) { rel, err := filepath.Rel(root, path) if err != nil { rel = filepath.Base(path) } parts := strings.Split(filepath.ToSlash(rel), "/") name := strings.TrimSpace(parts[0]) if name == "" || strings.EqualFold(name, ".") { name = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) } code := sourceCode(name) return code, name } func sourceCode(name string) string { switch strings.TrimSpace(strings.ToLower(name)) { case "g7s": return "g7s" case "信达": return "xinda" case "广安北斗", "广安车联": return "guangan_beidou" case "东方北斗": return "dongfang_beidou" case "赛格": return "saige" default: return normalizeASCIIKey(name) } } func mappingHeader(rows [][]string) (map[string]int, int) { for index, row := range rows { header := map[string]int{} for columnIndex, value := range row { key := normalizeHeader(value) switch key { case "车牌", "车牌号", "车牌号码": header["plate"] = columnIndex case "sim", "sim卡号", "手机号", "终端手机号", "终端id", "终端标识": header["phone"] = columnIndex } } if len(header) > 0 { return header, index + 1 } if index >= 5 { break } } return nil, 0 } func cellByHeader(row []string, header map[string]int, key string) string { if len(header) == 0 { return "" } index, ok := header[key] if !ok { return "" } return cell(row, index) } func cell(row []string, index int) string { if index < 0 || index >= len(row) { return "" } return strings.TrimSpace(row[index]) } func normalizeHeader(value string) string { value = strings.TrimSpace(strings.ToLower(value)) value = strings.ReplaceAll(value, " ", "") value = strings.ReplaceAll(value, "\t", "") value = strings.ReplaceAll(value, "(", "(") value = strings.ReplaceAll(value, ")", ")") return value } func normalizePlate(value string) string { value = strings.TrimSpace(value) value = strings.ReplaceAll(value, " ", "") value = strings.ReplaceAll(value, "\t", "") return value } func normalizeMappingPhone(value string) string { value = strings.TrimSpace(value) if value == "" { return "" } if strings.ContainsAny(value, ".eE") { if parsed, err := strconv.ParseFloat(value, 64); err == nil && parsed > 0 { return normalizePhone(strconv.FormatFloat(parsed, 'f', 0, 64)) } } digits := digitPattern.ReplaceAllString(value, "") return normalizePhone(digits) } func normalizeASCIIKey(value string) string { value = strings.TrimSpace(strings.ToLower(value)) var b strings.Builder lastUnderscore := false for _, r := range value { if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { b.WriteRune(r) lastUnderscore = false continue } if !lastUnderscore { b.WriteByte('_') lastUnderscore = true } } return strings.Trim(b.String(), "_") }