feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
773
go/vehicle-gateway/internal/identity/mapping_import.go
Normal file
773
go/vehicle-gateway/internal/identity/mapping_import.go
Normal file
@@ -0,0 +1,773 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
IdentifierTypeJT808Phone = "JT808_PHONE"
|
||||
IdentifierTypePlate = "PLATE"
|
||||
)
|
||||
|
||||
type MappingRecord struct {
|
||||
File string `json:"file"`
|
||||
Sheet string `json:"sheet"`
|
||||
Row int `json:"row"`
|
||||
SourceCode string `json:"source_code"`
|
||||
SourceName string `json:"source_name"`
|
||||
Protocol string `json:"protocol"`
|
||||
IdentifierType string `json:"identifier_type"`
|
||||
IdentifierValue string `json:"identifier_value"`
|
||||
RawValue string `json:"raw_value,omitempty"`
|
||||
Plate string `json:"plate,omitempty"`
|
||||
OEM string `json:"oem,omitempty"`
|
||||
}
|
||||
|
||||
type MappingScanReport struct {
|
||||
Files int `json:"files"`
|
||||
Sheets int `json:"sheets"`
|
||||
Rows int `json:"rows"`
|
||||
Records int `json:"records"`
|
||||
Skipped int `json:"skipped"`
|
||||
UnsupportedFiles int `json:"unsupported_files,omitempty"`
|
||||
Sources []MappingSourceScanReport `json:"sources,omitempty"`
|
||||
UnsupportedItems []MappingUnsupportedFileReport `json:"unsupported_items,omitempty"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
type MappingSourceScanReport struct {
|
||||
SourceCode string `json:"source_code"`
|
||||
SourceName string `json:"source_name"`
|
||||
Files int `json:"files"`
|
||||
Sheets int `json:"sheets"`
|
||||
Rows int `json:"rows"`
|
||||
Records int `json:"records"`
|
||||
PhoneRecords int `json:"phone_records"`
|
||||
PlateRecords int `json:"plate_records"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
|
||||
type MappingUnsupportedFileReport struct {
|
||||
File string `json:"file"`
|
||||
Ext string `json:"ext"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type MappingImportOptions struct {
|
||||
Apply bool
|
||||
LegacyTable string
|
||||
ReportItemLimit int
|
||||
}
|
||||
|
||||
type MappingImportReport struct {
|
||||
Scan MappingScanReport `json:"scan"`
|
||||
Records int `json:"records"`
|
||||
Deduplicated int `json:"deduplicated"`
|
||||
Resolved int `json:"resolved"`
|
||||
Unresolved int `json:"unresolved"`
|
||||
Conflicts int `json:"conflicts"`
|
||||
WouldInsert int `json:"would_insert,omitempty"`
|
||||
WouldUpdate int `json:"would_update,omitempty"`
|
||||
Inserted int `json:"inserted,omitempty"`
|
||||
Updated int `json:"updated,omitempty"`
|
||||
SourceResults []MappingSourceImportStat `json:"source_results,omitempty"`
|
||||
UnresolvedItems []MappingRecord `json:"unresolved_items,omitempty"`
|
||||
ConflictItems []MappingConflict `json:"conflict_items,omitempty"`
|
||||
}
|
||||
|
||||
type MappingSourceImportStat struct {
|
||||
SourceCode string `json:"source_code"`
|
||||
SourceName string `json:"source_name"`
|
||||
Records int `json:"records"`
|
||||
Deduplicated int `json:"deduplicated"`
|
||||
Resolved int `json:"resolved"`
|
||||
Unresolved int `json:"unresolved"`
|
||||
Conflicts int `json:"conflicts"`
|
||||
WouldInsert int `json:"would_insert,omitempty"`
|
||||
WouldUpdate int `json:"would_update,omitempty"`
|
||||
Inserted int `json:"inserted,omitempty"`
|
||||
Updated int `json:"updated,omitempty"`
|
||||
}
|
||||
|
||||
type MappingConflict struct {
|
||||
Record MappingRecord `json:"record"`
|
||||
ExistingVIN string `json:"existing_vin,omitempty"`
|
||||
NewVIN string `json:"new_vin,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type resolvedMappingRecord struct {
|
||||
MappingRecord
|
||||
VIN string
|
||||
}
|
||||
|
||||
type mappingStore interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}
|
||||
|
||||
var digitPattern = regexp.MustCompile(`\D+`)
|
||||
|
||||
func EnsureVehicleIdentifierSchema(ctx context.Context, db *sql.DB) error {
|
||||
if db == nil {
|
||||
return errors.New("identity db must not be nil")
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, vehicleTableSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := db.ExecContext(ctx, vehicleIdentifierTableSQL)
|
||||
return err
|
||||
}
|
||||
|
||||
func ReadMappingDirectory(root string) ([]MappingRecord, MappingScanReport, error) {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
return nil, MappingScanReport{}, errors.New("mapping input directory is empty")
|
||||
}
|
||||
var records []MappingRecord
|
||||
report := MappingScanReport{}
|
||||
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
report.Errors = append(report.Errors, err.Error())
|
||||
return nil
|
||||
}
|
||||
if entry == nil || entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := entry.Name()
|
||||
if strings.HasPrefix(name, "~$") || strings.HasPrefix(name, "._") {
|
||||
return nil
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(name))
|
||||
if !isSupportedMappingWorkbookExt(ext) {
|
||||
if isUnsupportedMappingWorkbookExt(ext) {
|
||||
report.UnsupportedFiles++
|
||||
report.UnsupportedItems = append(report.UnsupportedItems, MappingUnsupportedFileReport{
|
||||
File: path,
|
||||
Ext: ext,
|
||||
Reason: "convert legacy workbook to .xlsx before import",
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fileRecords, fileReport, err := readMappingWorkbook(root, path)
|
||||
report.Files++
|
||||
report.Sheets += fileReport.Sheets
|
||||
report.Rows += fileReport.Rows
|
||||
report.Records += fileReport.Records
|
||||
report.Skipped += fileReport.Skipped
|
||||
report.Errors = append(report.Errors, fileReport.Errors...)
|
||||
for _, source := range fileReport.Sources {
|
||||
mergeMappingSourceScan(&report, source)
|
||||
}
|
||||
if err != nil {
|
||||
report.Errors = append(report.Errors, fmt.Sprintf("%s: %v", path, err))
|
||||
return nil
|
||||
}
|
||||
records = append(records, fileRecords...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return records, report, err
|
||||
}
|
||||
sortMappingSourceScans(report.Sources)
|
||||
return records, report, nil
|
||||
}
|
||||
|
||||
func readMappingWorkbook(root string, path string) ([]MappingRecord, MappingScanReport, error) {
|
||||
workbook, err := excelize.OpenFile(path)
|
||||
if err != nil {
|
||||
return nil, MappingScanReport{}, err
|
||||
}
|
||||
defer func() { _ = workbook.Close() }()
|
||||
|
||||
sourceCode, sourceName := mappingSource(root, path)
|
||||
sourceReport := MappingSourceScanReport{
|
||||
SourceCode: sourceCode,
|
||||
SourceName: sourceName,
|
||||
Files: 1,
|
||||
}
|
||||
var records []MappingRecord
|
||||
report := MappingScanReport{}
|
||||
for _, sheet := range workbook.GetSheetList() {
|
||||
rows, err := workbook.GetRows(sheet)
|
||||
if err != nil {
|
||||
report.Errors = append(report.Errors, fmt.Sprintf("%s/%s: %v", path, sheet, err))
|
||||
continue
|
||||
}
|
||||
report.Sheets++
|
||||
sourceReport.Sheets++
|
||||
report.Rows += len(rows)
|
||||
sourceReport.Rows += len(rows)
|
||||
if len(rows) == 0 {
|
||||
continue
|
||||
}
|
||||
header, dataStart := mappingHeader(rows)
|
||||
for rowIndex := dataStart; rowIndex < len(rows); rowIndex++ {
|
||||
row := rows[rowIndex]
|
||||
plate := normalizePlate(cellByHeader(row, header, "plate"))
|
||||
rawPhone := cellByHeader(row, header, "phone")
|
||||
phone := normalizeMappingPhone(rawPhone)
|
||||
if len(header) == 0 {
|
||||
plate = normalizePlate(cell(row, 0))
|
||||
rawPhone = cell(row, 1)
|
||||
phone = normalizeMappingPhone(rawPhone)
|
||||
}
|
||||
if plate == "" && phone == "" {
|
||||
report.Skipped++
|
||||
sourceReport.Skipped++
|
||||
continue
|
||||
}
|
||||
if phone != "" {
|
||||
records = append(records, MappingRecord{
|
||||
File: path,
|
||||
Sheet: sheet,
|
||||
Row: rowIndex + 1,
|
||||
SourceCode: sourceCode,
|
||||
SourceName: sourceName,
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: phone,
|
||||
RawValue: strings.TrimSpace(rawPhone),
|
||||
Plate: plate,
|
||||
OEM: sourceName,
|
||||
})
|
||||
sourceReport.PhoneRecords++
|
||||
sourceReport.Records++
|
||||
}
|
||||
if plate != "" {
|
||||
records = append(records, MappingRecord{
|
||||
File: path,
|
||||
Sheet: sheet,
|
||||
Row: rowIndex + 1,
|
||||
SourceCode: sourceCode,
|
||||
SourceName: sourceName,
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypePlate,
|
||||
IdentifierValue: plate,
|
||||
RawValue: plate,
|
||||
Plate: plate,
|
||||
OEM: sourceName,
|
||||
})
|
||||
sourceReport.PlateRecords++
|
||||
sourceReport.Records++
|
||||
}
|
||||
}
|
||||
}
|
||||
report.Records = len(records)
|
||||
report.Sources = []MappingSourceScanReport{sourceReport}
|
||||
return records, report, nil
|
||||
}
|
||||
|
||||
func mergeMappingSourceScan(report *MappingScanReport, source MappingSourceScanReport) {
|
||||
if report == nil || strings.TrimSpace(source.SourceCode) == "" {
|
||||
return
|
||||
}
|
||||
for index := range report.Sources {
|
||||
if report.Sources[index].SourceCode != source.SourceCode {
|
||||
continue
|
||||
}
|
||||
report.Sources[index].Files += source.Files
|
||||
report.Sources[index].Sheets += source.Sheets
|
||||
report.Sources[index].Rows += source.Rows
|
||||
report.Sources[index].Records += source.Records
|
||||
report.Sources[index].PhoneRecords += source.PhoneRecords
|
||||
report.Sources[index].PlateRecords += source.PlateRecords
|
||||
report.Sources[index].Skipped += source.Skipped
|
||||
if report.Sources[index].SourceName == "" {
|
||||
report.Sources[index].SourceName = source.SourceName
|
||||
}
|
||||
return
|
||||
}
|
||||
report.Sources = append(report.Sources, source)
|
||||
}
|
||||
|
||||
func sortMappingSourceScans(sources []MappingSourceScanReport) {
|
||||
sort.SliceStable(sources, func(i, j int) bool {
|
||||
return sources[i].SourceCode < sources[j].SourceCode
|
||||
})
|
||||
}
|
||||
|
||||
func isSupportedMappingWorkbookExt(ext string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(ext)) {
|
||||
case ".xlsx", ".xlsm":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isUnsupportedMappingWorkbookExt(ext string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(ext)) {
|
||||
case ".xls", ".xlsb":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func ImportMappingRecords(ctx context.Context, db *sql.DB, records []MappingRecord, scan MappingScanReport, opts MappingImportOptions) (MappingImportReport, error) {
|
||||
if db == nil {
|
||||
return MappingImportReport{}, errors.New("identity db must not be nil")
|
||||
}
|
||||
legacyTable := strings.TrimSpace(opts.LegacyTable)
|
||||
if legacyTable == "" || !safeIdentifier(legacyTable) {
|
||||
legacyTable = "vehicle_identity_binding"
|
||||
}
|
||||
report := MappingImportReport{
|
||||
Scan: scan,
|
||||
Records: len(records),
|
||||
}
|
||||
sourceStats := map[string]*MappingSourceImportStat{}
|
||||
for _, record := range records {
|
||||
sourceImportStat(sourceStats, record).Records++
|
||||
}
|
||||
reportLimit := opts.ReportItemLimit
|
||||
if reportLimit == 0 {
|
||||
reportLimit = 50
|
||||
}
|
||||
deduped, conflicts := dedupeMappingRecords(records)
|
||||
report.Deduplicated = len(deduped)
|
||||
report.Conflicts += len(conflicts)
|
||||
for _, conflict := range conflicts {
|
||||
sourceImportStat(sourceStats, conflict.Record).Conflicts++
|
||||
appendConflictItem(&report, conflict, reportLimit)
|
||||
}
|
||||
|
||||
store := mappingStore(db)
|
||||
var tx *sql.Tx
|
||||
if opts.Apply {
|
||||
var err error
|
||||
tx, err = db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
store = tx
|
||||
defer func() {
|
||||
if tx != nil {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for _, record := range deduped {
|
||||
sourceStat := sourceImportStat(sourceStats, record)
|
||||
sourceStat.Deduplicated++
|
||||
vin, err := resolveMappingVIN(ctx, store, legacyTable, record)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if vin == "" {
|
||||
report.Unresolved++
|
||||
sourceStat.Unresolved++
|
||||
appendUnresolvedItem(&report, record, reportLimit)
|
||||
continue
|
||||
}
|
||||
resolved := resolvedMappingRecord{MappingRecord: record, VIN: vin}
|
||||
existingVIN, exists, err := existingIdentifierVIN(ctx, store, resolved)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if exists && !strings.EqualFold(existingVIN, vin) {
|
||||
report.Conflicts++
|
||||
sourceStat.Conflicts++
|
||||
appendConflictItem(&report, MappingConflict{
|
||||
Record: record,
|
||||
ExistingVIN: existingVIN,
|
||||
NewVIN: vin,
|
||||
Reason: "identifier already points to another vin",
|
||||
}, reportLimit)
|
||||
continue
|
||||
}
|
||||
report.Resolved++
|
||||
sourceStat.Resolved++
|
||||
if exists {
|
||||
if opts.Apply {
|
||||
if err := updateVehicleIdentifier(ctx, store, resolved); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.Updated++
|
||||
sourceStat.Updated++
|
||||
} else {
|
||||
report.WouldUpdate++
|
||||
sourceStat.WouldUpdate++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if opts.Apply {
|
||||
if err := upsertVehicle(ctx, store, resolved); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := insertVehicleIdentifier(ctx, store, resolved); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.Inserted++
|
||||
sourceStat.Inserted++
|
||||
} else {
|
||||
report.WouldInsert++
|
||||
sourceStat.WouldInsert++
|
||||
}
|
||||
}
|
||||
if tx != nil {
|
||||
if err := tx.Commit(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
tx = nil
|
||||
}
|
||||
report.SourceResults = sortedMappingSourceImportStats(sourceStats)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func sourceImportStat(stats map[string]*MappingSourceImportStat, record MappingRecord) *MappingSourceImportStat {
|
||||
sourceCode := strings.TrimSpace(record.SourceCode)
|
||||
if sourceCode == "" {
|
||||
sourceCode = "unknown"
|
||||
}
|
||||
stat := stats[sourceCode]
|
||||
if stat != nil {
|
||||
if stat.SourceName == "" {
|
||||
stat.SourceName = strings.TrimSpace(record.SourceName)
|
||||
}
|
||||
return stat
|
||||
}
|
||||
stat = &MappingSourceImportStat{
|
||||
SourceCode: sourceCode,
|
||||
SourceName: strings.TrimSpace(record.SourceName),
|
||||
}
|
||||
stats[sourceCode] = stat
|
||||
return stat
|
||||
}
|
||||
|
||||
func sortedMappingSourceImportStats(stats map[string]*MappingSourceImportStat) []MappingSourceImportStat {
|
||||
if len(stats) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(stats))
|
||||
for key := range stats {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]MappingSourceImportStat, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
out = append(out, *stats[key])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendUnresolvedItem(report *MappingImportReport, record MappingRecord, limit int) {
|
||||
if report == nil {
|
||||
return
|
||||
}
|
||||
if limit >= 0 && len(report.UnresolvedItems) >= limit {
|
||||
return
|
||||
}
|
||||
report.UnresolvedItems = append(report.UnresolvedItems, record)
|
||||
}
|
||||
|
||||
func appendConflictItem(report *MappingImportReport, conflict MappingConflict, limit int) {
|
||||
if report == nil {
|
||||
return
|
||||
}
|
||||
if limit >= 0 && len(report.ConflictItems) >= limit {
|
||||
return
|
||||
}
|
||||
report.ConflictItems = append(report.ConflictItems, conflict)
|
||||
}
|
||||
|
||||
func dedupeMappingRecords(records []MappingRecord) ([]MappingRecord, []MappingConflict) {
|
||||
seen := map[string]MappingRecord{}
|
||||
indexByKey := map[string]int{}
|
||||
var out []MappingRecord
|
||||
var conflicts []MappingConflict
|
||||
for _, record := range records {
|
||||
record.IdentifierValue = normalizeIdentifierValue(record.IdentifierType, record.IdentifierValue)
|
||||
record.Plate = normalizePlate(record.Plate)
|
||||
if record.Protocol == "" {
|
||||
record.Protocol = "JT808"
|
||||
}
|
||||
if record.IdentifierValue == "" || record.IdentifierType == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.Join([]string{record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue}, "\x00")
|
||||
existing, ok := seen[key]
|
||||
if !ok {
|
||||
seen[key] = record
|
||||
indexByKey[key] = len(out)
|
||||
out = append(out, record)
|
||||
continue
|
||||
}
|
||||
if existing.Plate != "" && record.Plate != "" && existing.Plate != record.Plate {
|
||||
conflicts = append(conflicts, MappingConflict{
|
||||
Record: record,
|
||||
Reason: fmt.Sprintf("same source identifier maps to multiple plates: %s/%s", existing.Plate, record.Plate),
|
||||
})
|
||||
continue
|
||||
}
|
||||
merged := mergeMappingRecord(existing, record)
|
||||
seen[key] = merged
|
||||
if index, ok := indexByKey[key]; ok && index >= 0 && index < len(out) {
|
||||
out[index] = merged
|
||||
}
|
||||
}
|
||||
return out, conflicts
|
||||
}
|
||||
|
||||
func mergeMappingRecord(existing MappingRecord, incoming MappingRecord) MappingRecord {
|
||||
merged := existing
|
||||
if merged.File == "" {
|
||||
merged.File = incoming.File
|
||||
}
|
||||
if merged.Sheet == "" {
|
||||
merged.Sheet = incoming.Sheet
|
||||
}
|
||||
if merged.Row == 0 {
|
||||
merged.Row = incoming.Row
|
||||
}
|
||||
if merged.SourceName == "" {
|
||||
merged.SourceName = incoming.SourceName
|
||||
}
|
||||
if merged.Protocol == "" {
|
||||
merged.Protocol = incoming.Protocol
|
||||
}
|
||||
if merged.IdentifierType == "" {
|
||||
merged.IdentifierType = incoming.IdentifierType
|
||||
}
|
||||
if merged.IdentifierValue == "" {
|
||||
merged.IdentifierValue = incoming.IdentifierValue
|
||||
}
|
||||
if merged.RawValue == "" {
|
||||
merged.RawValue = incoming.RawValue
|
||||
}
|
||||
if merged.Plate == "" {
|
||||
merged.Plate = incoming.Plate
|
||||
}
|
||||
if merged.OEM == "" {
|
||||
merged.OEM = incoming.OEM
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func resolveMappingVIN(ctx context.Context, db mappingStore, legacyTable string, record MappingRecord) (string, error) {
|
||||
if record.Plate != "" {
|
||||
vin, err := lookupLegacyVIN(ctx, db, legacyTable, "plate", record.Plate)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return "", err
|
||||
}
|
||||
if vin != "" {
|
||||
return vin, nil
|
||||
}
|
||||
}
|
||||
if record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue != "" {
|
||||
vin, err := lookupLegacyVIN(ctx, db, legacyTable, "phone", record.IdentifierValue)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return "", err
|
||||
}
|
||||
return vin, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func lookupLegacyVIN(ctx context.Context, db mappingStore, table string, column string, value string) (string, error) {
|
||||
if !safeIdentifier(table) || !safeIdentifier(column) {
|
||||
return "", sql.ErrNoRows
|
||||
}
|
||||
query := "SELECT vin FROM " + table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' LIMIT 1"
|
||||
var vin string
|
||||
err := db.QueryRowContext(ctx, query, value).Scan(&vin)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(vin), nil
|
||||
}
|
||||
|
||||
func existingIdentifierVIN(ctx context.Context, db mappingStore, record resolvedMappingRecord) (string, bool, error) {
|
||||
var vin string
|
||||
err := db.QueryRowContext(ctx, `SELECT vin FROM vehicle_identifier
|
||||
WHERE protocol = ? AND source_code = ? AND identifier_type = ? AND identifier_value = ?`,
|
||||
record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue).Scan(&vin)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return strings.TrimSpace(vin), true, nil
|
||||
}
|
||||
|
||||
func upsertVehicle(ctx context.Context, db mappingStore, record resolvedMappingRecord) error {
|
||||
_, err := db.ExecContext(ctx, `INSERT INTO vehicle (vin, plate, oem, enabled)
|
||||
VALUES (?, ?, ?, 1)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
|
||||
oem = IF(VALUES(oem) <> '', VALUES(oem), oem),
|
||||
enabled = 1`,
|
||||
record.VIN, record.Plate, record.OEM)
|
||||
return err
|
||||
}
|
||||
|
||||
func insertVehicleIdentifier(ctx context.Context, db mappingStore, record resolvedMappingRecord) error {
|
||||
_, err := db.ExecContext(ctx, `INSERT INTO vehicle_identifier
|
||||
(protocol, source_code, identifier_type, identifier_value, vin, plate, oem, raw_value, enabled, latest_import_file)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`,
|
||||
record.Protocol,
|
||||
record.SourceCode,
|
||||
record.IdentifierType,
|
||||
record.IdentifierValue,
|
||||
record.VIN,
|
||||
record.Plate,
|
||||
record.OEM,
|
||||
record.RawValue,
|
||||
record.File,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func updateVehicleIdentifier(ctx context.Context, db mappingStore, record resolvedMappingRecord) error {
|
||||
_, err := db.ExecContext(ctx, `UPDATE vehicle_identifier
|
||||
SET plate = IF(? <> '', ?, plate),
|
||||
oem = IF(? <> '', ?, oem),
|
||||
raw_value = IF(? <> '', ?, raw_value),
|
||||
latest_import_file = ?,
|
||||
enabled = 1
|
||||
WHERE protocol = ? AND source_code = ? AND identifier_type = ? AND identifier_value = ?`,
|
||||
record.Plate, record.Plate,
|
||||
record.OEM, record.OEM,
|
||||
record.RawValue, record.RawValue,
|
||||
record.File,
|
||||
record.Protocol,
|
||||
record.SourceCode,
|
||||
record.IdentifierType,
|
||||
record.IdentifierValue,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func mappingSource(root string, path string) (string, string) {
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
rel = filepath.Base(path)
|
||||
}
|
||||
parts := strings.Split(filepath.ToSlash(rel), "/")
|
||||
name := strings.TrimSpace(parts[0])
|
||||
if name == "" || strings.EqualFold(name, ".") {
|
||||
name = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
||||
}
|
||||
code := sourceCode(name)
|
||||
return code, name
|
||||
}
|
||||
|
||||
func sourceCode(name string) string {
|
||||
switch strings.TrimSpace(strings.ToLower(name)) {
|
||||
case "g7s":
|
||||
return "g7s"
|
||||
case "信达":
|
||||
return "xinda"
|
||||
case "广安北斗", "广安车联":
|
||||
return "guangan_beidou"
|
||||
case "东方北斗":
|
||||
return "dongfang_beidou"
|
||||
case "赛格":
|
||||
return "saige"
|
||||
default:
|
||||
return normalizeASCIIKey(name)
|
||||
}
|
||||
}
|
||||
|
||||
func mappingHeader(rows [][]string) (map[string]int, int) {
|
||||
for index, row := range rows {
|
||||
header := map[string]int{}
|
||||
for columnIndex, value := range row {
|
||||
key := normalizeHeader(value)
|
||||
switch key {
|
||||
case "车牌", "车牌号", "车牌号码":
|
||||
header["plate"] = columnIndex
|
||||
case "sim", "sim卡号", "手机号", "终端手机号", "终端id", "终端标识":
|
||||
header["phone"] = columnIndex
|
||||
}
|
||||
}
|
||||
if len(header) > 0 {
|
||||
return header, index + 1
|
||||
}
|
||||
if index >= 5 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
func cellByHeader(row []string, header map[string]int, key string) string {
|
||||
if len(header) == 0 {
|
||||
return ""
|
||||
}
|
||||
index, ok := header[key]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return cell(row, index)
|
||||
}
|
||||
|
||||
func cell(row []string, index int) string {
|
||||
if index < 0 || index >= len(row) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(row[index])
|
||||
}
|
||||
|
||||
func normalizeHeader(value string) string {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
value = strings.ReplaceAll(value, " ", "")
|
||||
value = strings.ReplaceAll(value, "\t", "")
|
||||
value = strings.ReplaceAll(value, "（", "(")
|
||||
value = strings.ReplaceAll(value, "）", ")")
|
||||
return value
|
||||
}
|
||||
|
||||
func normalizePlate(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
value = strings.ReplaceAll(value, " ", "")
|
||||
value = strings.ReplaceAll(value, "\t", "")
|
||||
return value
|
||||
}
|
||||
|
||||
func normalizeMappingPhone(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.ContainsAny(value, ".eE") {
|
||||
if parsed, err := strconv.ParseFloat(value, 64); err == nil && parsed > 0 {
|
||||
return normalizePhone(strconv.FormatFloat(parsed, 'f', 0, 64))
|
||||
}
|
||||
}
|
||||
digits := digitPattern.ReplaceAllString(value, "")
|
||||
return normalizePhone(digits)
|
||||
}
|
||||
|
||||
func normalizeASCIIKey(value string) string {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
var b strings.Builder
|
||||
lastUnderscore := false
|
||||
for _, r := range value {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
lastUnderscore = false
|
||||
continue
|
||||
}
|
||||
if !lastUnderscore {
|
||||
b.WriteByte('_')
|
||||
lastUnderscore = true
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), "_")
|
||||
}
|
||||
Reference in New Issue
Block a user