feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
345
go/vehicle-gateway/internal/identity/mapping_import_test.go
Normal file
345
go/vehicle-gateway/internal/identity/mapping_import_test.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
func TestReadMappingDirectoryExtractsPhoneAndPlate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeWorkbook(t, filepath.Join(dir, "G7s", "宇速全量.xlsx"), [][]string{
|
||||
{"车牌号", "sim卡号", "设备号"},
|
||||
{"粤AG18312", "013307795425", "DEV001"},
|
||||
})
|
||||
writeWorkbook(t, filepath.Join(dir, "东方北斗", "无标题0703.xlsx"), [][]string{
|
||||
{"沪A01559F", "64341233712"},
|
||||
})
|
||||
if err := os.MkdirAll(filepath.Join(dir, "信达"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "信达", "旧格式.xls"), []byte("legacy xls"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "信达", "说明.txt"), []byte("ignored"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
|
||||
records, report, err := ReadMappingDirectory(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadMappingDirectory() error = %v", err)
|
||||
}
|
||||
if report.Files != 2 {
|
||||
t.Fatalf("files = %d, report=%#v", report.Files, report)
|
||||
}
|
||||
if report.UnsupportedFiles != 1 || len(report.UnsupportedItems) != 1 {
|
||||
t.Fatalf("unsupported files = %d, items=%#v", report.UnsupportedFiles, report.UnsupportedItems)
|
||||
}
|
||||
if got := report.UnsupportedItems[0]; got.Ext != ".xls" || got.Reason == "" {
|
||||
t.Fatalf("unsupported item = %#v", got)
|
||||
}
|
||||
sources := map[string]MappingSourceScanReport{}
|
||||
for _, source := range report.Sources {
|
||||
sources[source.SourceCode] = source
|
||||
}
|
||||
if got := sources["g7s"]; got.Files != 1 || got.Sheets != 1 || got.PhoneRecords != 1 || got.PlateRecords != 1 || got.Records != 2 {
|
||||
t.Fatalf("g7s source report = %#v", got)
|
||||
}
|
||||
if got := sources["dongfang_beidou"]; got.Files != 1 || got.Sheets != 1 || got.PhoneRecords != 1 || got.PlateRecords != 1 || got.Records != 2 {
|
||||
t.Fatalf("dongfang source report = %#v", got)
|
||||
}
|
||||
var phoneSeen bool
|
||||
var plateSeen bool
|
||||
var headerlessSeen bool
|
||||
for _, record := range records {
|
||||
if record.SourceCode == "g7s" && record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue == "13307795425" && record.Plate == "粤AG18312" {
|
||||
phoneSeen = true
|
||||
}
|
||||
if record.SourceCode == "g7s" && record.IdentifierType == IdentifierTypePlate && record.IdentifierValue == "粤AG18312" {
|
||||
plateSeen = true
|
||||
}
|
||||
if record.SourceCode == "dongfang_beidou" && record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue == "64341233712" && record.Plate == "沪A01559F" {
|
||||
headerlessSeen = true
|
||||
}
|
||||
}
|
||||
if !phoneSeen || !plateSeen || !headerlessSeen {
|
||||
t.Fatalf("records missing expected mappings: %#v", records)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsDryRunResolvesVINFromLegacyPlate(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "013307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportMappingRecords() error = %v", err)
|
||||
}
|
||||
if report.Resolved != 1 || report.WouldInsert != 1 || report.Unresolved != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if len(report.SourceResults) != 1 {
|
||||
t.Fatalf("source results = %#v, want one source", report.SourceResults)
|
||||
}
|
||||
if got := report.SourceResults[0]; got.SourceCode != "g7s" || got.Records != 1 || got.Deduplicated != 1 || got.Resolved != 1 || got.WouldInsert != 1 {
|
||||
t.Fatalf("source result = %#v", got)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsReportsSourceResults(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤B00000").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
|
||||
WithArgs("14400000000").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
{
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
Plate: "粤B99999",
|
||||
OEM: "G7s",
|
||||
},
|
||||
{
|
||||
SourceCode: "xinda",
|
||||
SourceName: "信达",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "14400000000",
|
||||
Plate: "粤B00000",
|
||||
OEM: "信达",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportMappingRecords() error = %v", err)
|
||||
}
|
||||
if report.Records != 3 || report.Deduplicated != 2 || report.Resolved != 1 || report.Unresolved != 1 || report.Conflicts != 1 || report.WouldInsert != 1 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
got := map[string]MappingSourceImportStat{}
|
||||
for _, source := range report.SourceResults {
|
||||
got[source.SourceCode] = source
|
||||
}
|
||||
if source := got["g7s"]; source.Records != 2 || source.Deduplicated != 1 || source.Resolved != 1 || source.Conflicts != 1 || source.WouldInsert != 1 {
|
||||
t.Fatalf("g7s source result = %#v", source)
|
||||
}
|
||||
if source := got["xinda"]; source.Records != 1 || source.Deduplicated != 1 || source.Unresolved != 1 {
|
||||
t.Fatalf("xinda source result = %#v", source)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsMergesDuplicateIdentifierDetails(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
File: "G7s/no-plate.xlsx",
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
OEM: "G7s",
|
||||
},
|
||||
{
|
||||
File: "G7s/with-plate.xlsx",
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportMappingRecords() error = %v", err)
|
||||
}
|
||||
if report.Records != 2 || report.Deduplicated != 1 || report.Resolved != 1 || report.WouldInsert != 1 || report.Unresolved != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsApplyCommitsSingleTransaction(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
mock.ExpectExec("INSERT INTO vehicle").
|
||||
WithArgs("LB9A32A22P0LS1230", "粤AG18312", "G7s").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("INSERT INTO vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425", "LB9A32A22P0LS1230", "粤AG18312", "G7s", "13307795425", "G7s/example.xlsx").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
File: "G7s/example.xlsx",
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
RawValue: "13307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{
|
||||
Apply: true,
|
||||
LegacyTable: "vehicle_identity_binding",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportMappingRecords() error = %v", err)
|
||||
}
|
||||
if report.Inserted != 1 || report.Resolved != 1 || report.WouldInsert != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingRecordsApplyRollsBackOnWriteError(t *testing.T) {
|
||||
db, mock := newMockDB(t)
|
||||
defer db.Close()
|
||||
errWrite := errors.New("insert vehicle failed")
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?").
|
||||
WithArgs("粤AG18312").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230"))
|
||||
mock.ExpectQuery("SELECT vin FROM vehicle_identifier").
|
||||
WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin"}))
|
||||
mock.ExpectExec("INSERT INTO vehicle").
|
||||
WithArgs("LB9A32A22P0LS1230", "粤AG18312", "G7s").
|
||||
WillReturnError(errWrite)
|
||||
mock.ExpectRollback()
|
||||
|
||||
report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{
|
||||
{
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
RawValue: "13307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
}, MappingScanReport{}, MappingImportOptions{
|
||||
Apply: true,
|
||||
LegacyTable: "vehicle_identity_binding",
|
||||
})
|
||||
if !errors.Is(err, errWrite) {
|
||||
t.Fatalf("ImportMappingRecords() error = %v, want %v", err, errWrite)
|
||||
}
|
||||
if report.Inserted != 0 || report.Resolved != 1 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMappingPhoneHandlesExcelNumericFormats(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"013307795425": "13307795425",
|
||||
"13307795425.0": "13307795425",
|
||||
"1.3307795425E10": "13307795425",
|
||||
}
|
||||
for input, want := range tests {
|
||||
if got := normalizeMappingPhone(input); got != want {
|
||||
t.Fatalf("normalizeMappingPhone(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeWorkbook(t *testing.T, path string, rows [][]string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
workbook := excelize.NewFile()
|
||||
sheet := "Sheet1"
|
||||
for rowIndex, row := range rows {
|
||||
for columnIndex, value := range row {
|
||||
cellName, err := excelize.CoordinatesToCellName(columnIndex+1, rowIndex+1)
|
||||
if err != nil {
|
||||
t.Fatalf("CoordinatesToCellName() error = %v", err)
|
||||
}
|
||||
if err := workbook.SetCellValue(sheet, cellName, value); err != nil {
|
||||
t.Fatalf("SetCellValue() error = %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := workbook.SaveAs(path); err != nil {
|
||||
t.Fatalf("SaveAs() error = %v", err)
|
||||
}
|
||||
if err := workbook.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user