feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
426
go/vehicle-gateway/cmd/identity-import/main_test.go
Normal file
426
go/vehicle-gateway/cmd/identity-import/main_test.go
Normal file
@@ -0,0 +1,426 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
)
|
||||
|
||||
func TestReportItemLimitExpandsWhenCSVOutputIsRequested(t *testing.T) {
|
||||
if got := reportItemLimit(50, "", ""); got != 50 {
|
||||
t.Fatalf("reportItemLimit without csv = %d, want 50", got)
|
||||
}
|
||||
if got := reportItemLimit(50, "unresolved.csv", ""); got != -1 {
|
||||
t.Fatalf("reportItemLimit with unresolved csv = %d, want -1", got)
|
||||
}
|
||||
if got := reportItemLimit(50, "", "conflicts.csv"); got != -1 {
|
||||
t.Fatalf("reportItemLimit with conflicts csv = %d, want -1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedMappingFilesError(t *testing.T) {
|
||||
err := unsupportedMappingFilesError(identity.MappingScanReport{})
|
||||
if err != nil {
|
||||
t.Fatalf("unsupportedMappingFilesError(empty) = %v", err)
|
||||
}
|
||||
err = unsupportedMappingFilesError(identity.MappingScanReport{
|
||||
UnsupportedFiles: 2,
|
||||
UnsupportedItems: []identity.MappingUnsupportedFileReport{
|
||||
{File: "G7s/legacy.xls", Ext: ".xls"},
|
||||
{File: "信达/legacy.xlsb", Ext: ".xlsb"},
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("unsupportedMappingFilesError() nil, want error")
|
||||
}
|
||||
text := err.Error()
|
||||
for _, want := range []string{"2 unsupported workbook", "G7s/legacy.xls", "信达/legacy.xlsb", "before -apply"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("error missing %q: %s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMappingApplyInputAllowsDryRunButBlocksApply(t *testing.T) {
|
||||
scan := identity.MappingScanReport{
|
||||
UnsupportedFiles: 1,
|
||||
UnsupportedItems: []identity.MappingUnsupportedFileReport{
|
||||
{File: "G7s/legacy.xls", Ext: ".xls"},
|
||||
},
|
||||
}
|
||||
if err := validateMappingApplyInput(false, scan); err != nil {
|
||||
t.Fatalf("validateMappingApplyInput(dry-run) = %v", err)
|
||||
}
|
||||
if err := validateMappingApplyInput(true, scan); err == nil {
|
||||
t.Fatal("validateMappingApplyInput(apply) nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteUnresolvedCSV(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "unresolved.csv")
|
||||
err := writeUnresolvedCSV(path, []identity.MappingRecord{
|
||||
{
|
||||
File: "G7s/example.xlsx",
|
||||
Sheet: "Sheet1",
|
||||
Row: 2,
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: identity.IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
RawValue: "013307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("writeUnresolvedCSV() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
text := string(data)
|
||||
for _, want := range []string{"identifier_type", "JT808_PHONE", "13307795425", "vin_not_found_in_legacy_binding"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("csv missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteConflictsCSV(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "conflicts.csv")
|
||||
err := writeConflictsCSV(path, []identity.MappingConflict{
|
||||
{
|
||||
Record: identity.MappingRecord{
|
||||
File: "source.xlsx",
|
||||
Sheet: "Sheet1",
|
||||
Row: 3,
|
||||
SourceCode: "xinda",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: identity.IdentifierTypePlate,
|
||||
IdentifierValue: "粤AG18312",
|
||||
Plate: "粤AG18312",
|
||||
},
|
||||
ExistingVIN: "VIN001",
|
||||
NewVIN: "VIN002",
|
||||
Reason: "identifier already points to another vin",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("writeConflictsCSV() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
text := string(data)
|
||||
for _, want := range []string{"existing_vin", "VIN001", "VIN002", "identifier already points to another vin"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("csv missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncJT808DataSourcesFromIdentifiersPreservesManualPlatformNames(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(3))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2))
|
||||
mock.ExpectExec("INSERT INTO vehicle_data_source").
|
||||
WillReturnResult(driver.RowsAffected(3))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(5))
|
||||
mock.ExpectExec("UPDATE vehicle_data_source").
|
||||
WillReturnResult(driver.RowsAffected(5))
|
||||
mock.ExpectQuery("SELECT DISTINCT s.vin, s.stat_date, s.protocol").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol"}).
|
||||
AddRow("LNXNEGRRXSR319449", "2026-07-13", "JT808"))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage").
|
||||
WillReturnResult(driver.RowsAffected(1))
|
||||
mock.ExpectExec("UPDATE vehicle_daily_mileage_source s").
|
||||
WillReturnResult(driver.RowsAffected(1))
|
||||
mock.ExpectExec("DELETE FROM vehicle_daily_mileage").
|
||||
WillReturnResult(driver.RowsAffected(0))
|
||||
mock.ExpectCommit()
|
||||
|
||||
report, err := syncJT808DataSourcesFromIdentifiers(context.Background(), db, true)
|
||||
if err != nil {
|
||||
t.Fatalf("syncJT808DataSourcesFromIdentifiers() error = %v", err)
|
||||
}
|
||||
if !report.Apply || report.CandidateSources != 3 || report.SkippedSources != 1 || report.ConflictingSources != 2 || report.Synced != 3 || report.PlatformKindCandidates != 5 || report.PlatformKindClassified != 5 || report.ReprojectDailyMileageTargets != 1 || report.ReprojectedDailyMileageTargets != 1 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"COUNT(DISTINCT",
|
||||
"vehicle_identifier vi",
|
||||
"TRIM(r.source_ip) AS source_ip",
|
||||
"GROUP BY TRIM(r.source_ip)",
|
||||
"inferred.source_code",
|
||||
"'PLATFORM'",
|
||||
"inferred.source_code_count = 1",
|
||||
"vehicle_data_source.source_code IS NULL OR TRIM(vehicle_data_source.source_code) = ''",
|
||||
"ELSE vehicle_data_source.source_code",
|
||||
"vehicle_data_source.source_kind IS NULL OR TRIM(vehicle_data_source.source_kind) = '' OR vehicle_data_source.source_kind = 'UNKNOWN'",
|
||||
"ELSE vehicle_data_source.source_kind",
|
||||
"vehicle_data_source.enabled = 0",
|
||||
"vehicle_data_source.remark LIKE 'auto-retired:%'",
|
||||
"vehicle_data_source.remark = 'auto-reenabled: source evidence restored'",
|
||||
"THEN 'auto-reenabled: source evidence restored'",
|
||||
"THEN 1",
|
||||
"LEFT JOIN vehicle_data_source ds",
|
||||
"ds.id IS NULL",
|
||||
} {
|
||||
if !strings.Contains(syncJT808DataSourcesSQL, want) {
|
||||
t.Fatalf("sync sql missing %q:\n%s", want, syncJT808DataSourcesSQL)
|
||||
}
|
||||
}
|
||||
if strings.Index(syncJT808DataSourcesSQL, "enabled = CASE") < 0 ||
|
||||
strings.Index(syncJT808DataSourcesSQL, "remark = CASE") < 0 ||
|
||||
strings.Index(syncJT808DataSourcesSQL, "enabled = CASE") > strings.Index(syncJT808DataSourcesSQL, "remark = CASE") {
|
||||
t.Fatalf("sync sql must restore enabled before updating remark because MySQL evaluates assignments in order:\n%s", syncJT808DataSourcesSQL)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"LEFT JOIN vehicle_data_source ds",
|
||||
"ds.id IS NULL",
|
||||
"ds.source_code IS NULL OR TRIM(ds.source_code) = ''",
|
||||
"ds.enabled = 0 AND (ds.remark LIKE 'auto-retired:%'",
|
||||
"ds.remark = 'auto-reenabled: source evidence restored'",
|
||||
} {
|
||||
if !strings.Contains(syncJT808DataSourcesCandidateCountSQL, want) {
|
||||
t.Fatalf("candidate count sql missing %q:\n%s", want, syncJT808DataSourcesCandidateCountSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"JOIN vehicle_data_source ds",
|
||||
"ds.source_code <> inferred.source_code",
|
||||
} {
|
||||
if !strings.Contains(syncJT808DataSourcesConflictCountSQL, want) {
|
||||
t.Fatalf("conflict count sql missing %q:\n%s", want, syncJT808DataSourcesConflictCountSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"vehicle_data_source.platform_name IS NULL OR TRIM(vehicle_data_source.platform_name) = ''",
|
||||
"ELSE vehicle_data_source.platform_name",
|
||||
} {
|
||||
if !strings.Contains(syncJT808DataSourcesSQL, want) {
|
||||
t.Fatalf("sync sql missing %q:\n%s", want, syncJT808DataSourcesSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"source_kind IS NULL OR TRIM(source_kind) = '' OR source_kind = 'UNKNOWN'",
|
||||
"source_code IS NOT NULL AND TRIM(source_code) <> ''",
|
||||
"platform_name IS NOT NULL AND TRIM(platform_name) <> ''",
|
||||
"SET source_kind = 'PLATFORM'",
|
||||
} {
|
||||
if !strings.Contains(classifyConfiguredDataSourcesSQL, want) && !strings.Contains(classifyConfiguredDataSourcesCountSQL, want) {
|
||||
t.Fatalf("configured-source classification sql missing %q:\n%s\n%s", want, classifyConfiguredDataSourcesSQL, classifyConfiguredDataSourcesCountSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"SELECT DISTINCT s.vin, s.stat_date, s.protocol",
|
||||
"vehicle_daily_mileage_source s",
|
||||
"LEFT JOIN vehicle_data_source ds",
|
||||
"LEFT JOIN vehicle_daily_mileage m",
|
||||
"s.protocol = 'JT808'",
|
||||
"s.quality_status = 'OK'",
|
||||
"ds.enabled = 1",
|
||||
"m.vin IS NULL",
|
||||
"selected.is_selected = 1",
|
||||
} {
|
||||
if !strings.Contains(reprojectSelectableDailyMileageSourcesSQL, want) {
|
||||
t.Fatalf("reproject sql missing %q:\n%s", want, reprojectSelectableDailyMileageSourcesSQL)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncJT808DataSourcesDryRunDoesNotWrite(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(4))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(6))
|
||||
mock.ExpectQuery("SELECT DISTINCT s.vin, s.stat_date, s.protocol").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol"}).
|
||||
AddRow("LNXNEGRRXSR319449", "2026-07-13", "JT808"))
|
||||
|
||||
report, err := syncJT808DataSourcesFromIdentifiers(context.Background(), db, false)
|
||||
if err != nil {
|
||||
t.Fatalf("syncJT808DataSourcesFromIdentifiers() error = %v", err)
|
||||
}
|
||||
if report.Apply || report.CandidateSources != 4 || report.SkippedSources != 2 || report.ConflictingSources != 1 || report.PlatformKindCandidates != 6 || report.PlatformKindClassified != 0 || report.Synced != 0 || report.ReprojectDailyMileageTargets != 1 || report.ReprojectedDailyMileageTargets != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenanceModeNamesAllSourceMaintenanceSteps(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
apply bool
|
||||
sync bool
|
||||
prune bool
|
||||
retire bool
|
||||
want string
|
||||
}{
|
||||
{name: "dry run", want: "dry_run"},
|
||||
{name: "apply", apply: true, want: "apply"},
|
||||
{name: "sync dry run", sync: true, want: "sync_dry_run"},
|
||||
{name: "sync prune retire apply", apply: true, sync: true, prune: true, retire: true, want: "sync_prune_retire_apply"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := maintenanceMode(tt.apply, tt.sync, tt.prune, tt.retire); got != tt.want {
|
||||
t.Fatalf("maintenanceMode() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneUnmanagedDataSourcesDryRunDoesNotDelete(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WithArgs(int64(3600)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(339))
|
||||
|
||||
report, err := pruneUnmanagedDataSourcesWithoutEvidence(context.Background(), db, false, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("pruneUnmanagedDataSourcesWithoutEvidence() error = %v", err)
|
||||
}
|
||||
if report.Apply || report.MinAgeSeconds != 3600 || report.CandidateSources != 339 || report.Pruned != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"vehicle_data_source ds",
|
||||
"vehicle_daily_mileage m",
|
||||
"m.source_id = ds.id",
|
||||
"jt808_registration r",
|
||||
"r.source_ip = ds.source_ip",
|
||||
"ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN'",
|
||||
"TIMESTAMPDIFF(SECOND",
|
||||
} {
|
||||
if !strings.Contains(pruneUnmanagedDataSourcesCountSQL, want) {
|
||||
t.Fatalf("prune count sql missing %q:\n%s", want, pruneUnmanagedDataSourcesCountSQL)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetireStaleUnmanagedDataSourcesDryRunDoesNotDisable(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WithArgs(int64(86400)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(83))
|
||||
|
||||
report, err := retireStaleUnmanagedDataSourcesWithoutEvidence(context.Background(), db, false, 24*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("retireStaleUnmanagedDataSourcesWithoutEvidence() error = %v", err)
|
||||
}
|
||||
if report.Apply || report.MinAgeSeconds != 86400 || report.CandidateSources != 83 || report.Retired != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"vehicle_data_source ds",
|
||||
"ds.enabled = 1",
|
||||
"jt808_registration r",
|
||||
"r.source_ip = ds.source_ip",
|
||||
"ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN'",
|
||||
"TIMESTAMPDIFF(SECOND",
|
||||
} {
|
||||
if !strings.Contains(retireStaleUnmanagedDataSourcesCountSQL, want) {
|
||||
t.Fatalf("retire count sql missing %q:\n%s", want, retireStaleUnmanagedDataSourcesCountSQL)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetireStaleUnmanagedDataSourcesApplyDisablesOnlyCandidates(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WithArgs(int64(7200)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(7))
|
||||
mock.ExpectExec("UPDATE vehicle_data_source ds").
|
||||
WithArgs(int64(7200)).
|
||||
WillReturnResult(driver.RowsAffected(7))
|
||||
|
||||
report, err := retireStaleUnmanagedDataSourcesWithoutEvidence(context.Background(), db, true, 2*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("retireStaleUnmanagedDataSourcesWithoutEvidence() error = %v", err)
|
||||
}
|
||||
if !report.Apply || report.MinAgeSeconds != 7200 || report.CandidateSources != 7 || report.Retired != 7 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneUnmanagedDataSourcesApplyDeletesOnlyCandidates(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WithArgs(int64(1800)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(12))
|
||||
mock.ExpectExec("DELETE ds\\s+FROM vehicle_data_source ds").
|
||||
WithArgs(int64(1800)).
|
||||
WillReturnResult(driver.RowsAffected(12))
|
||||
|
||||
report, err := pruneUnmanagedDataSourcesWithoutEvidence(context.Background(), db, true, 30*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("pruneUnmanagedDataSourcesWithoutEvidence() error = %v", err)
|
||||
}
|
||||
if !report.Apply || report.MinAgeSeconds != 1800 || report.CandidateSources != 12 || report.Pruned != 12 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user