feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
695
go/vehicle-gateway/cmd/identity-import/main.go
Normal file
695
go/vehicle-gateway/cmd/identity-import/main.go
Normal file
@@ -0,0 +1,695 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var input string
|
||||
var dsn string
|
||||
var legacyTable string
|
||||
var apply bool
|
||||
var ensureSchema bool
|
||||
var reportLimit int
|
||||
var unresolvedOut string
|
||||
var conflictsOut string
|
||||
var syncDataSources bool
|
||||
var pruneUnmanagedDataSources bool
|
||||
var pruneDataSourceMinAge time.Duration
|
||||
var retireStaleUnmanagedDataSources bool
|
||||
var retireDataSourceMinAge time.Duration
|
||||
var timeout time.Duration
|
||||
flag.StringVar(&input, "input", env("IDENTITY_MAPPING_INPUT", ""), "directory that contains 808 plate/phone mapping workbooks")
|
||||
flag.StringVar(&dsn, "mysql-dsn", env("IDENTITY_MYSQL_DSN", env("MYSQL_DSN", "")), "MySQL DSN; empty means scan only")
|
||||
flag.StringVar(&legacyTable, "legacy-table", env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding"), "legacy VIN/plate binding table used to resolve VIN")
|
||||
flag.BoolVar(&apply, "apply", false, "write resolved mappings to vehicle and vehicle_identifier")
|
||||
flag.BoolVar(&ensureSchema, "ensure-schema", false, "create vehicle and vehicle_identifier tables before import")
|
||||
flag.IntVar(&reportLimit, "report-limit", 50, "max unresolved/conflict items embedded in JSON; use -1 for all")
|
||||
flag.StringVar(&unresolvedOut, "unresolved-out", "", "optional CSV path for all unresolved mappings")
|
||||
flag.StringVar(&conflictsOut, "conflicts-out", "", "optional CSV path for all conflict mappings")
|
||||
flag.BoolVar(&syncDataSources, "sync-data-sources", false, "infer JT808 vehicle_data_source platform names from jt808_registration and vehicle_identifier")
|
||||
flag.BoolVar(&pruneUnmanagedDataSources, "prune-unmanaged-data-sources", false, "delete stale unclassified vehicle_data_source rows that have no registration evidence and are not referenced by final mileage")
|
||||
flag.DurationVar(&pruneDataSourceMinAge, "prune-data-source-min-age", time.Hour, "minimum latest_seen/updated age before -prune-unmanaged-data-sources can delete rows")
|
||||
flag.BoolVar(&retireStaleUnmanagedDataSources, "retire-stale-unmanaged-data-sources", false, "disable stale unclassified vehicle_data_source rows that have no registration evidence but may be referenced by historical mileage")
|
||||
flag.DurationVar(&retireDataSourceMinAge, "retire-data-source-min-age", 24*time.Hour, "minimum latest_seen/updated age before -retire-stale-unmanaged-data-sources can disable rows")
|
||||
flag.DurationVar(&timeout, "timeout", 2*time.Minute, "import timeout")
|
||||
flag.Parse()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
input = strings.TrimSpace(input)
|
||||
output := map[string]any{}
|
||||
var records []identity.MappingRecord
|
||||
var scan identity.MappingScanReport
|
||||
if input != "" {
|
||||
var err error
|
||||
records, scan, err = identity.ReadMappingDirectory(input)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
output["input"] = input
|
||||
output["scan"] = scan
|
||||
} else if !syncDataSources && !pruneUnmanagedDataSources && !retireStaleUnmanagedDataSources {
|
||||
fail(errors.New("-input is required unless -sync-data-sources, -prune-unmanaged-data-sources or -retire-stale-unmanaged-data-sources is set"))
|
||||
}
|
||||
if err := validateMappingApplyInput(apply, scan); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
if strings.TrimSpace(dsn) == "" {
|
||||
if syncDataSources || pruneUnmanagedDataSources || retireStaleUnmanagedDataSources {
|
||||
fail(errors.New("-mysql-dsn is required when -sync-data-sources, -prune-unmanaged-data-sources or -retire-stale-unmanaged-data-sources is set"))
|
||||
}
|
||||
output["mode"] = "scan_only"
|
||||
writeJSON(output)
|
||||
return
|
||||
}
|
||||
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
if input != "" && (ensureSchema || apply) {
|
||||
if err := identity.EnsureVehicleIdentifierSchema(ctx, db); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
}
|
||||
if input != "" {
|
||||
report, err := identity.ImportMappingRecords(ctx, db, records, scan, identity.MappingImportOptions{
|
||||
Apply: apply,
|
||||
LegacyTable: legacyTable,
|
||||
ReportItemLimit: reportItemLimit(reportLimit, unresolvedOut, conflictsOut),
|
||||
})
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
if strings.TrimSpace(unresolvedOut) != "" {
|
||||
if err := writeUnresolvedCSV(unresolvedOut, report.UnresolvedItems); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(conflictsOut) != "" {
|
||||
if err := writeConflictsCSV(conflictsOut, report.ConflictItems); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
}
|
||||
output["report"] = report
|
||||
output["mode"] = map[bool]string{true: "apply", false: "dry_run"}[apply]
|
||||
}
|
||||
if syncDataSources {
|
||||
syncReport, err := syncJT808DataSourcesFromIdentifiers(ctx, db, apply)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
output["data_source_sync"] = syncReport
|
||||
}
|
||||
if pruneUnmanagedDataSources {
|
||||
pruneReport, err := pruneUnmanagedDataSourcesWithoutEvidence(ctx, db, apply, pruneDataSourceMinAge)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
output["data_source_prune"] = pruneReport
|
||||
}
|
||||
if retireStaleUnmanagedDataSources {
|
||||
retireReport, err := retireStaleUnmanagedDataSourcesWithoutEvidence(ctx, db, apply, retireDataSourceMinAge)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
output["data_source_retire"] = retireReport
|
||||
}
|
||||
if input == "" {
|
||||
output["mode"] = maintenanceMode(apply, syncDataSources, pruneUnmanagedDataSources, retireStaleUnmanagedDataSources)
|
||||
}
|
||||
writeJSON(output)
|
||||
}
|
||||
|
||||
func maintenanceMode(apply bool, syncDataSources bool, pruneUnmanagedDataSources bool, retireStaleUnmanagedDataSources bool) string {
|
||||
var parts []string
|
||||
if syncDataSources {
|
||||
parts = append(parts, "sync")
|
||||
}
|
||||
if pruneUnmanagedDataSources {
|
||||
parts = append(parts, "prune")
|
||||
}
|
||||
if retireStaleUnmanagedDataSources {
|
||||
parts = append(parts, "retire")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return map[bool]string{true: "apply", false: "dry_run"}[apply]
|
||||
}
|
||||
parts = append(parts, map[bool]string{true: "apply", false: "dry_run"}[apply])
|
||||
return strings.Join(parts, "_")
|
||||
}
|
||||
|
||||
func reportItemLimit(limit int, unresolvedOut string, conflictsOut string) int {
|
||||
if strings.TrimSpace(unresolvedOut) != "" || strings.TrimSpace(conflictsOut) != "" {
|
||||
return -1
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func validateMappingApplyInput(apply bool, scan identity.MappingScanReport) error {
|
||||
if !apply {
|
||||
return nil
|
||||
}
|
||||
return unsupportedMappingFilesError(scan)
|
||||
}
|
||||
|
||||
func unsupportedMappingFilesError(scan identity.MappingScanReport) error {
|
||||
if scan.UnsupportedFiles <= 0 {
|
||||
return nil
|
||||
}
|
||||
items := make([]string, 0, len(scan.UnsupportedItems))
|
||||
for _, item := range scan.UnsupportedItems {
|
||||
path := strings.TrimSpace(item.File)
|
||||
if path == "" {
|
||||
path = strings.TrimSpace(item.Ext)
|
||||
}
|
||||
if path != "" {
|
||||
items = append(items, path)
|
||||
}
|
||||
if len(items) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
suffix := ""
|
||||
if len(items) > 0 {
|
||||
suffix = ": " + strings.Join(items, ", ")
|
||||
}
|
||||
return fmt.Errorf("identity mapping import has %d unsupported workbook(s); convert .xls/.xlsb to .xlsx before -apply%s", scan.UnsupportedFiles, suffix)
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func writeJSON(value any) {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(value); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeUnresolvedCSV(path string, items []identity.MappingRecord) error {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
writer := csv.NewWriter(file)
|
||||
defer writer.Flush()
|
||||
if err := writer.Write([]string{
|
||||
"file", "sheet", "row", "source_code", "source_name", "protocol",
|
||||
"identifier_type", "identifier_value", "raw_value", "plate", "oem", "reason",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
if err := writer.Write([]string{
|
||||
item.File,
|
||||
item.Sheet,
|
||||
fmt.Sprint(item.Row),
|
||||
item.SourceCode,
|
||||
item.SourceName,
|
||||
item.Protocol,
|
||||
item.IdentifierType,
|
||||
item.IdentifierValue,
|
||||
item.RawValue,
|
||||
item.Plate,
|
||||
item.OEM,
|
||||
"vin_not_found_in_legacy_binding",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writer.Error()
|
||||
}
|
||||
|
||||
func writeConflictsCSV(path string, items []identity.MappingConflict) error {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
writer := csv.NewWriter(file)
|
||||
defer writer.Flush()
|
||||
if err := writer.Write([]string{
|
||||
"file", "sheet", "row", "source_code", "source_name", "protocol",
|
||||
"identifier_type", "identifier_value", "raw_value", "plate", "oem",
|
||||
"existing_vin", "new_vin", "reason",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
record := item.Record
|
||||
if err := writer.Write([]string{
|
||||
record.File,
|
||||
record.Sheet,
|
||||
fmt.Sprint(record.Row),
|
||||
record.SourceCode,
|
||||
record.SourceName,
|
||||
record.Protocol,
|
||||
record.IdentifierType,
|
||||
record.IdentifierValue,
|
||||
record.RawValue,
|
||||
record.Plate,
|
||||
record.OEM,
|
||||
item.ExistingVIN,
|
||||
item.NewVIN,
|
||||
item.Reason,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writer.Error()
|
||||
}
|
||||
|
||||
type dataSourceSyncReport struct {
|
||||
Apply bool `json:"apply"`
|
||||
CandidateSources int64 `json:"candidate_sources"`
|
||||
SkippedSources int64 `json:"skipped_sources"`
|
||||
ConflictingSources int64 `json:"conflicting_sources"`
|
||||
PlatformKindCandidates int64 `json:"platform_kind_candidates"`
|
||||
PlatformKindClassified int64 `json:"platform_kind_classified,omitempty"`
|
||||
ReprojectDailyMileageTargets int64 `json:"reproject_daily_mileage_targets,omitempty"`
|
||||
ReprojectedDailyMileageTargets int64 `json:"reprojected_daily_mileage_targets,omitempty"`
|
||||
Synced int64 `json:"synced,omitempty"`
|
||||
}
|
||||
|
||||
type dailyMileageProjectionTarget struct {
|
||||
VIN string
|
||||
StatDate string
|
||||
Protocol envelope.Protocol
|
||||
}
|
||||
|
||||
type dataSourcePruneReport struct {
|
||||
Apply bool `json:"apply"`
|
||||
MinAgeSeconds int64 `json:"min_age_seconds"`
|
||||
CandidateSources int64 `json:"candidate_sources"`
|
||||
Pruned int64 `json:"pruned,omitempty"`
|
||||
}
|
||||
|
||||
type dataSourceRetireReport struct {
|
||||
Apply bool `json:"apply"`
|
||||
MinAgeSeconds int64 `json:"min_age_seconds"`
|
||||
CandidateSources int64 `json:"candidate_sources"`
|
||||
Retired int64 `json:"retired,omitempty"`
|
||||
}
|
||||
|
||||
func syncJT808DataSourcesFromIdentifiers(ctx context.Context, db *sql.DB, apply bool) (dataSourceSyncReport, error) {
|
||||
report := dataSourceSyncReport{Apply: apply}
|
||||
var err error
|
||||
report.CandidateSources, err = queryInt64(ctx, db, syncJT808DataSourcesCandidateCountSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.SkippedSources, err = queryInt64(ctx, db, syncJT808DataSourcesSkippedCountSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.ConflictingSources, err = queryInt64(ctx, db, syncJT808DataSourcesConflictCountSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if apply {
|
||||
result, err := db.ExecContext(ctx, syncJT808DataSourcesSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil {
|
||||
report.Synced = rowsAffected
|
||||
}
|
||||
}
|
||||
report.PlatformKindCandidates, err = queryInt64(ctx, db, classifyConfiguredDataSourcesCountSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if apply {
|
||||
result, err := db.ExecContext(ctx, classifyConfiguredDataSourcesSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil {
|
||||
report.PlatformKindClassified = rowsAffected
|
||||
}
|
||||
}
|
||||
targets, err := queryDailyMileageProjectionTargets(ctx, db)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.ReprojectDailyMileageTargets = int64(len(targets))
|
||||
if apply {
|
||||
for _, target := range targets {
|
||||
if err := stats.ProjectDailyMileage(ctx, db, target.VIN, target.StatDate, target.Protocol); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.ReprojectedDailyMileageTargets++
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func queryDailyMileageProjectionTargets(ctx context.Context, db *sql.DB) ([]dailyMileageProjectionTarget, error) {
|
||||
rows, err := db.QueryContext(ctx, reprojectSelectableDailyMileageSourcesSQL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var targets []dailyMileageProjectionTarget
|
||||
for rows.Next() {
|
||||
var target dailyMileageProjectionTarget
|
||||
var protocol string
|
||||
if err := rows.Scan(&target.VIN, &target.StatDate, &protocol); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target.Protocol = envelope.Protocol(strings.TrimSpace(protocol))
|
||||
if strings.TrimSpace(target.VIN) != "" && strings.TrimSpace(target.StatDate) != "" && strings.TrimSpace(string(target.Protocol)) != "" {
|
||||
targets = append(targets, target)
|
||||
}
|
||||
}
|
||||
return targets, rows.Err()
|
||||
}
|
||||
|
||||
func pruneUnmanagedDataSourcesWithoutEvidence(ctx context.Context, db *sql.DB, apply bool, minAge time.Duration) (dataSourcePruneReport, error) {
|
||||
if minAge < 0 {
|
||||
minAge = 0
|
||||
}
|
||||
minAgeSeconds := int64(minAge.Seconds())
|
||||
report := dataSourcePruneReport{Apply: apply, MinAgeSeconds: minAgeSeconds}
|
||||
var err error
|
||||
report.CandidateSources, err = queryInt64WithArgs(ctx, db, pruneUnmanagedDataSourcesCountSQL, minAgeSeconds)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if !apply {
|
||||
return report, nil
|
||||
}
|
||||
result, err := db.ExecContext(ctx, pruneUnmanagedDataSourcesSQL, minAgeSeconds)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil {
|
||||
report.Pruned = rowsAffected
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func retireStaleUnmanagedDataSourcesWithoutEvidence(ctx context.Context, db *sql.DB, apply bool, minAge time.Duration) (dataSourceRetireReport, error) {
|
||||
if minAge < 0 {
|
||||
minAge = 0
|
||||
}
|
||||
minAgeSeconds := int64(minAge.Seconds())
|
||||
report := dataSourceRetireReport{Apply: apply, MinAgeSeconds: minAgeSeconds}
|
||||
var err error
|
||||
report.CandidateSources, err = queryInt64WithArgs(ctx, db, retireStaleUnmanagedDataSourcesCountSQL, minAgeSeconds)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if !apply {
|
||||
return report, nil
|
||||
}
|
||||
result, err := db.ExecContext(ctx, retireStaleUnmanagedDataSourcesSQL, minAgeSeconds)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil {
|
||||
report.Retired = rowsAffected
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func queryInt64(ctx context.Context, db *sql.DB, query string) (int64, error) {
|
||||
return queryInt64WithArgs(ctx, db, query)
|
||||
}
|
||||
|
||||
func queryInt64WithArgs(ctx context.Context, db *sql.DB, query string, args ...any) (int64, error) {
|
||||
var count int64
|
||||
err := db.QueryRowContext(ctx, query, args...).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
const jt808DataSourceInferenceSQL = `
|
||||
SELECT
|
||||
'JT808' AS protocol,
|
||||
TRIM(r.source_ip) AS source_ip,
|
||||
MAX(TRIM(r.source_endpoint)) AS latest_source_endpoint,
|
||||
MIN(COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), ''))) AS platform_name,
|
||||
MIN(NULLIF(TRIM(vi.source_code), '')) AS source_code,
|
||||
MIN(COALESCE(r.first_registered_at, r.latest_registered_at, r.latest_authenticated_at, r.latest_seen_at, r.updated_at, CURRENT_TIMESTAMP)) AS first_seen_at,
|
||||
MAX(COALESCE(r.latest_seen_at, r.latest_authenticated_at, r.latest_registered_at, r.updated_at, CURRENT_TIMESTAMP)) AS latest_seen_at,
|
||||
COUNT(DISTINCT COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), ''))) AS platform_count,
|
||||
COUNT(DISTINCT NULLIF(TRIM(vi.source_code), '')) AS source_code_count
|
||||
FROM jt808_registration r
|
||||
JOIN vehicle_identifier vi
|
||||
ON vi.protocol = 'JT808'
|
||||
AND vi.identifier_type = 'JT808_PHONE'
|
||||
AND vi.identifier_value = r.phone
|
||||
AND vi.enabled = 1
|
||||
WHERE r.source_endpoint IS NOT NULL
|
||||
AND TRIM(r.source_endpoint) <> ''
|
||||
AND r.source_ip IS NOT NULL
|
||||
AND TRIM(r.source_ip) <> ''
|
||||
GROUP BY TRIM(r.source_ip)
|
||||
`
|
||||
|
||||
const jt808DataSourceCandidateWhereSQL = `inferred.source_ip IS NOT NULL
|
||||
AND inferred.source_ip <> ''
|
||||
AND inferred.platform_name IS NOT NULL
|
||||
AND inferred.platform_name <> ''
|
||||
AND inferred.platform_count = 1
|
||||
AND inferred.source_code_count = 1`
|
||||
|
||||
const jt808DataSourceNeedsSyncWhereSQL = `(ds.id IS NULL
|
||||
OR ds.platform_name IS NULL OR TRIM(ds.platform_name) = ''
|
||||
OR ds.source_code IS NULL OR TRIM(ds.source_code) = ''
|
||||
OR ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN'
|
||||
OR (ds.enabled = 0 AND (ds.remark LIKE 'auto-retired:%' OR ds.remark = 'auto-reenabled: source evidence restored')))`
|
||||
|
||||
const syncJT808DataSourcesSQL = `
|
||||
INSERT INTO vehicle_data_source
|
||||
(protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind, first_seen_at, latest_seen_at)
|
||||
SELECT
|
||||
inferred.protocol,
|
||||
inferred.source_ip,
|
||||
inferred.latest_source_endpoint,
|
||||
inferred.platform_name,
|
||||
inferred.source_code,
|
||||
'PLATFORM',
|
||||
inferred.first_seen_at,
|
||||
inferred.latest_seen_at
|
||||
FROM (` + jt808DataSourceInferenceSQL + `) inferred
|
||||
LEFT JOIN vehicle_data_source ds
|
||||
ON ds.protocol = inferred.protocol
|
||||
AND ds.source_ip = inferred.source_ip
|
||||
WHERE ` + jt808DataSourceCandidateWhereSQL + `
|
||||
AND ` + jt808DataSourceNeedsSyncWhereSQL + `
|
||||
ON DUPLICATE KEY UPDATE
|
||||
latest_source_endpoint = VALUES(latest_source_endpoint),
|
||||
platform_name = CASE
|
||||
WHEN vehicle_data_source.platform_name IS NULL OR TRIM(vehicle_data_source.platform_name) = ''
|
||||
THEN VALUES(platform_name)
|
||||
ELSE vehicle_data_source.platform_name
|
||||
END,
|
||||
source_code = CASE
|
||||
WHEN vehicle_data_source.source_code IS NULL OR TRIM(vehicle_data_source.source_code) = ''
|
||||
THEN VALUES(source_code)
|
||||
ELSE vehicle_data_source.source_code
|
||||
END,
|
||||
source_kind = CASE
|
||||
WHEN vehicle_data_source.source_kind IS NULL OR TRIM(vehicle_data_source.source_kind) = '' OR vehicle_data_source.source_kind = 'UNKNOWN'
|
||||
THEN VALUES(source_kind)
|
||||
ELSE vehicle_data_source.source_kind
|
||||
END,
|
||||
enabled = CASE
|
||||
WHEN vehicle_data_source.enabled = 0
|
||||
AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored')
|
||||
AND (
|
||||
(VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '')
|
||||
OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '')
|
||||
OR VALUES(source_kind) <> 'UNKNOWN'
|
||||
)
|
||||
THEN 1
|
||||
ELSE vehicle_data_source.enabled
|
||||
END,
|
||||
remark = CASE
|
||||
WHEN vehicle_data_source.enabled = 0
|
||||
AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored')
|
||||
AND (
|
||||
(VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '')
|
||||
OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '')
|
||||
OR VALUES(source_kind) <> 'UNKNOWN'
|
||||
)
|
||||
THEN 'auto-reenabled: source evidence restored'
|
||||
ELSE vehicle_data_source.remark
|
||||
END,
|
||||
first_seen_at = COALESCE(vehicle_data_source.first_seen_at, VALUES(first_seen_at)),
|
||||
latest_seen_at = GREATEST(COALESCE(vehicle_data_source.latest_seen_at, VALUES(latest_seen_at)), VALUES(latest_seen_at)),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`
|
||||
|
||||
const reprojectSelectableDailyMileageSourcesSQL = `
|
||||
SELECT DISTINCT s.vin, s.stat_date, s.protocol
|
||||
FROM vehicle_daily_mileage_source s
|
||||
LEFT JOIN vehicle_data_source ds
|
||||
ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip
|
||||
LEFT JOIN vehicle_daily_mileage m
|
||||
ON m.vin = s.vin AND m.stat_date = s.stat_date AND m.protocol = s.protocol
|
||||
WHERE s.protocol = 'JT808'
|
||||
AND s.quality_status = 'OK'
|
||||
AND (ds.id IS NULL OR ds.enabled = 1 OR (
|
||||
COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN'
|
||||
AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')
|
||||
AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')
|
||||
))
|
||||
AND (
|
||||
m.vin IS NULL
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM vehicle_daily_mileage_source selected
|
||||
WHERE selected.vin = s.vin
|
||||
AND selected.stat_date = s.stat_date
|
||||
AND selected.protocol = s.protocol
|
||||
AND selected.is_selected = 1
|
||||
)
|
||||
)
|
||||
ORDER BY s.stat_date DESC, s.vin ASC
|
||||
`
|
||||
|
||||
const syncJT808DataSourcesCandidateCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM (` + jt808DataSourceInferenceSQL + `) inferred
|
||||
LEFT JOIN vehicle_data_source ds
|
||||
ON ds.protocol = inferred.protocol
|
||||
AND ds.source_ip = inferred.source_ip
|
||||
WHERE ` + jt808DataSourceCandidateWhereSQL + `
|
||||
AND ` + jt808DataSourceNeedsSyncWhereSQL
|
||||
|
||||
const syncJT808DataSourcesSkippedCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM (` + jt808DataSourceInferenceSQL + `) inferred
|
||||
WHERE inferred.source_ip IS NOT NULL
|
||||
AND inferred.source_ip <> ''
|
||||
AND inferred.platform_name IS NOT NULL
|
||||
AND inferred.platform_name <> ''
|
||||
AND NOT (` + jt808DataSourceCandidateWhereSQL + `)`
|
||||
|
||||
const syncJT808DataSourcesConflictCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM (` + jt808DataSourceInferenceSQL + `) inferred
|
||||
JOIN vehicle_data_source ds
|
||||
ON ds.protocol = inferred.protocol
|
||||
AND ds.source_ip = inferred.source_ip
|
||||
WHERE ` + jt808DataSourceCandidateWhereSQL + `
|
||||
AND ds.source_code IS NOT NULL
|
||||
AND TRIM(ds.source_code) <> ''
|
||||
AND ds.source_code <> inferred.source_code`
|
||||
|
||||
const classifyConfiguredDataSourcesWhereSQL = `
|
||||
(source_kind IS NULL OR TRIM(source_kind) = '' OR source_kind = 'UNKNOWN')
|
||||
AND (
|
||||
(source_code IS NOT NULL AND TRIM(source_code) <> '')
|
||||
OR (platform_name IS NOT NULL AND TRIM(platform_name) <> '')
|
||||
)`
|
||||
|
||||
const classifyConfiguredDataSourcesCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_data_source
|
||||
WHERE ` + classifyConfiguredDataSourcesWhereSQL
|
||||
|
||||
const classifyConfiguredDataSourcesSQL = `
|
||||
UPDATE vehicle_data_source
|
||||
SET source_kind = 'PLATFORM',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE ` + classifyConfiguredDataSourcesWhereSQL
|
||||
|
||||
const pruneUnmanagedDataSourcesWhereSQL = `
|
||||
(ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')
|
||||
AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')
|
||||
AND (ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN')
|
||||
AND TIMESTAMPDIFF(SECOND, COALESCE(ds.latest_seen_at, ds.updated_at, ds.created_at), CURRENT_TIMESTAMP) >= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM vehicle_daily_mileage m
|
||||
WHERE m.source_id = ds.id
|
||||
LIMIT 1
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM jt808_registration r
|
||||
WHERE ds.protocol = 'JT808'
|
||||
AND r.source_ip = ds.source_ip
|
||||
LIMIT 1
|
||||
)`
|
||||
|
||||
const pruneUnmanagedDataSourcesCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_data_source ds
|
||||
WHERE ` + pruneUnmanagedDataSourcesWhereSQL
|
||||
|
||||
const pruneUnmanagedDataSourcesSQL = `
|
||||
DELETE ds
|
||||
FROM vehicle_data_source ds
|
||||
WHERE ` + pruneUnmanagedDataSourcesWhereSQL
|
||||
|
||||
const retireStaleUnmanagedDataSourcesWhereSQL = `
|
||||
ds.enabled = 1
|
||||
AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')
|
||||
AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')
|
||||
AND (ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN')
|
||||
AND TIMESTAMPDIFF(SECOND, COALESCE(ds.latest_seen_at, ds.updated_at, ds.created_at), CURRENT_TIMESTAMP) >= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM jt808_registration r
|
||||
WHERE ds.protocol = 'JT808'
|
||||
AND r.source_ip = ds.source_ip
|
||||
LIMIT 1
|
||||
)`
|
||||
|
||||
const retireStaleUnmanagedDataSourcesCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_data_source ds
|
||||
WHERE ` + retireStaleUnmanagedDataSourcesWhereSQL
|
||||
|
||||
const retireStaleUnmanagedDataSourcesSQL = `
|
||||
UPDATE vehicle_data_source ds
|
||||
SET enabled = 0,
|
||||
remark = CASE
|
||||
WHEN remark IS NULL OR TRIM(remark) = ''
|
||||
THEN 'auto-retired: stale unmanaged source without registration evidence'
|
||||
ELSE remark
|
||||
END,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE ` + retireStaleUnmanagedDataSourcesWhereSQL
|
||||
|
||||
func fail(err error) {
|
||||
_, _ = fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
426
go/vehicle-gateway/cmd/identity-import/main_test.go
Normal file
426
go/vehicle-gateway/cmd/identity-import/main_test.go
Normal file
@@ -0,0 +1,426 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
)
|
||||
|
||||
func TestReportItemLimitExpandsWhenCSVOutputIsRequested(t *testing.T) {
|
||||
if got := reportItemLimit(50, "", ""); got != 50 {
|
||||
t.Fatalf("reportItemLimit without csv = %d, want 50", got)
|
||||
}
|
||||
if got := reportItemLimit(50, "unresolved.csv", ""); got != -1 {
|
||||
t.Fatalf("reportItemLimit with unresolved csv = %d, want -1", got)
|
||||
}
|
||||
if got := reportItemLimit(50, "", "conflicts.csv"); got != -1 {
|
||||
t.Fatalf("reportItemLimit with conflicts csv = %d, want -1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedMappingFilesError(t *testing.T) {
|
||||
err := unsupportedMappingFilesError(identity.MappingScanReport{})
|
||||
if err != nil {
|
||||
t.Fatalf("unsupportedMappingFilesError(empty) = %v", err)
|
||||
}
|
||||
err = unsupportedMappingFilesError(identity.MappingScanReport{
|
||||
UnsupportedFiles: 2,
|
||||
UnsupportedItems: []identity.MappingUnsupportedFileReport{
|
||||
{File: "G7s/legacy.xls", Ext: ".xls"},
|
||||
{File: "信达/legacy.xlsb", Ext: ".xlsb"},
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("unsupportedMappingFilesError() nil, want error")
|
||||
}
|
||||
text := err.Error()
|
||||
for _, want := range []string{"2 unsupported workbook", "G7s/legacy.xls", "信达/legacy.xlsb", "before -apply"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("error missing %q: %s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMappingApplyInputAllowsDryRunButBlocksApply(t *testing.T) {
|
||||
scan := identity.MappingScanReport{
|
||||
UnsupportedFiles: 1,
|
||||
UnsupportedItems: []identity.MappingUnsupportedFileReport{
|
||||
{File: "G7s/legacy.xls", Ext: ".xls"},
|
||||
},
|
||||
}
|
||||
if err := validateMappingApplyInput(false, scan); err != nil {
|
||||
t.Fatalf("validateMappingApplyInput(dry-run) = %v", err)
|
||||
}
|
||||
if err := validateMappingApplyInput(true, scan); err == nil {
|
||||
t.Fatal("validateMappingApplyInput(apply) nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteUnresolvedCSV(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "unresolved.csv")
|
||||
err := writeUnresolvedCSV(path, []identity.MappingRecord{
|
||||
{
|
||||
File: "G7s/example.xlsx",
|
||||
Sheet: "Sheet1",
|
||||
Row: 2,
|
||||
SourceCode: "g7s",
|
||||
SourceName: "G7s",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: identity.IdentifierTypeJT808Phone,
|
||||
IdentifierValue: "13307795425",
|
||||
RawValue: "013307795425",
|
||||
Plate: "粤AG18312",
|
||||
OEM: "G7s",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("writeUnresolvedCSV() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
text := string(data)
|
||||
for _, want := range []string{"identifier_type", "JT808_PHONE", "13307795425", "vin_not_found_in_legacy_binding"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("csv missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteConflictsCSV(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "conflicts.csv")
|
||||
err := writeConflictsCSV(path, []identity.MappingConflict{
|
||||
{
|
||||
Record: identity.MappingRecord{
|
||||
File: "source.xlsx",
|
||||
Sheet: "Sheet1",
|
||||
Row: 3,
|
||||
SourceCode: "xinda",
|
||||
Protocol: "JT808",
|
||||
IdentifierType: identity.IdentifierTypePlate,
|
||||
IdentifierValue: "粤AG18312",
|
||||
Plate: "粤AG18312",
|
||||
},
|
||||
ExistingVIN: "VIN001",
|
||||
NewVIN: "VIN002",
|
||||
Reason: "identifier already points to another vin",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("writeConflictsCSV() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
text := string(data)
|
||||
for _, want := range []string{"existing_vin", "VIN001", "VIN002", "identifier already points to another vin"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("csv missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncJT808DataSourcesFromIdentifiersPreservesManualPlatformNames(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(3))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2))
|
||||
mock.ExpectExec("INSERT INTO vehicle_data_source").
|
||||
WillReturnResult(driver.RowsAffected(3))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(5))
|
||||
mock.ExpectExec("UPDATE vehicle_data_source").
|
||||
WillReturnResult(driver.RowsAffected(5))
|
||||
mock.ExpectQuery("SELECT DISTINCT s.vin, s.stat_date, s.protocol").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol"}).
|
||||
AddRow("LNXNEGRRXSR319449", "2026-07-13", "JT808"))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage").
|
||||
WillReturnResult(driver.RowsAffected(1))
|
||||
mock.ExpectExec("UPDATE vehicle_daily_mileage_source s").
|
||||
WillReturnResult(driver.RowsAffected(1))
|
||||
mock.ExpectExec("DELETE FROM vehicle_daily_mileage").
|
||||
WillReturnResult(driver.RowsAffected(0))
|
||||
mock.ExpectCommit()
|
||||
|
||||
report, err := syncJT808DataSourcesFromIdentifiers(context.Background(), db, true)
|
||||
if err != nil {
|
||||
t.Fatalf("syncJT808DataSourcesFromIdentifiers() error = %v", err)
|
||||
}
|
||||
if !report.Apply || report.CandidateSources != 3 || report.SkippedSources != 1 || report.ConflictingSources != 2 || report.Synced != 3 || report.PlatformKindCandidates != 5 || report.PlatformKindClassified != 5 || report.ReprojectDailyMileageTargets != 1 || report.ReprojectedDailyMileageTargets != 1 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"COUNT(DISTINCT",
|
||||
"vehicle_identifier vi",
|
||||
"TRIM(r.source_ip) AS source_ip",
|
||||
"GROUP BY TRIM(r.source_ip)",
|
||||
"inferred.source_code",
|
||||
"'PLATFORM'",
|
||||
"inferred.source_code_count = 1",
|
||||
"vehicle_data_source.source_code IS NULL OR TRIM(vehicle_data_source.source_code) = ''",
|
||||
"ELSE vehicle_data_source.source_code",
|
||||
"vehicle_data_source.source_kind IS NULL OR TRIM(vehicle_data_source.source_kind) = '' OR vehicle_data_source.source_kind = 'UNKNOWN'",
|
||||
"ELSE vehicle_data_source.source_kind",
|
||||
"vehicle_data_source.enabled = 0",
|
||||
"vehicle_data_source.remark LIKE 'auto-retired:%'",
|
||||
"vehicle_data_source.remark = 'auto-reenabled: source evidence restored'",
|
||||
"THEN 'auto-reenabled: source evidence restored'",
|
||||
"THEN 1",
|
||||
"LEFT JOIN vehicle_data_source ds",
|
||||
"ds.id IS NULL",
|
||||
} {
|
||||
if !strings.Contains(syncJT808DataSourcesSQL, want) {
|
||||
t.Fatalf("sync sql missing %q:\n%s", want, syncJT808DataSourcesSQL)
|
||||
}
|
||||
}
|
||||
if strings.Index(syncJT808DataSourcesSQL, "enabled = CASE") < 0 ||
|
||||
strings.Index(syncJT808DataSourcesSQL, "remark = CASE") < 0 ||
|
||||
strings.Index(syncJT808DataSourcesSQL, "enabled = CASE") > strings.Index(syncJT808DataSourcesSQL, "remark = CASE") {
|
||||
t.Fatalf("sync sql must restore enabled before updating remark because MySQL evaluates assignments in order:\n%s", syncJT808DataSourcesSQL)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"LEFT JOIN vehicle_data_source ds",
|
||||
"ds.id IS NULL",
|
||||
"ds.source_code IS NULL OR TRIM(ds.source_code) = ''",
|
||||
"ds.enabled = 0 AND (ds.remark LIKE 'auto-retired:%'",
|
||||
"ds.remark = 'auto-reenabled: source evidence restored'",
|
||||
} {
|
||||
if !strings.Contains(syncJT808DataSourcesCandidateCountSQL, want) {
|
||||
t.Fatalf("candidate count sql missing %q:\n%s", want, syncJT808DataSourcesCandidateCountSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"JOIN vehicle_data_source ds",
|
||||
"ds.source_code <> inferred.source_code",
|
||||
} {
|
||||
if !strings.Contains(syncJT808DataSourcesConflictCountSQL, want) {
|
||||
t.Fatalf("conflict count sql missing %q:\n%s", want, syncJT808DataSourcesConflictCountSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"vehicle_data_source.platform_name IS NULL OR TRIM(vehicle_data_source.platform_name) = ''",
|
||||
"ELSE vehicle_data_source.platform_name",
|
||||
} {
|
||||
if !strings.Contains(syncJT808DataSourcesSQL, want) {
|
||||
t.Fatalf("sync sql missing %q:\n%s", want, syncJT808DataSourcesSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"source_kind IS NULL OR TRIM(source_kind) = '' OR source_kind = 'UNKNOWN'",
|
||||
"source_code IS NOT NULL AND TRIM(source_code) <> ''",
|
||||
"platform_name IS NOT NULL AND TRIM(platform_name) <> ''",
|
||||
"SET source_kind = 'PLATFORM'",
|
||||
} {
|
||||
if !strings.Contains(classifyConfiguredDataSourcesSQL, want) && !strings.Contains(classifyConfiguredDataSourcesCountSQL, want) {
|
||||
t.Fatalf("configured-source classification sql missing %q:\n%s\n%s", want, classifyConfiguredDataSourcesSQL, classifyConfiguredDataSourcesCountSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"SELECT DISTINCT s.vin, s.stat_date, s.protocol",
|
||||
"vehicle_daily_mileage_source s",
|
||||
"LEFT JOIN vehicle_data_source ds",
|
||||
"LEFT JOIN vehicle_daily_mileage m",
|
||||
"s.protocol = 'JT808'",
|
||||
"s.quality_status = 'OK'",
|
||||
"ds.enabled = 1",
|
||||
"m.vin IS NULL",
|
||||
"selected.is_selected = 1",
|
||||
} {
|
||||
if !strings.Contains(reprojectSelectableDailyMileageSourcesSQL, want) {
|
||||
t.Fatalf("reproject sql missing %q:\n%s", want, reprojectSelectableDailyMileageSourcesSQL)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncJT808DataSourcesDryRunDoesNotWrite(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(4))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(6))
|
||||
mock.ExpectQuery("SELECT DISTINCT s.vin, s.stat_date, s.protocol").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol"}).
|
||||
AddRow("LNXNEGRRXSR319449", "2026-07-13", "JT808"))
|
||||
|
||||
report, err := syncJT808DataSourcesFromIdentifiers(context.Background(), db, false)
|
||||
if err != nil {
|
||||
t.Fatalf("syncJT808DataSourcesFromIdentifiers() error = %v", err)
|
||||
}
|
||||
if report.Apply || report.CandidateSources != 4 || report.SkippedSources != 2 || report.ConflictingSources != 1 || report.PlatformKindCandidates != 6 || report.PlatformKindClassified != 0 || report.Synced != 0 || report.ReprojectDailyMileageTargets != 1 || report.ReprojectedDailyMileageTargets != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenanceModeNamesAllSourceMaintenanceSteps(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
apply bool
|
||||
sync bool
|
||||
prune bool
|
||||
retire bool
|
||||
want string
|
||||
}{
|
||||
{name: "dry run", want: "dry_run"},
|
||||
{name: "apply", apply: true, want: "apply"},
|
||||
{name: "sync dry run", sync: true, want: "sync_dry_run"},
|
||||
{name: "sync prune retire apply", apply: true, sync: true, prune: true, retire: true, want: "sync_prune_retire_apply"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := maintenanceMode(tt.apply, tt.sync, tt.prune, tt.retire); got != tt.want {
|
||||
t.Fatalf("maintenanceMode() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneUnmanagedDataSourcesDryRunDoesNotDelete(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WithArgs(int64(3600)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(339))
|
||||
|
||||
report, err := pruneUnmanagedDataSourcesWithoutEvidence(context.Background(), db, false, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("pruneUnmanagedDataSourcesWithoutEvidence() error = %v", err)
|
||||
}
|
||||
if report.Apply || report.MinAgeSeconds != 3600 || report.CandidateSources != 339 || report.Pruned != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"vehicle_data_source ds",
|
||||
"vehicle_daily_mileage m",
|
||||
"m.source_id = ds.id",
|
||||
"jt808_registration r",
|
||||
"r.source_ip = ds.source_ip",
|
||||
"ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN'",
|
||||
"TIMESTAMPDIFF(SECOND",
|
||||
} {
|
||||
if !strings.Contains(pruneUnmanagedDataSourcesCountSQL, want) {
|
||||
t.Fatalf("prune count sql missing %q:\n%s", want, pruneUnmanagedDataSourcesCountSQL)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetireStaleUnmanagedDataSourcesDryRunDoesNotDisable(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WithArgs(int64(86400)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(83))
|
||||
|
||||
report, err := retireStaleUnmanagedDataSourcesWithoutEvidence(context.Background(), db, false, 24*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("retireStaleUnmanagedDataSourcesWithoutEvidence() error = %v", err)
|
||||
}
|
||||
if report.Apply || report.MinAgeSeconds != 86400 || report.CandidateSources != 83 || report.Retired != 0 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"vehicle_data_source ds",
|
||||
"ds.enabled = 1",
|
||||
"jt808_registration r",
|
||||
"r.source_ip = ds.source_ip",
|
||||
"ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN'",
|
||||
"TIMESTAMPDIFF(SECOND",
|
||||
} {
|
||||
if !strings.Contains(retireStaleUnmanagedDataSourcesCountSQL, want) {
|
||||
t.Fatalf("retire count sql missing %q:\n%s", want, retireStaleUnmanagedDataSourcesCountSQL)
|
||||
}
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetireStaleUnmanagedDataSourcesApplyDisablesOnlyCandidates(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WithArgs(int64(7200)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(7))
|
||||
mock.ExpectExec("UPDATE vehicle_data_source ds").
|
||||
WithArgs(int64(7200)).
|
||||
WillReturnResult(driver.RowsAffected(7))
|
||||
|
||||
report, err := retireStaleUnmanagedDataSourcesWithoutEvidence(context.Background(), db, true, 2*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("retireStaleUnmanagedDataSourcesWithoutEvidence() error = %v", err)
|
||||
}
|
||||
if !report.Apply || report.MinAgeSeconds != 7200 || report.CandidateSources != 7 || report.Retired != 7 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneUnmanagedDataSourcesApplyDeletesOnlyCandidates(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\)").
|
||||
WithArgs(int64(1800)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(12))
|
||||
mock.ExpectExec("DELETE ds\\s+FROM vehicle_data_source ds").
|
||||
WithArgs(int64(1800)).
|
||||
WillReturnResult(driver.RowsAffected(12))
|
||||
|
||||
report, err := pruneUnmanagedDataSourcesWithoutEvidence(context.Background(), db, true, 30*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("pruneUnmanagedDataSourcesWithoutEvidence() error = %v", err)
|
||||
}
|
||||
if !report.Apply || report.MinAgeSeconds != 1800 || report.CandidateSources != 12 || report.Pruned != 12 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user