feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
695
go/vehicle-gateway/cmd/identity-import/main.go
Normal file
695
go/vehicle-gateway/cmd/identity-import/main.go
Normal file
@@ -0,0 +1,695 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var input string
|
||||
var dsn string
|
||||
var legacyTable string
|
||||
var apply bool
|
||||
var ensureSchema bool
|
||||
var reportLimit int
|
||||
var unresolvedOut string
|
||||
var conflictsOut string
|
||||
var syncDataSources bool
|
||||
var pruneUnmanagedDataSources bool
|
||||
var pruneDataSourceMinAge time.Duration
|
||||
var retireStaleUnmanagedDataSources bool
|
||||
var retireDataSourceMinAge time.Duration
|
||||
var timeout time.Duration
|
||||
flag.StringVar(&input, "input", env("IDENTITY_MAPPING_INPUT", ""), "directory that contains 808 plate/phone mapping workbooks")
|
||||
flag.StringVar(&dsn, "mysql-dsn", env("IDENTITY_MYSQL_DSN", env("MYSQL_DSN", "")), "MySQL DSN; empty means scan only")
|
||||
flag.StringVar(&legacyTable, "legacy-table", env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding"), "legacy VIN/plate binding table used to resolve VIN")
|
||||
flag.BoolVar(&apply, "apply", false, "write resolved mappings to vehicle and vehicle_identifier")
|
||||
flag.BoolVar(&ensureSchema, "ensure-schema", false, "create vehicle and vehicle_identifier tables before import")
|
||||
flag.IntVar(&reportLimit, "report-limit", 50, "max unresolved/conflict items embedded in JSON; use -1 for all")
|
||||
flag.StringVar(&unresolvedOut, "unresolved-out", "", "optional CSV path for all unresolved mappings")
|
||||
flag.StringVar(&conflictsOut, "conflicts-out", "", "optional CSV path for all conflict mappings")
|
||||
flag.BoolVar(&syncDataSources, "sync-data-sources", false, "infer JT808 vehicle_data_source platform names from jt808_registration and vehicle_identifier")
|
||||
flag.BoolVar(&pruneUnmanagedDataSources, "prune-unmanaged-data-sources", false, "delete stale unclassified vehicle_data_source rows that have no registration evidence and are not referenced by final mileage")
|
||||
flag.DurationVar(&pruneDataSourceMinAge, "prune-data-source-min-age", time.Hour, "minimum latest_seen/updated age before -prune-unmanaged-data-sources can delete rows")
|
||||
flag.BoolVar(&retireStaleUnmanagedDataSources, "retire-stale-unmanaged-data-sources", false, "disable stale unclassified vehicle_data_source rows that have no registration evidence but may be referenced by historical mileage")
|
||||
flag.DurationVar(&retireDataSourceMinAge, "retire-data-source-min-age", 24*time.Hour, "minimum latest_seen/updated age before -retire-stale-unmanaged-data-sources can disable rows")
|
||||
flag.DurationVar(&timeout, "timeout", 2*time.Minute, "import timeout")
|
||||
flag.Parse()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
input = strings.TrimSpace(input)
|
||||
output := map[string]any{}
|
||||
var records []identity.MappingRecord
|
||||
var scan identity.MappingScanReport
|
||||
if input != "" {
|
||||
var err error
|
||||
records, scan, err = identity.ReadMappingDirectory(input)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
output["input"] = input
|
||||
output["scan"] = scan
|
||||
} else if !syncDataSources && !pruneUnmanagedDataSources && !retireStaleUnmanagedDataSources {
|
||||
fail(errors.New("-input is required unless -sync-data-sources, -prune-unmanaged-data-sources or -retire-stale-unmanaged-data-sources is set"))
|
||||
}
|
||||
if err := validateMappingApplyInput(apply, scan); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
if strings.TrimSpace(dsn) == "" {
|
||||
if syncDataSources || pruneUnmanagedDataSources || retireStaleUnmanagedDataSources {
|
||||
fail(errors.New("-mysql-dsn is required when -sync-data-sources, -prune-unmanaged-data-sources or -retire-stale-unmanaged-data-sources is set"))
|
||||
}
|
||||
output["mode"] = "scan_only"
|
||||
writeJSON(output)
|
||||
return
|
||||
}
|
||||
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
if input != "" && (ensureSchema || apply) {
|
||||
if err := identity.EnsureVehicleIdentifierSchema(ctx, db); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
}
|
||||
if input != "" {
|
||||
report, err := identity.ImportMappingRecords(ctx, db, records, scan, identity.MappingImportOptions{
|
||||
Apply: apply,
|
||||
LegacyTable: legacyTable,
|
||||
ReportItemLimit: reportItemLimit(reportLimit, unresolvedOut, conflictsOut),
|
||||
})
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
if strings.TrimSpace(unresolvedOut) != "" {
|
||||
if err := writeUnresolvedCSV(unresolvedOut, report.UnresolvedItems); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(conflictsOut) != "" {
|
||||
if err := writeConflictsCSV(conflictsOut, report.ConflictItems); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
}
|
||||
output["report"] = report
|
||||
output["mode"] = map[bool]string{true: "apply", false: "dry_run"}[apply]
|
||||
}
|
||||
if syncDataSources {
|
||||
syncReport, err := syncJT808DataSourcesFromIdentifiers(ctx, db, apply)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
output["data_source_sync"] = syncReport
|
||||
}
|
||||
if pruneUnmanagedDataSources {
|
||||
pruneReport, err := pruneUnmanagedDataSourcesWithoutEvidence(ctx, db, apply, pruneDataSourceMinAge)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
output["data_source_prune"] = pruneReport
|
||||
}
|
||||
if retireStaleUnmanagedDataSources {
|
||||
retireReport, err := retireStaleUnmanagedDataSourcesWithoutEvidence(ctx, db, apply, retireDataSourceMinAge)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
}
|
||||
output["data_source_retire"] = retireReport
|
||||
}
|
||||
if input == "" {
|
||||
output["mode"] = maintenanceMode(apply, syncDataSources, pruneUnmanagedDataSources, retireStaleUnmanagedDataSources)
|
||||
}
|
||||
writeJSON(output)
|
||||
}
|
||||
|
||||
func maintenanceMode(apply bool, syncDataSources bool, pruneUnmanagedDataSources bool, retireStaleUnmanagedDataSources bool) string {
|
||||
var parts []string
|
||||
if syncDataSources {
|
||||
parts = append(parts, "sync")
|
||||
}
|
||||
if pruneUnmanagedDataSources {
|
||||
parts = append(parts, "prune")
|
||||
}
|
||||
if retireStaleUnmanagedDataSources {
|
||||
parts = append(parts, "retire")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return map[bool]string{true: "apply", false: "dry_run"}[apply]
|
||||
}
|
||||
parts = append(parts, map[bool]string{true: "apply", false: "dry_run"}[apply])
|
||||
return strings.Join(parts, "_")
|
||||
}
|
||||
|
||||
func reportItemLimit(limit int, unresolvedOut string, conflictsOut string) int {
|
||||
if strings.TrimSpace(unresolvedOut) != "" || strings.TrimSpace(conflictsOut) != "" {
|
||||
return -1
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func validateMappingApplyInput(apply bool, scan identity.MappingScanReport) error {
|
||||
if !apply {
|
||||
return nil
|
||||
}
|
||||
return unsupportedMappingFilesError(scan)
|
||||
}
|
||||
|
||||
func unsupportedMappingFilesError(scan identity.MappingScanReport) error {
|
||||
if scan.UnsupportedFiles <= 0 {
|
||||
return nil
|
||||
}
|
||||
items := make([]string, 0, len(scan.UnsupportedItems))
|
||||
for _, item := range scan.UnsupportedItems {
|
||||
path := strings.TrimSpace(item.File)
|
||||
if path == "" {
|
||||
path = strings.TrimSpace(item.Ext)
|
||||
}
|
||||
if path != "" {
|
||||
items = append(items, path)
|
||||
}
|
||||
if len(items) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
suffix := ""
|
||||
if len(items) > 0 {
|
||||
suffix = ": " + strings.Join(items, ", ")
|
||||
}
|
||||
return fmt.Errorf("identity mapping import has %d unsupported workbook(s); convert .xls/.xlsb to .xlsx before -apply%s", scan.UnsupportedFiles, suffix)
|
||||
}
|
||||
|
||||
func env(key string, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func writeJSON(value any) {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(value); err != nil {
|
||||
fail(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeUnresolvedCSV(path string, items []identity.MappingRecord) error {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
writer := csv.NewWriter(file)
|
||||
defer writer.Flush()
|
||||
if err := writer.Write([]string{
|
||||
"file", "sheet", "row", "source_code", "source_name", "protocol",
|
||||
"identifier_type", "identifier_value", "raw_value", "plate", "oem", "reason",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
if err := writer.Write([]string{
|
||||
item.File,
|
||||
item.Sheet,
|
||||
fmt.Sprint(item.Row),
|
||||
item.SourceCode,
|
||||
item.SourceName,
|
||||
item.Protocol,
|
||||
item.IdentifierType,
|
||||
item.IdentifierValue,
|
||||
item.RawValue,
|
||||
item.Plate,
|
||||
item.OEM,
|
||||
"vin_not_found_in_legacy_binding",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writer.Error()
|
||||
}
|
||||
|
||||
func writeConflictsCSV(path string, items []identity.MappingConflict) error {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
writer := csv.NewWriter(file)
|
||||
defer writer.Flush()
|
||||
if err := writer.Write([]string{
|
||||
"file", "sheet", "row", "source_code", "source_name", "protocol",
|
||||
"identifier_type", "identifier_value", "raw_value", "plate", "oem",
|
||||
"existing_vin", "new_vin", "reason",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
record := item.Record
|
||||
if err := writer.Write([]string{
|
||||
record.File,
|
||||
record.Sheet,
|
||||
fmt.Sprint(record.Row),
|
||||
record.SourceCode,
|
||||
record.SourceName,
|
||||
record.Protocol,
|
||||
record.IdentifierType,
|
||||
record.IdentifierValue,
|
||||
record.RawValue,
|
||||
record.Plate,
|
||||
record.OEM,
|
||||
item.ExistingVIN,
|
||||
item.NewVIN,
|
||||
item.Reason,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writer.Error()
|
||||
}
|
||||
|
||||
type dataSourceSyncReport struct {
|
||||
Apply bool `json:"apply"`
|
||||
CandidateSources int64 `json:"candidate_sources"`
|
||||
SkippedSources int64 `json:"skipped_sources"`
|
||||
ConflictingSources int64 `json:"conflicting_sources"`
|
||||
PlatformKindCandidates int64 `json:"platform_kind_candidates"`
|
||||
PlatformKindClassified int64 `json:"platform_kind_classified,omitempty"`
|
||||
ReprojectDailyMileageTargets int64 `json:"reproject_daily_mileage_targets,omitempty"`
|
||||
ReprojectedDailyMileageTargets int64 `json:"reprojected_daily_mileage_targets,omitempty"`
|
||||
Synced int64 `json:"synced,omitempty"`
|
||||
}
|
||||
|
||||
type dailyMileageProjectionTarget struct {
|
||||
VIN string
|
||||
StatDate string
|
||||
Protocol envelope.Protocol
|
||||
}
|
||||
|
||||
type dataSourcePruneReport struct {
|
||||
Apply bool `json:"apply"`
|
||||
MinAgeSeconds int64 `json:"min_age_seconds"`
|
||||
CandidateSources int64 `json:"candidate_sources"`
|
||||
Pruned int64 `json:"pruned,omitempty"`
|
||||
}
|
||||
|
||||
type dataSourceRetireReport struct {
|
||||
Apply bool `json:"apply"`
|
||||
MinAgeSeconds int64 `json:"min_age_seconds"`
|
||||
CandidateSources int64 `json:"candidate_sources"`
|
||||
Retired int64 `json:"retired,omitempty"`
|
||||
}
|
||||
|
||||
func syncJT808DataSourcesFromIdentifiers(ctx context.Context, db *sql.DB, apply bool) (dataSourceSyncReport, error) {
|
||||
report := dataSourceSyncReport{Apply: apply}
|
||||
var err error
|
||||
report.CandidateSources, err = queryInt64(ctx, db, syncJT808DataSourcesCandidateCountSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.SkippedSources, err = queryInt64(ctx, db, syncJT808DataSourcesSkippedCountSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.ConflictingSources, err = queryInt64(ctx, db, syncJT808DataSourcesConflictCountSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if apply {
|
||||
result, err := db.ExecContext(ctx, syncJT808DataSourcesSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil {
|
||||
report.Synced = rowsAffected
|
||||
}
|
||||
}
|
||||
report.PlatformKindCandidates, err = queryInt64(ctx, db, classifyConfiguredDataSourcesCountSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if apply {
|
||||
result, err := db.ExecContext(ctx, classifyConfiguredDataSourcesSQL)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil {
|
||||
report.PlatformKindClassified = rowsAffected
|
||||
}
|
||||
}
|
||||
targets, err := queryDailyMileageProjectionTargets(ctx, db)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.ReprojectDailyMileageTargets = int64(len(targets))
|
||||
if apply {
|
||||
for _, target := range targets {
|
||||
if err := stats.ProjectDailyMileage(ctx, db, target.VIN, target.StatDate, target.Protocol); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.ReprojectedDailyMileageTargets++
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func queryDailyMileageProjectionTargets(ctx context.Context, db *sql.DB) ([]dailyMileageProjectionTarget, error) {
|
||||
rows, err := db.QueryContext(ctx, reprojectSelectableDailyMileageSourcesSQL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var targets []dailyMileageProjectionTarget
|
||||
for rows.Next() {
|
||||
var target dailyMileageProjectionTarget
|
||||
var protocol string
|
||||
if err := rows.Scan(&target.VIN, &target.StatDate, &protocol); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target.Protocol = envelope.Protocol(strings.TrimSpace(protocol))
|
||||
if strings.TrimSpace(target.VIN) != "" && strings.TrimSpace(target.StatDate) != "" && strings.TrimSpace(string(target.Protocol)) != "" {
|
||||
targets = append(targets, target)
|
||||
}
|
||||
}
|
||||
return targets, rows.Err()
|
||||
}
|
||||
|
||||
func pruneUnmanagedDataSourcesWithoutEvidence(ctx context.Context, db *sql.DB, apply bool, minAge time.Duration) (dataSourcePruneReport, error) {
|
||||
if minAge < 0 {
|
||||
minAge = 0
|
||||
}
|
||||
minAgeSeconds := int64(minAge.Seconds())
|
||||
report := dataSourcePruneReport{Apply: apply, MinAgeSeconds: minAgeSeconds}
|
||||
var err error
|
||||
report.CandidateSources, err = queryInt64WithArgs(ctx, db, pruneUnmanagedDataSourcesCountSQL, minAgeSeconds)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if !apply {
|
||||
return report, nil
|
||||
}
|
||||
result, err := db.ExecContext(ctx, pruneUnmanagedDataSourcesSQL, minAgeSeconds)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil {
|
||||
report.Pruned = rowsAffected
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func retireStaleUnmanagedDataSourcesWithoutEvidence(ctx context.Context, db *sql.DB, apply bool, minAge time.Duration) (dataSourceRetireReport, error) {
|
||||
if minAge < 0 {
|
||||
minAge = 0
|
||||
}
|
||||
minAgeSeconds := int64(minAge.Seconds())
|
||||
report := dataSourceRetireReport{Apply: apply, MinAgeSeconds: minAgeSeconds}
|
||||
var err error
|
||||
report.CandidateSources, err = queryInt64WithArgs(ctx, db, retireStaleUnmanagedDataSourcesCountSQL, minAgeSeconds)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if !apply {
|
||||
return report, nil
|
||||
}
|
||||
result, err := db.ExecContext(ctx, retireStaleUnmanagedDataSourcesSQL, minAgeSeconds)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil {
|
||||
report.Retired = rowsAffected
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func queryInt64(ctx context.Context, db *sql.DB, query string) (int64, error) {
|
||||
return queryInt64WithArgs(ctx, db, query)
|
||||
}
|
||||
|
||||
func queryInt64WithArgs(ctx context.Context, db *sql.DB, query string, args ...any) (int64, error) {
|
||||
var count int64
|
||||
err := db.QueryRowContext(ctx, query, args...).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
const jt808DataSourceInferenceSQL = `
|
||||
SELECT
|
||||
'JT808' AS protocol,
|
||||
TRIM(r.source_ip) AS source_ip,
|
||||
MAX(TRIM(r.source_endpoint)) AS latest_source_endpoint,
|
||||
MIN(COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), ''))) AS platform_name,
|
||||
MIN(NULLIF(TRIM(vi.source_code), '')) AS source_code,
|
||||
MIN(COALESCE(r.first_registered_at, r.latest_registered_at, r.latest_authenticated_at, r.latest_seen_at, r.updated_at, CURRENT_TIMESTAMP)) AS first_seen_at,
|
||||
MAX(COALESCE(r.latest_seen_at, r.latest_authenticated_at, r.latest_registered_at, r.updated_at, CURRENT_TIMESTAMP)) AS latest_seen_at,
|
||||
COUNT(DISTINCT COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), ''))) AS platform_count,
|
||||
COUNT(DISTINCT NULLIF(TRIM(vi.source_code), '')) AS source_code_count
|
||||
FROM jt808_registration r
|
||||
JOIN vehicle_identifier vi
|
||||
ON vi.protocol = 'JT808'
|
||||
AND vi.identifier_type = 'JT808_PHONE'
|
||||
AND vi.identifier_value = r.phone
|
||||
AND vi.enabled = 1
|
||||
WHERE r.source_endpoint IS NOT NULL
|
||||
AND TRIM(r.source_endpoint) <> ''
|
||||
AND r.source_ip IS NOT NULL
|
||||
AND TRIM(r.source_ip) <> ''
|
||||
GROUP BY TRIM(r.source_ip)
|
||||
`
|
||||
|
||||
const jt808DataSourceCandidateWhereSQL = `inferred.source_ip IS NOT NULL
|
||||
AND inferred.source_ip <> ''
|
||||
AND inferred.platform_name IS NOT NULL
|
||||
AND inferred.platform_name <> ''
|
||||
AND inferred.platform_count = 1
|
||||
AND inferred.source_code_count = 1`
|
||||
|
||||
const jt808DataSourceNeedsSyncWhereSQL = `(ds.id IS NULL
|
||||
OR ds.platform_name IS NULL OR TRIM(ds.platform_name) = ''
|
||||
OR ds.source_code IS NULL OR TRIM(ds.source_code) = ''
|
||||
OR ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN'
|
||||
OR (ds.enabled = 0 AND (ds.remark LIKE 'auto-retired:%' OR ds.remark = 'auto-reenabled: source evidence restored')))`
|
||||
|
||||
const syncJT808DataSourcesSQL = `
|
||||
INSERT INTO vehicle_data_source
|
||||
(protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind, first_seen_at, latest_seen_at)
|
||||
SELECT
|
||||
inferred.protocol,
|
||||
inferred.source_ip,
|
||||
inferred.latest_source_endpoint,
|
||||
inferred.platform_name,
|
||||
inferred.source_code,
|
||||
'PLATFORM',
|
||||
inferred.first_seen_at,
|
||||
inferred.latest_seen_at
|
||||
FROM (` + jt808DataSourceInferenceSQL + `) inferred
|
||||
LEFT JOIN vehicle_data_source ds
|
||||
ON ds.protocol = inferred.protocol
|
||||
AND ds.source_ip = inferred.source_ip
|
||||
WHERE ` + jt808DataSourceCandidateWhereSQL + `
|
||||
AND ` + jt808DataSourceNeedsSyncWhereSQL + `
|
||||
ON DUPLICATE KEY UPDATE
|
||||
latest_source_endpoint = VALUES(latest_source_endpoint),
|
||||
platform_name = CASE
|
||||
WHEN vehicle_data_source.platform_name IS NULL OR TRIM(vehicle_data_source.platform_name) = ''
|
||||
THEN VALUES(platform_name)
|
||||
ELSE vehicle_data_source.platform_name
|
||||
END,
|
||||
source_code = CASE
|
||||
WHEN vehicle_data_source.source_code IS NULL OR TRIM(vehicle_data_source.source_code) = ''
|
||||
THEN VALUES(source_code)
|
||||
ELSE vehicle_data_source.source_code
|
||||
END,
|
||||
source_kind = CASE
|
||||
WHEN vehicle_data_source.source_kind IS NULL OR TRIM(vehicle_data_source.source_kind) = '' OR vehicle_data_source.source_kind = 'UNKNOWN'
|
||||
THEN VALUES(source_kind)
|
||||
ELSE vehicle_data_source.source_kind
|
||||
END,
|
||||
enabled = CASE
|
||||
WHEN vehicle_data_source.enabled = 0
|
||||
AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored')
|
||||
AND (
|
||||
(VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '')
|
||||
OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '')
|
||||
OR VALUES(source_kind) <> 'UNKNOWN'
|
||||
)
|
||||
THEN 1
|
||||
ELSE vehicle_data_source.enabled
|
||||
END,
|
||||
remark = CASE
|
||||
WHEN vehicle_data_source.enabled = 0
|
||||
AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored')
|
||||
AND (
|
||||
(VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '')
|
||||
OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '')
|
||||
OR VALUES(source_kind) <> 'UNKNOWN'
|
||||
)
|
||||
THEN 'auto-reenabled: source evidence restored'
|
||||
ELSE vehicle_data_source.remark
|
||||
END,
|
||||
first_seen_at = COALESCE(vehicle_data_source.first_seen_at, VALUES(first_seen_at)),
|
||||
latest_seen_at = GREATEST(COALESCE(vehicle_data_source.latest_seen_at, VALUES(latest_seen_at)), VALUES(latest_seen_at)),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`
|
||||
|
||||
const reprojectSelectableDailyMileageSourcesSQL = `
|
||||
SELECT DISTINCT s.vin, s.stat_date, s.protocol
|
||||
FROM vehicle_daily_mileage_source s
|
||||
LEFT JOIN vehicle_data_source ds
|
||||
ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip
|
||||
LEFT JOIN vehicle_daily_mileage m
|
||||
ON m.vin = s.vin AND m.stat_date = s.stat_date AND m.protocol = s.protocol
|
||||
WHERE s.protocol = 'JT808'
|
||||
AND s.quality_status = 'OK'
|
||||
AND (ds.id IS NULL OR ds.enabled = 1 OR (
|
||||
COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN'
|
||||
AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')
|
||||
AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')
|
||||
))
|
||||
AND (
|
||||
m.vin IS NULL
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM vehicle_daily_mileage_source selected
|
||||
WHERE selected.vin = s.vin
|
||||
AND selected.stat_date = s.stat_date
|
||||
AND selected.protocol = s.protocol
|
||||
AND selected.is_selected = 1
|
||||
)
|
||||
)
|
||||
ORDER BY s.stat_date DESC, s.vin ASC
|
||||
`
|
||||
|
||||
const syncJT808DataSourcesCandidateCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM (` + jt808DataSourceInferenceSQL + `) inferred
|
||||
LEFT JOIN vehicle_data_source ds
|
||||
ON ds.protocol = inferred.protocol
|
||||
AND ds.source_ip = inferred.source_ip
|
||||
WHERE ` + jt808DataSourceCandidateWhereSQL + `
|
||||
AND ` + jt808DataSourceNeedsSyncWhereSQL
|
||||
|
||||
const syncJT808DataSourcesSkippedCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM (` + jt808DataSourceInferenceSQL + `) inferred
|
||||
WHERE inferred.source_ip IS NOT NULL
|
||||
AND inferred.source_ip <> ''
|
||||
AND inferred.platform_name IS NOT NULL
|
||||
AND inferred.platform_name <> ''
|
||||
AND NOT (` + jt808DataSourceCandidateWhereSQL + `)`
|
||||
|
||||
const syncJT808DataSourcesConflictCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM (` + jt808DataSourceInferenceSQL + `) inferred
|
||||
JOIN vehicle_data_source ds
|
||||
ON ds.protocol = inferred.protocol
|
||||
AND ds.source_ip = inferred.source_ip
|
||||
WHERE ` + jt808DataSourceCandidateWhereSQL + `
|
||||
AND ds.source_code IS NOT NULL
|
||||
AND TRIM(ds.source_code) <> ''
|
||||
AND ds.source_code <> inferred.source_code`
|
||||
|
||||
const classifyConfiguredDataSourcesWhereSQL = `
|
||||
(source_kind IS NULL OR TRIM(source_kind) = '' OR source_kind = 'UNKNOWN')
|
||||
AND (
|
||||
(source_code IS NOT NULL AND TRIM(source_code) <> '')
|
||||
OR (platform_name IS NOT NULL AND TRIM(platform_name) <> '')
|
||||
)`
|
||||
|
||||
const classifyConfiguredDataSourcesCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_data_source
|
||||
WHERE ` + classifyConfiguredDataSourcesWhereSQL
|
||||
|
||||
const classifyConfiguredDataSourcesSQL = `
|
||||
UPDATE vehicle_data_source
|
||||
SET source_kind = 'PLATFORM',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE ` + classifyConfiguredDataSourcesWhereSQL
|
||||
|
||||
const pruneUnmanagedDataSourcesWhereSQL = `
|
||||
(ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')
|
||||
AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')
|
||||
AND (ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN')
|
||||
AND TIMESTAMPDIFF(SECOND, COALESCE(ds.latest_seen_at, ds.updated_at, ds.created_at), CURRENT_TIMESTAMP) >= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM vehicle_daily_mileage m
|
||||
WHERE m.source_id = ds.id
|
||||
LIMIT 1
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM jt808_registration r
|
||||
WHERE ds.protocol = 'JT808'
|
||||
AND r.source_ip = ds.source_ip
|
||||
LIMIT 1
|
||||
)`
|
||||
|
||||
const pruneUnmanagedDataSourcesCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_data_source ds
|
||||
WHERE ` + pruneUnmanagedDataSourcesWhereSQL
|
||||
|
||||
const pruneUnmanagedDataSourcesSQL = `
|
||||
DELETE ds
|
||||
FROM vehicle_data_source ds
|
||||
WHERE ` + pruneUnmanagedDataSourcesWhereSQL
|
||||
|
||||
const retireStaleUnmanagedDataSourcesWhereSQL = `
|
||||
ds.enabled = 1
|
||||
AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')
|
||||
AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')
|
||||
AND (ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN')
|
||||
AND TIMESTAMPDIFF(SECOND, COALESCE(ds.latest_seen_at, ds.updated_at, ds.created_at), CURRENT_TIMESTAMP) >= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM jt808_registration r
|
||||
WHERE ds.protocol = 'JT808'
|
||||
AND r.source_ip = ds.source_ip
|
||||
LIMIT 1
|
||||
)`
|
||||
|
||||
const retireStaleUnmanagedDataSourcesCountSQL = `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_data_source ds
|
||||
WHERE ` + retireStaleUnmanagedDataSourcesWhereSQL
|
||||
|
||||
const retireStaleUnmanagedDataSourcesSQL = `
|
||||
UPDATE vehicle_data_source ds
|
||||
SET enabled = 0,
|
||||
remark = CASE
|
||||
WHEN remark IS NULL OR TRIM(remark) = ''
|
||||
THEN 'auto-retired: stale unmanaged source without registration evidence'
|
||||
ELSE remark
|
||||
END,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE ` + retireStaleUnmanagedDataSourcesWhereSQL
|
||||
|
||||
func fail(err error) {
|
||||
_, _ = fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
Reference in New Issue
Block a user