feat: expand vehicle data platform capabilities
This commit is contained in:
743
go/vehicle-gateway/cmd/legacy-mileage-migrate/main.go
Normal file
743
go/vehicle-gateway/cmd/legacy-mileage-migrate/main.go
Normal file
@@ -0,0 +1,743 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mysql "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
const (
|
||||
protocol = "GB32960"
|
||||
legacySourceIP = "legacy-mysql.lingniu-prod"
|
||||
legacySourceKey = "GB32960:LEGACY_MILEAGE@legacy-mysql.lingniu-prod"
|
||||
legacySourceCode = "legacy_lingniu_prod"
|
||||
legacyPlatformName = "历史生产库里程"
|
||||
legacyQuality = "legacy_mysql_backfill_before_2026-07-21"
|
||||
maxDailyMileageKM = 2500.0
|
||||
)
|
||||
|
||||
var vinPattern = regexp.MustCompile(`^[A-HJ-NPR-Z0-9]{17}$`)
|
||||
|
||||
type sourceRecord struct {
|
||||
ID int64
|
||||
Identifier string
|
||||
StatDate time.Time
|
||||
DailyKM sql.NullFloat64
|
||||
TotalKM sql.NullFloat64
|
||||
Now sql.NullTime
|
||||
MappingOrigin string
|
||||
VIN string
|
||||
}
|
||||
|
||||
type mappingIndex struct {
|
||||
Unique map[string]string
|
||||
Ambiguous map[string]struct{}
|
||||
}
|
||||
|
||||
type scanReport struct {
|
||||
SourceRows int `json:"source_rows"`
|
||||
SourceDistinctIdentifiers int `json:"source_distinct_identifiers"`
|
||||
DedupedSourceRows int `json:"deduped_source_rows"`
|
||||
DuplicateRows int `json:"duplicate_rows"`
|
||||
ConflictingDuplicateGroups int `json:"conflicting_duplicate_groups"`
|
||||
MappedByCurrentVehicle int `json:"mapped_by_current_vehicle"`
|
||||
MappedByLegacyVehicle int `json:"mapped_by_legacy_vehicle"`
|
||||
CurrentLegacyVINConflicts int `json:"current_legacy_vin_conflicts"`
|
||||
UnresolvedRows int `json:"unresolved_rows"`
|
||||
UnresolvedIdentifiers int `json:"unresolved_identifiers"`
|
||||
UnresolvedEligibleRows int `json:"unresolved_eligible_rows"`
|
||||
UnresolvedIdentifierValues []string `json:"unresolved_identifier_values,omitempty"`
|
||||
NullMileageRows int `json:"null_mileage_rows"`
|
||||
ZeroMileageRows int `json:"zero_mileage_rows"`
|
||||
NegativeMileageRows int `json:"negative_mileage_rows"`
|
||||
OverLimitMileageRows int `json:"over_limit_mileage_rows"`
|
||||
EligibleRows int `json:"eligible_rows"`
|
||||
VINDateCollisions int `json:"vin_date_collisions"`
|
||||
ConflictingVINDateCollisions int `json:"conflicting_vin_date_collisions"`
|
||||
}
|
||||
|
||||
type targetReport struct {
|
||||
StageRows int64 `json:"stage_rows"`
|
||||
MissingRows int64 `json:"missing_rows"`
|
||||
ZeroRows int64 `json:"zero_rows"`
|
||||
NonzeroProtectedRows int64 `json:"nonzero_protected_rows"`
|
||||
ExistingPositiveCandidateRows int64 `json:"existing_positive_candidate_rows"`
|
||||
ProtectedSameMileageRows int64 `json:"protected_same_mileage_rows"`
|
||||
ProtectedMileageSumKM float64 `json:"protected_mileage_sum_km"`
|
||||
ProtectedMileageFingerprint uint64 `json:"protected_mileage_fingerprint"`
|
||||
LegacyCandidateRows int64 `json:"legacy_candidate_rows"`
|
||||
LegacyCandidateTotalMatchedRows int64 `json:"legacy_candidate_total_matched_rows"`
|
||||
LegacyProjectedRows int64 `json:"legacy_projected_rows"`
|
||||
LegacyProjectedTotalMatchedRows int64 `json:"legacy_projected_total_matched_rows"`
|
||||
LegacySelectedCandidateRows int64 `json:"legacy_selected_candidate_rows"`
|
||||
PreexistingNonLegacyRows int64 `json:"preexisting_non_legacy_rows"`
|
||||
PreexistingNonLegacyMileageSumKM float64 `json:"preexisting_non_legacy_mileage_sum_km"`
|
||||
PreexistingNonLegacyFingerprint uint64 `json:"preexisting_non_legacy_fingerprint"`
|
||||
RemainingMissingOrZeroRows int64 `json:"remaining_missing_or_zero_rows"`
|
||||
PostProtectedRows int64 `json:"post_protected_rows"`
|
||||
PostProtectedMileageSumKM float64 `json:"post_protected_mileage_sum_km"`
|
||||
PostProtectedMileageFingerprint uint64 `json:"post_protected_mileage_fingerprint"`
|
||||
ProtectedRowsUnchanged int64 `json:"protected_rows_unchanged"`
|
||||
ProtectedMileageFingerprintStable bool `json:"protected_mileage_fingerprint_stable"`
|
||||
}
|
||||
|
||||
type output struct {
|
||||
Mode string `json:"mode"`
|
||||
CutoffExclusive string `json:"cutoff_exclusive"`
|
||||
Protocol string `json:"protocol"`
|
||||
SourceTable string `json:"source_table"`
|
||||
TotalMileageDivisor float64 `json:"total_mileage_divisor_m_to_km"`
|
||||
Scan scanReport `json:"scan"`
|
||||
Target targetReport `json:"target"`
|
||||
Duration string `json:"duration"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
var cutoffRaw, dsn string
|
||||
var apply bool
|
||||
var timeout time.Duration
|
||||
flag.StringVar(&cutoffRaw, "before", "2026-07-21", "exclusive upper date boundary (YYYY-MM-DD)")
|
||||
flag.StringVar(&dsn, "mysql-dsn", strings.TrimSpace(os.Getenv("MYSQL_DSN")), "target MySQL DSN")
|
||||
flag.BoolVar(&apply, "apply", false, "apply the migration; default is dry-run")
|
||||
flag.DurationVar(&timeout, "timeout", 10*time.Minute, "overall timeout")
|
||||
flag.Parse()
|
||||
|
||||
started := time.Now()
|
||||
cutoff, err := time.ParseInLocation("2006-01-02", strings.TrimSpace(cutoffRaw), time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid -before date: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(dsn) == "" {
|
||||
return errors.New("MYSQL_DSN or -mysql-dsn is required")
|
||||
}
|
||||
dsn, err = normalizedDSN(dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(2)
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
currentMappings, err := loadMappingIndex(ctx, conn, `
|
||||
SELECT CONVERT(plate USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM vehicle
|
||||
WHERE plate IS NOT NULL AND TRIM(plate) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM vehicle_identifier
|
||||
WHERE enabled = 1 AND plate IS NOT NULL AND TRIM(plate) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load current vehicle mappings: %w", err)
|
||||
}
|
||||
legacyMappings, err := loadMappingIndex(ctx, conn, `
|
||||
SELECT CONVERT(plate_number USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.tab_truck
|
||||
WHERE plate_number IS NOT NULL AND TRIM(plate_number) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate_number USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.truck_info
|
||||
WHERE plate_number IS NOT NULL AND TRIM(plate_number) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.v_vehicle_daily_stats
|
||||
WHERE plate IS NOT NULL AND TRIM(plate) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate_number USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.tab_truck_remote_sync_realtime_info
|
||||
WHERE plate_number IS NOT NULL AND TRIM(plate_number) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate_number USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.view_truck_comprehensive_info
|
||||
WHERE plate_number IS NOT NULL AND TRIM(plate_number) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate_number USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin_code USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.tab_hecri_hydrogen_order
|
||||
WHERE plate_number IS NOT NULL AND TRIM(plate_number) <> '' AND vin_code IS NOT NULL AND TRIM(vin_code) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.bi_ele_charge_record
|
||||
WHERE plate IS NOT NULL AND TRIM(plate) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate_number USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.v_truck_task_basic
|
||||
WHERE plate_number IS NOT NULL AND TRIM(plate_number) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT CONVERT(plate_number USING utf8mb4) COLLATE utf8mb4_unicode_ci AS plate,
|
||||
CONVERT(vin USING utf8mb4) COLLATE utf8mb4_unicode_ci AS vin FROM lingniu_prod.view_vehicle_annual_inspection
|
||||
WHERE plate_number IS NOT NULL AND TRIM(plate_number) <> '' AND vin IS NOT NULL AND TRIM(vin) <> ''`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load legacy vehicle mappings: %w", err)
|
||||
}
|
||||
records, report, err := loadAndResolveSource(ctx, conn, cutoff, currentMappings, legacyMappings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := createAndLoadStage(ctx, conn, records); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := inspectTarget(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mode := "dry_run"
|
||||
if apply {
|
||||
mode = "apply"
|
||||
if err := createProtectedSnapshot(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := applyMigration(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
post, err := inspectTarget(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
post.PostProtectedRows, post.PostProtectedMileageSumKM, post.PostProtectedMileageFingerprint, post.ProtectedRowsUnchanged, err = inspectProtectedSnapshot(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
post.ProtectedMileageFingerprintStable =
|
||||
target.PreexistingNonLegacyRows == post.PostProtectedRows &&
|
||||
math.Abs(target.PreexistingNonLegacyMileageSumKM-post.PostProtectedMileageSumKM) < 0.0005 &&
|
||||
target.PreexistingNonLegacyFingerprint == post.PostProtectedMileageFingerprint &&
|
||||
post.ProtectedRowsUnchanged == target.PreexistingNonLegacyRows
|
||||
post.NonzeroProtectedRows = target.NonzeroProtectedRows
|
||||
post.ProtectedSameMileageRows = target.ProtectedSameMileageRows
|
||||
post.ProtectedMileageSumKM = target.ProtectedMileageSumKM
|
||||
post.ProtectedMileageFingerprint = target.ProtectedMileageFingerprint
|
||||
target = post
|
||||
}
|
||||
|
||||
result := output{
|
||||
Mode: mode, CutoffExclusive: cutoff.Format("2006-01-02"), Protocol: protocol,
|
||||
SourceTable: "lingniu_prod.ln_vehicle_day_mileage", TotalMileageDivisor: 1000, Scan: report, Target: target,
|
||||
Duration: time.Since(started).Round(time.Millisecond).String(),
|
||||
}
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(result)
|
||||
}
|
||||
|
||||
func createProtectedSnapshot(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
CREATE TEMPORARY TABLE tmp_legacy_mileage_protected AS
|
||||
SELECT m.vin, m.stat_date, m.protocol, m.source_id, m.daily_mileage_km,
|
||||
m.latest_total_mileage_km, m.updated_at
|
||||
FROM tmp_legacy_mileage_stage st
|
||||
JOIN vehicle_daily_mileage m
|
||||
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
|
||||
LEFT JOIN vehicle_data_source ds ON ds.id = m.source_id
|
||||
WHERE m.daily_mileage_km > 0 AND COALESCE(ds.source_ip, '') <> ?`, legacySourceIP)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot protected mileage rows: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func inspectProtectedSnapshot(ctx context.Context, queryer interface {
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}) (count int64, sum float64, fingerprint uint64, unchanged int64, err error) {
|
||||
var nullableSum sql.NullFloat64
|
||||
err = queryer.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
COUNT(m.vin),
|
||||
SUM(CASE WHEN m.vin IS NOT NULL THEN m.daily_mileage_km ELSE 0 END),
|
||||
COALESCE(BIT_XOR(CASE WHEN m.vin IS NOT NULL THEN CRC32(CONCAT_WS('|', m.vin, m.stat_date, m.protocol, m.daily_mileage_km, COALESCE(m.latest_total_mileage_km, 'NULL'))) ELSE 0 END), 0),
|
||||
COALESCE(SUM(
|
||||
m.vin IS NOT NULL
|
||||
AND (m.source_id <=> p.source_id)
|
||||
AND (m.daily_mileage_km <=> p.daily_mileage_km)
|
||||
AND (m.latest_total_mileage_km <=> p.latest_total_mileage_km)
|
||||
AND (m.updated_at <=> p.updated_at)
|
||||
), 0)
|
||||
FROM tmp_legacy_mileage_protected p
|
||||
LEFT JOIN vehicle_daily_mileage m
|
||||
ON m.vin = p.vin AND m.stat_date = p.stat_date AND m.protocol = p.protocol`).Scan(
|
||||
&count, &nullableSum, &fingerprint, &unchanged,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, 0, 0, 0, fmt.Errorf("verify protected mileage snapshot: %w", err)
|
||||
}
|
||||
if nullableSum.Valid {
|
||||
sum = nullableSum.Float64
|
||||
}
|
||||
return count, sum, fingerprint, unchanged, nil
|
||||
}
|
||||
|
||||
func normalizedDSN(raw string) (string, error) {
|
||||
cfg, err := mysql.ParseDSN(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse mysql dsn: %w", err)
|
||||
}
|
||||
cfg.ParseTime = true
|
||||
cfg.Loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
cfg.Params = cloneMap(cfg.Params)
|
||||
cfg.Params["charset"] = "utf8mb4"
|
||||
return cfg.FormatDSN(), nil
|
||||
}
|
||||
|
||||
func cloneMap(source map[string]string) map[string]string {
|
||||
result := make(map[string]string, len(source)+1)
|
||||
for key, value := range source {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func loadMappingIndex(ctx context.Context, queryer interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}, query string) (mappingIndex, error) {
|
||||
rows, err := queryer.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return mappingIndex{}, err
|
||||
}
|
||||
defer rows.Close()
|
||||
values := map[string]map[string]struct{}{}
|
||||
for rows.Next() {
|
||||
var plate, vin sql.NullString
|
||||
if err := rows.Scan(&plate, &vin); err != nil {
|
||||
return mappingIndex{}, err
|
||||
}
|
||||
plateValue := normalizeIdentifier(plate.String)
|
||||
vinValue := normalizeVIN(vin.String)
|
||||
if plateValue == "" || !vinPattern.MatchString(vinValue) {
|
||||
continue
|
||||
}
|
||||
if values[plateValue] == nil {
|
||||
values[plateValue] = map[string]struct{}{}
|
||||
}
|
||||
values[plateValue][vinValue] = struct{}{}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return mappingIndex{}, err
|
||||
}
|
||||
result := mappingIndex{Unique: map[string]string{}, Ambiguous: map[string]struct{}{}}
|
||||
for plate, vins := range values {
|
||||
if len(vins) != 1 {
|
||||
result.Ambiguous[plate] = struct{}{}
|
||||
continue
|
||||
}
|
||||
for vin := range vins {
|
||||
result.Unique[plate] = vin
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func loadAndResolveSource(ctx context.Context, queryer interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}, cutoff time.Time, current, legacy mappingIndex) ([]sourceRecord, scanReport, error) {
|
||||
rows, err := queryer.QueryContext(ctx, `
|
||||
SELECT id, plateNumber, dates, dayMileage, totalMileage, nowDatetime
|
||||
FROM lingniu_prod.ln_vehicle_day_mileage
|
||||
WHERE dates < ?
|
||||
ORDER BY plateNumber, dates, (nowDatetime IS NULL), nowDatetime DESC, id DESC`, cutoff)
|
||||
if err != nil {
|
||||
return nil, scanReport{}, fmt.Errorf("query source mileage: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var report scanReport
|
||||
distinctIdentifiers := map[string]struct{}{}
|
||||
unresolvedIdentifiers := map[string]struct{}{}
|
||||
byIdentifierDate := map[string]sourceRecord{}
|
||||
duplicateValues := map[string]float64{}
|
||||
conflictingDuplicateKeys := map[string]struct{}{}
|
||||
for rows.Next() {
|
||||
var item sourceRecord
|
||||
var rawIdentifier sql.NullString
|
||||
if err := rows.Scan(&item.ID, &rawIdentifier, &item.StatDate, &item.DailyKM, &item.TotalKM, &item.Now); err != nil {
|
||||
return nil, report, err
|
||||
}
|
||||
report.SourceRows++
|
||||
item.Identifier = normalizeIdentifier(rawIdentifier.String)
|
||||
distinctIdentifiers[item.Identifier] = struct{}{}
|
||||
key := item.Identifier + "\x00" + item.StatDate.Format("2006-01-02")
|
||||
if existing, ok := byIdentifierDate[key]; ok {
|
||||
report.DuplicateRows++
|
||||
if item.DailyKM.Valid && existing.DailyKM.Valid && math.Abs(item.DailyKM.Float64-existing.DailyKM.Float64) > 0.0005 {
|
||||
conflictingDuplicateKeys[key] = struct{}{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
byIdentifierDate[key] = item
|
||||
if item.DailyKM.Valid {
|
||||
duplicateValues[key] = item.DailyKM.Float64
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, report, err
|
||||
}
|
||||
_ = duplicateValues
|
||||
report.SourceDistinctIdentifiers = len(distinctIdentifiers)
|
||||
report.DedupedSourceRows = len(byIdentifierDate)
|
||||
report.ConflictingDuplicateGroups = len(conflictingDuplicateKeys)
|
||||
|
||||
keys := make([]string, 0, len(byIdentifierDate))
|
||||
for key := range byIdentifierDate {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
stageByVINDate := map[string]sourceRecord{}
|
||||
var stageOrder []string
|
||||
for _, key := range keys {
|
||||
item := byIdentifierDate[key]
|
||||
vin, origin, conflict := resolveVIN(item.Identifier, current, legacy)
|
||||
if conflict {
|
||||
report.CurrentLegacyVINConflicts++
|
||||
}
|
||||
if vin == "" {
|
||||
report.UnresolvedRows++
|
||||
unresolvedIdentifiers[item.Identifier] = struct{}{}
|
||||
if item.DailyKM.Valid && item.DailyKM.Float64 > 0 && item.DailyKM.Float64 <= maxDailyMileageKM {
|
||||
report.UnresolvedEligibleRows++
|
||||
}
|
||||
continue
|
||||
}
|
||||
item.VIN = vin
|
||||
item.MappingOrigin = origin
|
||||
if origin == "current_vehicle" {
|
||||
report.MappedByCurrentVehicle++
|
||||
} else {
|
||||
report.MappedByLegacyVehicle++
|
||||
}
|
||||
if !item.DailyKM.Valid {
|
||||
report.NullMileageRows++
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case item.DailyKM.Float64 == 0:
|
||||
report.ZeroMileageRows++
|
||||
continue
|
||||
case item.DailyKM.Float64 < 0:
|
||||
report.NegativeMileageRows++
|
||||
continue
|
||||
case item.DailyKM.Float64 > maxDailyMileageKM:
|
||||
report.OverLimitMileageRows++
|
||||
continue
|
||||
}
|
||||
report.EligibleRows++
|
||||
stageKey := item.VIN + "\x00" + item.StatDate.Format("2006-01-02")
|
||||
if existing, ok := stageByVINDate[stageKey]; ok {
|
||||
report.VINDateCollisions++
|
||||
if math.Abs(existing.DailyKM.Float64-item.DailyKM.Float64) > 0.0005 {
|
||||
report.ConflictingVINDateCollisions++
|
||||
}
|
||||
if item.ID <= existing.ID {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
stageOrder = append(stageOrder, stageKey)
|
||||
}
|
||||
stageByVINDate[stageKey] = item
|
||||
}
|
||||
report.UnresolvedIdentifiers = len(unresolvedIdentifiers)
|
||||
for identifier := range unresolvedIdentifiers {
|
||||
report.UnresolvedIdentifierValues = append(report.UnresolvedIdentifierValues, identifier)
|
||||
}
|
||||
sort.Strings(report.UnresolvedIdentifierValues)
|
||||
sort.Strings(stageOrder)
|
||||
stage := make([]sourceRecord, 0, len(stageOrder))
|
||||
for _, key := range stageOrder {
|
||||
stage = append(stage, stageByVINDate[key])
|
||||
}
|
||||
return stage, report, nil
|
||||
}
|
||||
|
||||
func resolveVIN(identifier string, current, legacy mappingIndex) (vin, origin string, conflict bool) {
|
||||
if vinPattern.MatchString(identifier) {
|
||||
return identifier, "source_vin", false
|
||||
}
|
||||
currentVIN, currentOK := current.Unique[identifier]
|
||||
legacyVIN, legacyOK := legacy.Unique[identifier]
|
||||
if currentOK && legacyOK && currentVIN != legacyVIN {
|
||||
return "", "", true
|
||||
}
|
||||
if currentOK {
|
||||
return currentVIN, "current_vehicle", false
|
||||
}
|
||||
if legacyOK {
|
||||
return legacyVIN, "legacy_vehicle", false
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
func normalizeIdentifier(value string) string {
|
||||
return strings.ToUpper(strings.Join(strings.Fields(strings.TrimSpace(value)), ""))
|
||||
}
|
||||
|
||||
func normalizeVIN(value string) string {
|
||||
return strings.ToUpper(strings.TrimSpace(value))
|
||||
}
|
||||
|
||||
func createAndLoadStage(ctx context.Context, conn *sql.Conn, records []sourceRecord) error {
|
||||
if _, err := conn.ExecContext(ctx, `
|
||||
CREATE TEMPORARY TABLE tmp_legacy_mileage_stage (
|
||||
vin VARCHAR(32) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
daily_mileage_km DECIMAL(18,3) NOT NULL,
|
||||
total_mileage_km DECIMAL(18,3) NULL,
|
||||
source_record_id BIGINT NOT NULL,
|
||||
mapping_origin VARCHAR(32) NOT NULL,
|
||||
PRIMARY KEY (vin, stat_date)
|
||||
) ENGINE=InnoDB`); err != nil {
|
||||
return fmt.Errorf("create temporary stage: %w", err)
|
||||
}
|
||||
const batchSize = 500
|
||||
for start := 0; start < len(records); start += batchSize {
|
||||
end := min(start+batchSize, len(records))
|
||||
var query strings.Builder
|
||||
query.WriteString("INSERT INTO tmp_legacy_mileage_stage (vin, stat_date, daily_mileage_km, total_mileage_km, source_record_id, mapping_origin) VALUES ")
|
||||
args := make([]any, 0, (end-start)*6)
|
||||
for index, item := range records[start:end] {
|
||||
if index > 0 {
|
||||
query.WriteByte(',')
|
||||
}
|
||||
query.WriteString("(?,?,?,?,?,?)")
|
||||
args = append(args, item.VIN, item.StatDate, item.DailyKM.Float64, nullableTotalMileageKM(item.TotalKM), item.ID, item.MappingOrigin)
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, query.String(), args...); err != nil {
|
||||
return fmt.Errorf("load temporary stage batch %d: %w", start/batchSize+1, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func nullableTotalMileageKM(value sql.NullFloat64) any {
|
||||
if !value.Valid {
|
||||
return nil
|
||||
}
|
||||
return value.Float64 / 1000
|
||||
}
|
||||
|
||||
func inspectTarget(ctx context.Context, queryer interface {
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}) (targetReport, error) {
|
||||
var report targetReport
|
||||
var protectedSum sql.NullFloat64
|
||||
err := queryer.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
COUNT(*),
|
||||
COALESCE(SUM(m.vin IS NULL), 0),
|
||||
COALESCE(SUM(m.vin IS NOT NULL AND m.daily_mileage_km = 0), 0),
|
||||
COALESCE(SUM(m.daily_mileage_km > 0), 0),
|
||||
COALESCE(SUM(m.daily_mileage_km > 0 AND ABS(m.daily_mileage_km - st.daily_mileage_km) <= 0.011), 0),
|
||||
SUM(CASE WHEN m.daily_mileage_km > 0 THEN m.daily_mileage_km ELSE 0 END),
|
||||
COALESCE(BIT_XOR(CASE WHEN m.daily_mileage_km > 0 THEN CRC32(CONCAT_WS('|', m.vin, m.stat_date, m.protocol, m.daily_mileage_km, COALESCE(m.latest_total_mileage_km, 'NULL'))) ELSE 0 END), 0),
|
||||
COALESCE(SUM(candidate.vin IS NOT NULL AND (m.vin IS NULL OR m.daily_mileage_km = 0)), 0)
|
||||
FROM tmp_legacy_mileage_stage st
|
||||
LEFT JOIN vehicle_daily_mileage m
|
||||
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
|
||||
LEFT JOIN (
|
||||
SELECT vin, stat_date
|
||||
FROM vehicle_daily_mileage_source
|
||||
WHERE protocol = 'GB32960' AND quality_status = 'OK' AND daily_mileage_km > 0
|
||||
GROUP BY vin, stat_date
|
||||
) candidate ON candidate.vin = st.vin AND candidate.stat_date = st.stat_date`).Scan(
|
||||
&report.StageRows, &report.MissingRows, &report.ZeroRows, &report.NonzeroProtectedRows,
|
||||
&report.ProtectedSameMileageRows, &protectedSum, &report.ProtectedMileageFingerprint,
|
||||
&report.ExistingPositiveCandidateRows,
|
||||
)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("inspect migration target: %w", err)
|
||||
}
|
||||
if protectedSum.Valid {
|
||||
report.ProtectedMileageSumKM = protectedSum.Float64
|
||||
}
|
||||
if err := queryer.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*), COALESCE(SUM(s.latest_total_mileage_km <=> st.total_mileage_km), 0)
|
||||
FROM vehicle_daily_mileage_source s
|
||||
JOIN tmp_legacy_mileage_stage st ON st.vin = s.vin AND st.stat_date = s.stat_date
|
||||
WHERE s.protocol = 'GB32960' AND s.source_key = ?`, legacySourceKey).Scan(
|
||||
&report.LegacyCandidateRows, &report.LegacyCandidateTotalMatchedRows,
|
||||
); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := queryer.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_daily_mileage_source s
|
||||
JOIN tmp_legacy_mileage_stage st ON st.vin = s.vin AND st.stat_date = s.stat_date
|
||||
WHERE s.protocol = 'GB32960' AND s.source_key = ? AND s.is_selected = 1`, legacySourceKey).Scan(&report.LegacySelectedCandidateRows); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := queryer.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*), COALESCE(SUM(m.latest_total_mileage_km <=> st.total_mileage_km), 0)
|
||||
FROM vehicle_daily_mileage m
|
||||
JOIN tmp_legacy_mileage_stage st ON st.vin = m.vin AND st.stat_date = m.stat_date
|
||||
JOIN vehicle_data_source ds ON ds.id = m.source_id
|
||||
WHERE m.protocol = 'GB32960' AND ds.protocol = 'GB32960' AND ds.source_ip = ?
|
||||
AND ABS(m.daily_mileage_km - st.daily_mileage_km) <= 0.0005`, legacySourceIP).Scan(
|
||||
&report.LegacyProjectedRows, &report.LegacyProjectedTotalMatchedRows,
|
||||
); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := queryer.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM tmp_legacy_mileage_stage st
|
||||
LEFT JOIN vehicle_daily_mileage m
|
||||
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
|
||||
WHERE m.vin IS NULL OR m.daily_mileage_km = 0`).Scan(&report.RemainingMissingOrZeroRows); err != nil {
|
||||
return report, err
|
||||
}
|
||||
var nonLegacySum sql.NullFloat64
|
||||
if err := queryer.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
COUNT(*),
|
||||
SUM(m.daily_mileage_km),
|
||||
COALESCE(BIT_XOR(CRC32(CONCAT_WS('|', m.vin, m.stat_date, m.protocol, m.daily_mileage_km, COALESCE(m.latest_total_mileage_km, 'NULL')))), 0)
|
||||
FROM vehicle_daily_mileage m
|
||||
JOIN tmp_legacy_mileage_stage st ON st.vin = m.vin AND st.stat_date = m.stat_date
|
||||
LEFT JOIN vehicle_data_source ds ON ds.id = m.source_id
|
||||
WHERE m.protocol = 'GB32960' AND m.daily_mileage_km > 0
|
||||
AND COALESCE(ds.source_ip, '') <> ?`, legacySourceIP).Scan(
|
||||
&report.PreexistingNonLegacyRows, &nonLegacySum, &report.PreexistingNonLegacyFingerprint,
|
||||
); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if nonLegacySum.Valid {
|
||||
report.PreexistingNonLegacyMileageSumKM = nonLegacySum.Float64
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func applyMigration(ctx context.Context, conn *sql.Conn) error {
|
||||
tx, err := conn.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO vehicle_data_source
|
||||
(protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind,
|
||||
trust_priority, enabled, first_seen_at, latest_seen_at, remark)
|
||||
VALUES ('GB32960', ?, 'lingniu_prod.ln_vehicle_day_mileage', ?, ?, 'UNKNOWN', 1000, 1,
|
||||
'2024-10-17 00:00:00', '2026-07-20 23:59:59',
|
||||
'仅用于迁移 2026-07-21 之前缺失或为 0 的 GB32960 日里程')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
latest_source_endpoint = VALUES(latest_source_endpoint),
|
||||
platform_name = VALUES(platform_name),
|
||||
source_code = VALUES(source_code),
|
||||
source_kind = VALUES(source_kind),
|
||||
trust_priority = VALUES(trust_priority),
|
||||
enabled = 1,
|
||||
remark = VALUES(remark)`, legacySourceIP, legacyPlatformName, legacySourceCode); err != nil {
|
||||
return fmt.Errorf("upsert legacy data source: %w", err)
|
||||
}
|
||||
var sourceID int64
|
||||
if err := tx.QueryRowContext(ctx, `SELECT id FROM vehicle_data_source WHERE protocol = 'GB32960' AND source_ip = ?`, legacySourceIP).Scan(&sourceID); err != nil {
|
||||
return fmt.Errorf("lookup legacy data source: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO vehicle_daily_mileage_source
|
||||
(vin, stat_date, protocol, source_key, source_ip, source_endpoint, platform_name,
|
||||
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count,
|
||||
first_event_time, latest_event_time, quality_status, quality_reason, is_selected)
|
||||
SELECT
|
||||
st.vin, st.stat_date, 'GB32960', ?, ?, 'lingniu_prod.ln_vehicle_day_mileage', ?,
|
||||
CASE WHEN st.total_mileage_km IS NOT NULL AND st.total_mileage_km >= st.daily_mileage_km
|
||||
THEN st.total_mileage_km - st.daily_mileage_km ELSE NULL END,
|
||||
st.total_mileage_km, st.daily_mileage_km, 1,
|
||||
TIMESTAMP(st.stat_date, '00:00:00'), TIMESTAMP(st.stat_date, '23:59:59'),
|
||||
'OK', ?, 0
|
||||
FROM tmp_legacy_mileage_stage st
|
||||
LEFT JOIN vehicle_daily_mileage m
|
||||
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
|
||||
WHERE m.vin IS NULL OR m.daily_mileage_km = 0 OR m.source_id = ?
|
||||
ON DUPLICATE KEY UPDATE
|
||||
updated_at = IF(
|
||||
NOT (vehicle_daily_mileage_source.first_total_mileage_km <=> VALUES(first_total_mileage_km))
|
||||
OR NOT (vehicle_daily_mileage_source.latest_total_mileage_km <=> VALUES(latest_total_mileage_km))
|
||||
OR NOT (vehicle_daily_mileage_source.daily_mileage_km <=> VALUES(daily_mileage_km))
|
||||
OR NOT (vehicle_daily_mileage_source.quality_status <=> VALUES(quality_status))
|
||||
OR NOT (vehicle_daily_mileage_source.quality_reason <=> VALUES(quality_reason)),
|
||||
CURRENT_TIMESTAMP,
|
||||
vehicle_daily_mileage_source.updated_at
|
||||
),
|
||||
source_ip = VALUES(source_ip),
|
||||
source_endpoint = VALUES(source_endpoint),
|
||||
platform_name = VALUES(platform_name),
|
||||
first_total_mileage_km = VALUES(first_total_mileage_km),
|
||||
latest_total_mileage_km = VALUES(latest_total_mileage_km),
|
||||
daily_mileage_km = VALUES(daily_mileage_km),
|
||||
sample_count = VALUES(sample_count),
|
||||
first_event_time = VALUES(first_event_time),
|
||||
latest_event_time = VALUES(latest_event_time),
|
||||
quality_status = VALUES(quality_status),
|
||||
quality_reason = VALUES(quality_reason)`, legacySourceKey, legacySourceIP, legacyPlatformName, legacyQuality, sourceID); err != nil {
|
||||
return fmt.Errorf("upsert legacy mileage candidates: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO vehicle_daily_mileage
|
||||
(vin, stat_date, protocol, source_id, daily_mileage_km, latest_total_mileage_km)
|
||||
SELECT st.vin, st.stat_date, 'GB32960', ?, st.daily_mileage_km, st.total_mileage_km
|
||||
FROM tmp_legacy_mileage_stage st
|
||||
LEFT JOIN vehicle_daily_mileage m
|
||||
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
|
||||
WHERE m.vin IS NULL OR m.daily_mileage_km = 0 OR m.source_id = ?
|
||||
ON DUPLICATE KEY UPDATE
|
||||
updated_at = IF(
|
||||
vehicle_daily_mileage.daily_mileage_km = 0
|
||||
OR (
|
||||
(vehicle_daily_mileage.source_id <=> VALUES(source_id))
|
||||
AND (
|
||||
NOT (vehicle_daily_mileage.daily_mileage_km <=> VALUES(daily_mileage_km))
|
||||
OR NOT (vehicle_daily_mileage.latest_total_mileage_km <=> VALUES(latest_total_mileage_km))
|
||||
)
|
||||
),
|
||||
CURRENT_TIMESTAMP,
|
||||
vehicle_daily_mileage.updated_at
|
||||
),
|
||||
source_id = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(source_id), vehicle_daily_mileage.source_id),
|
||||
latest_total_mileage_km = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(latest_total_mileage_km), vehicle_daily_mileage.latest_total_mileage_km),
|
||||
daily_mileage_km = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(daily_mileage_km), vehicle_daily_mileage.daily_mileage_km)`, sourceID, sourceID); err != nil {
|
||||
return fmt.Errorf("upsert final daily mileage: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE vehicle_daily_mileage_source s
|
||||
JOIN tmp_legacy_mileage_stage st
|
||||
ON st.vin = s.vin AND st.stat_date = s.stat_date
|
||||
JOIN vehicle_daily_mileage m
|
||||
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
|
||||
SET s.is_selected = CASE WHEN s.source_key = ? THEN 1 ELSE 0 END
|
||||
WHERE s.protocol = 'GB32960'
|
||||
AND m.source_id = ?
|
||||
AND ABS(m.daily_mileage_km - st.daily_mileage_km) <= 0.0005`, legacySourceKey, sourceID); err != nil {
|
||||
return fmt.Errorf("mark legacy mileage selections: %w", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit migration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
56
go/vehicle-gateway/cmd/legacy-mileage-migrate/main_test.go
Normal file
56
go/vehicle-gateway/cmd/legacy-mileage-migrate/main_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveVINPrefersConsistentCurrentMapping(t *testing.T) {
|
||||
current := mappingIndex{Unique: map[string]string{"沪A12345": "LA9GG64L0NBAF4175"}}
|
||||
legacy := mappingIndex{Unique: map[string]string{"沪A12345": "LA9GG64L0NBAF4175"}}
|
||||
vin, origin, conflict := resolveVIN("沪A12345", current, legacy)
|
||||
if conflict || vin != "LA9GG64L0NBAF4175" || origin != "current_vehicle" {
|
||||
t.Fatalf("unexpected resolution: vin=%q origin=%q conflict=%v", vin, origin, conflict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveVINRejectsCurrentLegacyConflict(t *testing.T) {
|
||||
current := mappingIndex{Unique: map[string]string{"沪A12345": "LA9GG64L0NBAF4175"}}
|
||||
legacy := mappingIndex{Unique: map[string]string{"沪A12345": "LA9GG64L0NBAF4176"}}
|
||||
vin, _, conflict := resolveVIN("沪A12345", current, legacy)
|
||||
if !conflict || vin != "" {
|
||||
t.Fatalf("expected conflict rejection, got vin=%q conflict=%v", vin, conflict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveVINUsesLegacyFallback(t *testing.T) {
|
||||
current := mappingIndex{Unique: map[string]string{}}
|
||||
legacy := mappingIndex{Unique: map[string]string{"沪A12345": "LA9GG64L0NBAF4175"}}
|
||||
vin, origin, conflict := resolveVIN("沪A12345", current, legacy)
|
||||
if conflict || vin != "LA9GG64L0NBAF4175" || origin != "legacy_vehicle" {
|
||||
t.Fatalf("unexpected resolution: vin=%q origin=%q conflict=%v", vin, origin, conflict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveVINAcceptsSourceVIN(t *testing.T) {
|
||||
vin, origin, conflict := resolveVIN("LA9GG64L0NBAF4175", mappingIndex{}, mappingIndex{})
|
||||
if conflict || vin != "LA9GG64L0NBAF4175" || origin != "source_vin" {
|
||||
t.Fatalf("unexpected resolution: vin=%q origin=%q conflict=%v", vin, origin, conflict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeIdentifierRemovesWhitespace(t *testing.T) {
|
||||
if got := normalizeIdentifier(" 沪 a 12345 \n"); got != "沪A12345" {
|
||||
t.Fatalf("normalizeIdentifier() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNullableTotalMileageConvertsMetersToKM(t *testing.T) {
|
||||
got := nullableTotalMileageKM(sql.NullFloat64{Float64: 59198000, Valid: true})
|
||||
if got != 59198.0 {
|
||||
t.Fatalf("nullableTotalMileageKM() = %#v", got)
|
||||
}
|
||||
if got := nullableTotalMileageKM(sql.NullFloat64{}); got != nil {
|
||||
t.Fatalf("invalid mileage should remain nil, got %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -75,6 +75,13 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if cfg.HydrogenCapacitySyncEnabled {
|
||||
syncHydrogenCapacities(ctx, logger, registry, db, writer, cfg)
|
||||
} else if loaded, err := writer.ReloadHydrogenTankCapacities(ctx); err != nil {
|
||||
logger.Warn("hydrogen tank capacity cache load failed", "error", err)
|
||||
} else {
|
||||
registry.SetGauge("vehicle_stat_hydrogen_capacity_cache_entries", nil, float64(loaded))
|
||||
}
|
||||
if cfg.NormalizePlatformSourcesOnStart {
|
||||
statDate := time.Now().In(cfg.Location).Format("2006-01-02")
|
||||
normalizeCtx, cancel := context.WithTimeout(ctx, cfg.NormalizePlatformSourcesTimeout)
|
||||
@@ -108,6 +115,13 @@ func main() {
|
||||
|
||||
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "baseline_hit_ttl_seconds", cfg.BaselineHitTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds(), "quarantine_dir", cfg.QuarantineDir)
|
||||
var background sync.WaitGroup
|
||||
if cfg.HydrogenCapacitySyncEnabled && cfg.HydrogenCapacitySyncInterval > 0 {
|
||||
background.Add(1)
|
||||
go func() {
|
||||
defer background.Done()
|
||||
runHydrogenCapacitySync(ctx, logger, registry, db, writer, cfg)
|
||||
}()
|
||||
}
|
||||
if cfg.ProjectInterval > 0 {
|
||||
background.Add(1)
|
||||
go func() {
|
||||
@@ -132,6 +146,46 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
func runHydrogenCapacitySync(ctx context.Context, logger interface {
|
||||
Info(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, db *sql.DB, writer *stats.Writer, cfg config) {
|
||||
ticker := time.NewTicker(cfg.HydrogenCapacitySyncInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
syncHydrogenCapacities(ctx, logger, registry, db, writer, cfg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func syncHydrogenCapacities(ctx context.Context, logger interface {
|
||||
Info(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, db *sql.DB, writer *stats.Writer, cfg config) {
|
||||
syncCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cfg.HydrogenCapacitySyncTimeout)
|
||||
defer cancel()
|
||||
result, err := stats.SyncHydrogenTankCapacities(syncCtx, db, cfg.HydrogenCapacitySourceSchema)
|
||||
if err != nil {
|
||||
registry.IncCounter("vehicle_stat_hydrogen_capacity_sync_total", metrics.Labels{"status": "error"})
|
||||
logger.Warn("hydrogen tank capacity sync failed; retaining current cache", "source_schema", cfg.HydrogenCapacitySourceSchema, "error", err)
|
||||
return
|
||||
}
|
||||
loaded, err := writer.ReloadHydrogenTankCapacities(syncCtx)
|
||||
if err != nil {
|
||||
registry.IncCounter("vehicle_stat_hydrogen_capacity_sync_total", metrics.Labels{"status": "cache_error"})
|
||||
logger.Warn("hydrogen tank capacity cache reload failed", "error", err)
|
||||
return
|
||||
}
|
||||
registry.IncCounter("vehicle_stat_hydrogen_capacity_sync_total", metrics.Labels{"status": "ok"})
|
||||
registry.SetGauge("vehicle_stat_hydrogen_capacity_cache_entries", nil, float64(loaded))
|
||||
metrics.RecordLastActivity(registry, "vehicle_stat_hydrogen_capacity_last_sync_unix_seconds", nil)
|
||||
logger.Info("hydrogen tank capacities synchronized", "source_schema", cfg.HydrogenCapacitySourceSchema, "read", result.Read, "written", result.Written, "deactivated", result.Deactivated, "cache_entries", loaded)
|
||||
}
|
||||
|
||||
type pendingProjectionFlusher interface {
|
||||
FlushPendingProjections(context.Context, time.Time) (stats.ProjectionFlushResult, error)
|
||||
}
|
||||
@@ -790,6 +844,23 @@ func recordStatSampleMetrics(registry *metrics.Registry, message kafka.Message,
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_same_mileage", result.SamplesSkippedSameMileage)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_missing_source", result.SamplesSkippedMissingSource)
|
||||
addStatSampleMetric(registry, message, protocol, "event_time_future_adjusted", result.SamplesAdjustedFutureEventTime)
|
||||
addHydrogenStreamMetric(registry, message, "found", result.HydrogenSamplesFound)
|
||||
addHydrogenStreamMetric(registry, message, "written", result.HydrogenSamplesWritten)
|
||||
addHydrogenStreamMetric(registry, message, "duplicate_or_late", result.HydrogenSamplesDuplicate)
|
||||
addHydrogenStreamMetric(registry, message, "invalid", result.HydrogenSamplesInvalid)
|
||||
addHydrogenStreamMetric(registry, message, "not_current_date", result.HydrogenSamplesNotCurrent)
|
||||
}
|
||||
|
||||
func addHydrogenStreamMetric(registry *metrics.Registry, message kafka.Message, status string, count int) {
|
||||
if registry == nil || count == 0 {
|
||||
return
|
||||
}
|
||||
registry.AddCounter("vehicle_stat_hydrogen_stream_total", metrics.Labels{
|
||||
"topic": message.Topic, "status": status,
|
||||
}, float64(count))
|
||||
if status == "written" {
|
||||
metrics.RecordLastActivity(registry, "vehicle_stat_last_hydrogen_stream_write_unix_seconds", metrics.Labels{"status": status})
|
||||
}
|
||||
}
|
||||
|
||||
func addStatSampleMetric(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, status string, count int) {
|
||||
@@ -950,6 +1021,10 @@ type config struct {
|
||||
ReconcileProjectionsOnStart bool
|
||||
ReconcileProjectionsTimeout time.Duration
|
||||
QuarantineDir string
|
||||
HydrogenCapacitySyncEnabled bool
|
||||
HydrogenCapacitySourceSchema string
|
||||
HydrogenCapacitySyncInterval time.Duration
|
||||
HydrogenCapacitySyncTimeout time.Duration
|
||||
}
|
||||
|
||||
func (c config) Validate() error {
|
||||
@@ -996,6 +1071,10 @@ func loadConfig() config {
|
||||
ReconcileProjectionsOnStart: env("STATS_RECONCILE_PROJECTIONS_ON_START", "true") != "false",
|
||||
ReconcileProjectionsTimeout: time.Duration(envInt("STATS_RECONCILE_PROJECTIONS_TIMEOUT_SECONDS", 30)) * time.Second,
|
||||
QuarantineDir: env("STATS_QUARANTINE_DIR", "/var/lib/lingniu-go-native/stat-writer-quarantine"),
|
||||
HydrogenCapacitySyncEnabled: env("HYDROGEN_CAPACITY_SYNC_ENABLED", "true") != "false",
|
||||
HydrogenCapacitySourceSchema: env("HYDROGEN_CAPACITY_SOURCE_SCHEMA", "ln_asset_management"),
|
||||
HydrogenCapacitySyncInterval: time.Duration(envInt("HYDROGEN_CAPACITY_SYNC_INTERVAL_SECONDS", 21600)) * time.Second,
|
||||
HydrogenCapacitySyncTimeout: time.Duration(envInt("HYDROGEN_CAPACITY_SYNC_TIMEOUT_SECONDS", 60)) * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,23 +57,27 @@ type rawFrameRow struct {
|
||||
}
|
||||
|
||||
type metricAgg struct {
|
||||
VIN string
|
||||
Date string
|
||||
Protocol envelope.Protocol
|
||||
FirstKM float64
|
||||
LatestKM float64
|
||||
Count int64
|
||||
SourceKey string
|
||||
Phone string
|
||||
DeviceID string
|
||||
SourceEndpoint string
|
||||
SourceCode string
|
||||
PlatformName string
|
||||
SourceKind string
|
||||
FirstEventTime time.Time
|
||||
LatestEventTime time.Time
|
||||
QualityStatus string
|
||||
QualityReason string
|
||||
VIN string
|
||||
Date string
|
||||
Protocol envelope.Protocol
|
||||
FirstKM float64
|
||||
LatestKM float64
|
||||
PureHydrogenMileageKM float64
|
||||
Count int64
|
||||
PureHydrogenSampleCount int64
|
||||
SourceKey string
|
||||
Phone string
|
||||
DeviceID string
|
||||
SourceEndpoint string
|
||||
SourceCode string
|
||||
PlatformName string
|
||||
SourceKind string
|
||||
FirstEventTime time.Time
|
||||
LatestEventTime time.Time
|
||||
LatestPureHydrogenActive bool
|
||||
LatestPureHydrogenModeKnown bool
|
||||
QualityStatus string
|
||||
QualityReason string
|
||||
}
|
||||
|
||||
type dailySourceLast struct {
|
||||
@@ -266,21 +270,24 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
|
||||
agg, ok := aggregates[key]
|
||||
if !ok {
|
||||
aggregates[key] = &metricAgg{
|
||||
VIN: sample.VIN,
|
||||
Date: sample.StatDate,
|
||||
Protocol: sample.Protocol,
|
||||
FirstKM: sample.TotalMileageKM,
|
||||
LatestKM: sample.TotalMileageKM,
|
||||
Count: 1,
|
||||
SourceKey: sample.SourceKey,
|
||||
Phone: sample.Phone,
|
||||
DeviceID: sample.DeviceID,
|
||||
SourceEndpoint: sample.SourceEndpoint,
|
||||
PlatformName: sample.PlatformName,
|
||||
FirstEventTime: sample.EventTime,
|
||||
LatestEventTime: sample.EventTime,
|
||||
QualityStatus: stats.QualityOK,
|
||||
QualityReason: "current_day_first_sample",
|
||||
VIN: sample.VIN,
|
||||
Date: sample.StatDate,
|
||||
Protocol: sample.Protocol,
|
||||
FirstKM: sample.TotalMileageKM,
|
||||
LatestKM: sample.TotalMileageKM,
|
||||
Count: 1,
|
||||
PureHydrogenSampleCount: activeModeSampleCount(sample),
|
||||
SourceKey: sample.SourceKey,
|
||||
Phone: sample.Phone,
|
||||
DeviceID: sample.DeviceID,
|
||||
SourceEndpoint: sample.SourceEndpoint,
|
||||
PlatformName: sample.PlatformName,
|
||||
FirstEventTime: sample.EventTime,
|
||||
LatestEventTime: sample.EventTime,
|
||||
LatestPureHydrogenActive: sample.PureHydrogenActive,
|
||||
LatestPureHydrogenModeKnown: sample.PureHydrogenModeKnown,
|
||||
QualityStatus: stats.QualityOK,
|
||||
QualityReason: "current_day_first_sample",
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -289,8 +296,18 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
|
||||
agg.FirstEventTime = sample.EventTime
|
||||
}
|
||||
if sample.EventTime.After(agg.LatestEventTime) {
|
||||
agg.PureHydrogenMileageKM += stats.PureHydrogenMileageDelta(
|
||||
agg.LatestKM,
|
||||
sample.TotalMileageKM,
|
||||
agg.LatestPureHydrogenActive,
|
||||
agg.LatestPureHydrogenModeKnown,
|
||||
sample.PureHydrogenActive,
|
||||
sample.PureHydrogenModeKnown,
|
||||
)
|
||||
agg.LatestKM = sample.TotalMileageKM
|
||||
agg.LatestEventTime = sample.EventTime
|
||||
agg.LatestPureHydrogenActive = sample.PureHydrogenActive
|
||||
agg.LatestPureHydrogenModeKnown = sample.PureHydrogenModeKnown
|
||||
agg.SourceEndpoint = sample.SourceEndpoint
|
||||
if strings.TrimSpace(sample.Phone) != "" {
|
||||
agg.Phone = sample.Phone
|
||||
@@ -303,9 +320,17 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample)
|
||||
}
|
||||
}
|
||||
agg.Count++
|
||||
agg.PureHydrogenSampleCount += activeModeSampleCount(sample)
|
||||
}
|
||||
}
|
||||
|
||||
func activeModeSampleCount(sample stats.MetricSample) int64 {
|
||||
if sample.PureHydrogenModeKnown && sample.PureHydrogenActive {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg, batchSize int) (int64, error) {
|
||||
if len(aggregates) == 0 {
|
||||
return 0, nil
|
||||
@@ -338,23 +363,27 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met
|
||||
}
|
||||
dailyKM := stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM)
|
||||
candidate := stats.SourceMileageSample{
|
||||
VIN: agg.VIN,
|
||||
StatDate: agg.Date,
|
||||
Protocol: agg.Protocol,
|
||||
SourceKey: agg.SourceKey,
|
||||
SourceIP: identity.SourceIP,
|
||||
SourceEndpoint: agg.SourceEndpoint,
|
||||
Phone: agg.Phone,
|
||||
DeviceID: agg.DeviceID,
|
||||
PlatformName: agg.PlatformName,
|
||||
FirstTotalKM: agg.FirstKM,
|
||||
LatestTotalKM: agg.LatestKM,
|
||||
DailyKM: dailyKM,
|
||||
SampleCount: agg.Count,
|
||||
FirstEventTime: agg.FirstEventTime,
|
||||
LatestEventTime: agg.LatestEventTime,
|
||||
QualityStatus: agg.QualityStatus,
|
||||
QualityReason: agg.QualityReason,
|
||||
VIN: agg.VIN,
|
||||
StatDate: agg.Date,
|
||||
Protocol: agg.Protocol,
|
||||
SourceKey: agg.SourceKey,
|
||||
SourceIP: identity.SourceIP,
|
||||
SourceEndpoint: agg.SourceEndpoint,
|
||||
Phone: agg.Phone,
|
||||
DeviceID: agg.DeviceID,
|
||||
PlatformName: agg.PlatformName,
|
||||
FirstTotalKM: agg.FirstKM,
|
||||
LatestTotalKM: agg.LatestKM,
|
||||
DailyKM: dailyKM,
|
||||
PureHydrogenMileageKM: agg.PureHydrogenMileageKM,
|
||||
SampleCount: agg.Count,
|
||||
PureHydrogenSampleCount: agg.PureHydrogenSampleCount,
|
||||
FirstEventTime: agg.FirstEventTime,
|
||||
LatestEventTime: agg.LatestEventTime,
|
||||
LatestPureHydrogenActive: agg.LatestPureHydrogenActive,
|
||||
LatestPureHydrogenModeKnown: agg.LatestPureHydrogenModeKnown,
|
||||
QualityStatus: agg.QualityStatus,
|
||||
QualityReason: agg.QualityReason,
|
||||
}
|
||||
if candidate.QualityStatus == "" {
|
||||
candidate.QualityStatus = stats.QualityOK
|
||||
@@ -1266,9 +1295,49 @@ func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[str
|
||||
}
|
||||
fields[key] = value
|
||||
}
|
||||
promoteFuelCellWorkModeForStats(protocol, fields, text)
|
||||
return fields
|
||||
}
|
||||
|
||||
func promoteFuelCellWorkModeForStats(protocol envelope.Protocol, fields map[string]any, text string) {
|
||||
if fields == nil {
|
||||
return
|
||||
}
|
||||
if _, exists := fields[envelope.FieldFuelCellWorkMode]; exists {
|
||||
return
|
||||
}
|
||||
var candidates []string
|
||||
switch protocol {
|
||||
case envelope.ProtocolGB32960:
|
||||
candidates = []string{
|
||||
"gb32960.gd_fc_stack.engine_work_state",
|
||||
"engine_work_state",
|
||||
}
|
||||
case envelope.ProtocolYutongMQTT:
|
||||
candidates = []string{
|
||||
"yutong_mqtt.data.triangle_state",
|
||||
"yutong_mqtt.root.data.triangle_state",
|
||||
"TRIANGLE_STATE",
|
||||
"triangle_state",
|
||||
"triangleState",
|
||||
}
|
||||
default:
|
||||
return
|
||||
}
|
||||
for _, key := range candidates {
|
||||
if value, exists := fields[key]; exists {
|
||||
fields[envelope.FieldFuelCellWorkMode] = value
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, key := range candidates {
|
||||
if value, ok := extractJSONStringField(text, key); ok {
|
||||
fields[envelope.FieldFuelCellWorkMode] = value
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mileageKeys(protocol envelope.Protocol) []string {
|
||||
switch protocol {
|
||||
case envelope.ProtocolGB32960:
|
||||
|
||||
@@ -763,7 +763,8 @@ func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
|
||||
fields := fieldsForStats(envelope.ProtocolYutongMQTT, "LMRKH9AC6R1004108", `{
|
||||
"data": {
|
||||
"TOTAL_MILEAGE": 65423000,
|
||||
"METER_SPEED": 12.3
|
||||
"METER_SPEED": 12.3,
|
||||
"TRIANGLE_STATE": 11
|
||||
},
|
||||
"root": {
|
||||
"device": "LMRKH9AC6R1004108"
|
||||
@@ -773,6 +774,9 @@ func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
|
||||
if got := fields["yutong_mqtt.data.total_mileage"]; got == nil {
|
||||
t.Fatalf("fields missing raw yutong total mileage: %#v", fields)
|
||||
}
|
||||
if got := fields[envelope.FieldFuelCellWorkMode]; got == nil {
|
||||
t.Fatalf("fields missing raw yutong fuel-cell work mode: %#v", fields)
|
||||
}
|
||||
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
@@ -791,6 +795,32 @@ func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) {
|
||||
if samples[0].TotalMileageKM != 65423 {
|
||||
t.Fatalf("total mileage km = %v, want 65423", samples[0].TotalMileageKM)
|
||||
}
|
||||
if !samples[0].PureHydrogenModeKnown || !samples[0].PureHydrogenActive {
|
||||
t.Fatalf("pure-hydrogen mode = known:%v active:%v, want known active", samples[0].PureHydrogenModeKnown, samples[0].PureHydrogenActive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldsForStatsPromotesGB32960FuelCellEngineWorkState(t *testing.T) {
|
||||
fields := fieldsForStats(envelope.ProtocolGB32960, "LTEST000000000001", `{
|
||||
"data_units": [{
|
||||
"type": "0x30",
|
||||
"name": "gd_fc_stack",
|
||||
"value": {
|
||||
"summaries": [{"engine_work_state": 2}]
|
||||
}
|
||||
}]
|
||||
}`)
|
||||
|
||||
if got := fields[envelope.FieldFuelCellWorkMode]; got == nil {
|
||||
t.Fatalf("fields missing GB32960 fuel-cell work mode: %#v", fields)
|
||||
}
|
||||
active, known := stats.PureHydrogenModeFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Fields: fields,
|
||||
})
|
||||
if !known || !active {
|
||||
t.Fatalf("pure-hydrogen mode = known:%v active:%v, want known active", known, active)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) {
|
||||
@@ -863,9 +893,13 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T)
|
||||
float64(4100),
|
||||
float64(4101),
|
||||
float64(1),
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
projNow,
|
||||
projNow,
|
||||
false,
|
||||
false,
|
||||
stats.QualityInvalidDelta,
|
||||
"outside_daily_range",
|
||||
).
|
||||
@@ -994,6 +1028,51 @@ func TestAddSamplesUsesCurrentDayFirstSampleBaseline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddSamplesBackfillsOnlyContinuousPureHydrogenIntervals(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
base := stats.MetricSample{
|
||||
VIN: "LTEST000000000001",
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
StatDate: "2026-07-08",
|
||||
SourceKey: "GB32960|127.0.0.1",
|
||||
PureHydrogenModeKnown: true,
|
||||
}
|
||||
samples := []stats.MetricSample{
|
||||
base,
|
||||
base,
|
||||
base,
|
||||
base,
|
||||
}
|
||||
samples[0].TotalMileageKM = 100
|
||||
samples[0].EventTime = time.Date(2026, 7, 8, 8, 0, 0, 0, loc)
|
||||
samples[1].TotalMileageKM = 102
|
||||
samples[1].EventTime = time.Date(2026, 7, 8, 9, 0, 0, 0, loc)
|
||||
samples[1].PureHydrogenActive = true
|
||||
samples[2].TotalMileageKM = 107
|
||||
samples[2].EventTime = time.Date(2026, 7, 8, 10, 0, 0, 0, loc)
|
||||
samples[2].PureHydrogenActive = true
|
||||
samples[3].TotalMileageKM = 110
|
||||
samples[3].EventTime = time.Date(2026, 7, 8, 11, 0, 0, 0, loc)
|
||||
|
||||
aggregates := map[string]*metricAgg{}
|
||||
addSamples(aggregates, samples)
|
||||
|
||||
if len(aggregates) != 1 {
|
||||
t.Fatalf("aggregate count = %d, want 1", len(aggregates))
|
||||
}
|
||||
for _, agg := range aggregates {
|
||||
if agg.PureHydrogenMileageKM != 5 {
|
||||
t.Fatalf("pure-hydrogen mileage = %v, want 5", agg.PureHydrogenMileageKM)
|
||||
}
|
||||
if agg.PureHydrogenSampleCount != 2 {
|
||||
t.Fatalf("pure-hydrogen sample count = %d, want 2", agg.PureHydrogenSampleCount)
|
||||
}
|
||||
if !agg.LatestPureHydrogenModeKnown || agg.LatestPureHydrogenActive {
|
||||
t.Fatalf("latest mode = known:%v active:%v, want known inactive", agg.LatestPureHydrogenModeKnown, agg.LatestPureHydrogenActive)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) {
|
||||
t.Setenv("BACKFILL_METHOD", "")
|
||||
t.Setenv("BACKFILL_DAYS_BACK", "")
|
||||
|
||||
@@ -33,11 +33,12 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
FieldSpeedKMH = "speed_kmh"
|
||||
FieldTotalMileageKM = "total_mileage_km"
|
||||
FieldLongitude = "longitude"
|
||||
FieldLatitude = "latitude"
|
||||
FieldSOCPercent = "soc_percent"
|
||||
FieldSpeedKMH = "speed_kmh"
|
||||
FieldTotalMileageKM = "total_mileage_km"
|
||||
FieldLongitude = "longitude"
|
||||
FieldLatitude = "latitude"
|
||||
FieldSOCPercent = "soc_percent"
|
||||
FieldFuelCellWorkMode = "fuel_cell_work_mode"
|
||||
)
|
||||
|
||||
type FrameEnvelope struct {
|
||||
|
||||
@@ -18,7 +18,7 @@ const (
|
||||
|
||||
type Encoder struct{}
|
||||
|
||||
func (Encoder) DataFrame(command byte, vin string, at time.Time, record Record) ([]byte, error) {
|
||||
func (e Encoder) DataFrame(command byte, vin string, at time.Time, record Record) ([]byte, error) {
|
||||
if command != CommandRealtime && command != CommandReissue {
|
||||
return nil, fmt.Errorf("unsupported data command 0x%02X", command)
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestDataFrameMapsFeichiFields(t *testing.T) {
|
||||
"2607": "1", "2608": "3", "2609": "48",
|
||||
"2610": "1", "2611": "9", "2612": "37",
|
||||
"2003": "1:3.900_3.901_3.902",
|
||||
"2103": "1:35_36_37",
|
||||
"2103": "1:35_36_37", "2115": "44", "2119": "6.0", "100016": "20.8",
|
||||
}
|
||||
frame, err := (Encoder{}).DataFrame(CommandRealtime, "LTEST32960VIN0001", at, record)
|
||||
if err != nil {
|
||||
@@ -49,6 +49,9 @@ func TestDataFrameMapsFeichiFields(t *testing.T) {
|
||||
if got := env.Fields["longitude"]; got != 113.2644 {
|
||||
t.Fatalf("longitude = %#v", got)
|
||||
}
|
||||
if _, exists := env.Fields["gd_fc_vehicle_hydrogen_mass_kg"]; exists {
|
||||
t.Fatal("bridge must not derive hydrogen mass from platform percentage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataFrameSplitsMoreThan255CellVoltages(t *testing.T) {
|
||||
|
||||
@@ -250,6 +250,7 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
|
||||
unit := parseGdFCStackData(body[cursor : cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x30", "name": "gd_fc_stack", "value": unit})
|
||||
if summaries, ok := unit["summaries"].([]map[string]any); ok && len(summaries) > 0 {
|
||||
fields[envelope.FieldFuelCellWorkMode] = summaries[0]["engine_work_state"]
|
||||
fields["gd_fc_stack_hydrogen_inlet_pressure_kpa"] = summaries[0]["hydrogen_inlet_pressure_kpa"]
|
||||
fields["gd_fc_stack_water_outlet_temp_c"] = summaries[0]["stack_water_outlet_temp_c"]
|
||||
fields["gd_fc_stack_cell_count"] = summaries[0]["cell_count"]
|
||||
|
||||
@@ -197,6 +197,7 @@ func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testin
|
||||
t.Fatalf("vin = %q", env.VIN)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldTotalMileageKM, 53490.9)
|
||||
assertIntField(t, env, envelope.FieldFuelCellWorkMode, 2)
|
||||
assertFloatField(t, env, "fuel_cell_hydrogen_consumption_kg_per_100km", 1.8)
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 120.800326)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 31.634907)
|
||||
|
||||
@@ -115,6 +115,9 @@ func fieldsFromData(data map[string]any) map[string]any {
|
||||
if gear, ok := firstFloat(data, "ON_GEAR", "gear"); ok {
|
||||
fields["gear"] = gear
|
||||
}
|
||||
if workMode, ok := firstFloat(data, "TRIANGLE_STATE", "triangleState", "triangle_state"); ok {
|
||||
fields[envelope.FieldFuelCellWorkMode] = workMode
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@ func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
|
||||
"BATTERY_CAPACITY_SOC": 70,
|
||||
"LONGITUDE": 116397128,
|
||||
"LATITUDE": 39916527,
|
||||
"GPSDirection": 88
|
||||
"GPSDirection": 88,
|
||||
"TRIANGLE_STATE": 11
|
||||
}
|
||||
}`)
|
||||
|
||||
@@ -41,6 +42,7 @@ func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) {
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 116.397128)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 39.916527)
|
||||
assertFloatField(t, env, "direction_deg", 88)
|
||||
assertFloatField(t, env, envelope.FieldFuelCellWorkMode, 11)
|
||||
if env.EventTimeMS == 1782745114999 {
|
||||
t.Fatal("event time should come from device time")
|
||||
}
|
||||
|
||||
@@ -468,49 +468,52 @@ func realtimeOpenAPISpec() map[string]any {
|
||||
"DailyMetricRow": map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"vin": stringSchema("LA9GG64L7PBAF4001"),
|
||||
"stat_date": stringSchema("2026-07-08"),
|
||||
"protocol": stringSchema("JT808"),
|
||||
"source_id": integerSchema(3),
|
||||
"source_ip": stringSchema("115.231.168.135"),
|
||||
"latest_source_endpoint": stringSchema("115.231.168.135:41561"),
|
||||
"platform_name": stringSchema("G7 平台"),
|
||||
"source_code": stringSchema("G7S"),
|
||||
"source_kind": stringSchema("PLATFORM"),
|
||||
"daily_mileage_km": numberSchema(23.1),
|
||||
"latest_total_mileage_km": numberSchema(4123.9),
|
||||
"updated_at": stringSchema("2026-07-08 13:30:57"),
|
||||
"vin": stringSchema("LA9GG64L7PBAF4001"),
|
||||
"stat_date": stringSchema("2026-07-08"),
|
||||
"protocol": stringSchema("JT808"),
|
||||
"source_id": integerSchema(3),
|
||||
"source_ip": stringSchema("115.231.168.135"),
|
||||
"latest_source_endpoint": stringSchema("115.231.168.135:41561"),
|
||||
"platform_name": stringSchema("G7 平台"),
|
||||
"source_code": stringSchema("G7S"),
|
||||
"source_kind": stringSchema("PLATFORM"),
|
||||
"daily_mileage_km": numberSchema(23.1),
|
||||
"pure_hydrogen_mileage_km": numberSchema(18.6),
|
||||
"latest_total_mileage_km": numberSchema(4123.9),
|
||||
"updated_at": stringSchema("2026-07-08 13:30:57"),
|
||||
},
|
||||
},
|
||||
"DailyMetricSourceRow": map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"vin": stringSchema("LA9GG64L7PBAF4001"),
|
||||
"stat_date": stringSchema("2026-07-08"),
|
||||
"protocol": stringSchema("JT808"),
|
||||
"source_key": stringSchema("JT808:13307795425@115.231.168.135"),
|
||||
"source_ip": stringSchema("115.231.168.135"),
|
||||
"source_endpoint": stringSchema("115.231.168.135:41561"),
|
||||
"phone": stringSchema("13307795425"),
|
||||
"platform_name": stringSchema("信达"),
|
||||
"source_id": integerSchema(5),
|
||||
"source_code": stringSchema("xinda"),
|
||||
"source_kind": stringSchema("PLATFORM"),
|
||||
"source_enabled": map[string]any{"type": "boolean", "example": true},
|
||||
"trust_priority": integerSchema(10),
|
||||
"first_total_mileage_km": numberSchema(4100.8),
|
||||
"latest_total_mileage_km": numberSchema(4123.9),
|
||||
"daily_mileage_km": numberSchema(23.1),
|
||||
"sample_count": integerSchema(128),
|
||||
"first_event_time": stringSchema("2026-07-08 00:01:00"),
|
||||
"latest_event_time": stringSchema("2026-07-08 23:59:00"),
|
||||
"quality_status": stringSchema("OK"),
|
||||
"quality_reason": stringSchema("historical_source_baseline"),
|
||||
"is_selected": map[string]any{"type": "boolean", "example": true},
|
||||
"selection_status": stringSchema("selected"),
|
||||
"selection_reason": stringSchema("selected_current_projection"),
|
||||
"selection_action": stringSchema("当前来源已被投影到最终日里程表"),
|
||||
"updated_at": stringSchema("2026-07-08 23:59:10"),
|
||||
"vin": stringSchema("LA9GG64L7PBAF4001"),
|
||||
"stat_date": stringSchema("2026-07-08"),
|
||||
"protocol": stringSchema("JT808"),
|
||||
"source_key": stringSchema("JT808:13307795425@115.231.168.135"),
|
||||
"source_ip": stringSchema("115.231.168.135"),
|
||||
"source_endpoint": stringSchema("115.231.168.135:41561"),
|
||||
"phone": stringSchema("13307795425"),
|
||||
"platform_name": stringSchema("信达"),
|
||||
"source_id": integerSchema(5),
|
||||
"source_code": stringSchema("xinda"),
|
||||
"source_kind": stringSchema("PLATFORM"),
|
||||
"source_enabled": map[string]any{"type": "boolean", "example": true},
|
||||
"trust_priority": integerSchema(10),
|
||||
"first_total_mileage_km": numberSchema(4100.8),
|
||||
"latest_total_mileage_km": numberSchema(4123.9),
|
||||
"daily_mileage_km": numberSchema(23.1),
|
||||
"pure_hydrogen_mileage_km": numberSchema(18.6),
|
||||
"pure_hydrogen_sample_count": integerSchema(42),
|
||||
"sample_count": integerSchema(128),
|
||||
"first_event_time": stringSchema("2026-07-08 00:01:00"),
|
||||
"latest_event_time": stringSchema("2026-07-08 23:59:00"),
|
||||
"quality_status": stringSchema("OK"),
|
||||
"quality_reason": stringSchema("historical_source_baseline"),
|
||||
"is_selected": map[string]any{"type": "boolean", "example": true},
|
||||
"selection_status": stringSchema("selected"),
|
||||
"selection_reason": stringSchema("selected_current_projection"),
|
||||
"selection_action": stringSchema("当前来源已被投影到最终日里程表"),
|
||||
"updated_at": stringSchema("2026-07-08 23:59:10"),
|
||||
},
|
||||
},
|
||||
"DailyMetricSourceQualityRow": map[string]any{
|
||||
|
||||
@@ -53,20 +53,23 @@ type Writer struct {
|
||||
projectionKeysByPrefix map[string]map[string]struct{}
|
||||
baselineKeysByPrefix map[string]map[string]struct{}
|
||||
lastGPSAccumulation map[string]time.Time
|
||||
hydrogenTankCapacities map[string]float64
|
||||
gpsAccumulationInterval time.Duration
|
||||
}
|
||||
|
||||
type MetricSample struct {
|
||||
VIN string
|
||||
Protocol envelope.Protocol
|
||||
StatDate string
|
||||
TotalMileageKM float64
|
||||
EventTime time.Time
|
||||
SourceKey string
|
||||
Phone string
|
||||
DeviceID string
|
||||
SourceEndpoint string
|
||||
PlatformName string
|
||||
VIN string
|
||||
Protocol envelope.Protocol
|
||||
StatDate string
|
||||
TotalMileageKM float64
|
||||
EventTime time.Time
|
||||
SourceKey string
|
||||
Phone string
|
||||
DeviceID string
|
||||
SourceEndpoint string
|
||||
PlatformName string
|
||||
PureHydrogenActive bool
|
||||
PureHydrogenModeKnown bool
|
||||
}
|
||||
|
||||
type AppendResult struct {
|
||||
@@ -91,6 +94,11 @@ type AppendResult struct {
|
||||
ProjectionsAttempted int
|
||||
ProjectionsWritten int
|
||||
ProjectionsSkippedThrottled int
|
||||
HydrogenSamplesFound int
|
||||
HydrogenSamplesWritten int
|
||||
HydrogenSamplesDuplicate int
|
||||
HydrogenSamplesInvalid int
|
||||
HydrogenSamplesNotCurrent int
|
||||
}
|
||||
|
||||
type ProjectionFlushResult struct {
|
||||
@@ -161,6 +169,7 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
|
||||
projectionKeysByPrefix: map[string]map[string]struct{}{},
|
||||
baselineKeysByPrefix: map[string]map[string]struct{}{},
|
||||
lastGPSAccumulation: map[string]time.Time{},
|
||||
hydrogenTankCapacities: map[string]float64{},
|
||||
gpsAccumulationInterval: defaultGPSAccumulationInterval,
|
||||
}
|
||||
if query, ok := exec.(Queryer); ok {
|
||||
@@ -169,6 +178,25 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
|
||||
return writer
|
||||
}
|
||||
|
||||
func (w *Writer) ReloadHydrogenTankCapacities(ctx context.Context) (int, error) {
|
||||
capacities, err := LoadHydrogenTankCapacities(ctx, w.query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.hydrogenTankCapacities = capacities
|
||||
w.mu.Unlock()
|
||||
return len(capacities), nil
|
||||
}
|
||||
|
||||
func (w *Writer) HydrogenTankCapacityLiters(vin string) (float64, bool) {
|
||||
vin = strings.ToUpper(strings.TrimSpace(vin))
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
capacity, ok := w.hydrogenTankCapacities[vin]
|
||||
return capacity, ok
|
||||
}
|
||||
|
||||
func (w *Writer) SetSourceTouchInterval(interval time.Duration) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
@@ -273,6 +301,8 @@ func (w *Writer) EnsureSchema(ctx context.Context) error {
|
||||
GPSMileageStateTableSQL,
|
||||
DailyMileageSourceTableSQL,
|
||||
DailyMileageTableSQL,
|
||||
HydrogenTankCapacityTableSQL,
|
||||
HydrogenStreamStateTableSQL,
|
||||
} {
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
@@ -300,6 +330,16 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
|
||||
seenAt := w.sourceSeenAt(env)
|
||||
w.maybeCleanupCaches(seenAt)
|
||||
identity, hasSource := NewSourceIdentityFromEnvelope(env)
|
||||
capacityLiters, _ := w.HydrogenTankCapacityLiters(env.VIN)
|
||||
hydrogen, err := AppendHydrogenStream(ctx, w.exec, env, w.loc, time.Now(), capacityLiters)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.HydrogenSamplesFound = hydrogen.Found
|
||||
result.HydrogenSamplesWritten = hydrogen.Written
|
||||
result.HydrogenSamplesDuplicate = hydrogen.Duplicate
|
||||
result.HydrogenSamplesInvalid = hydrogen.Invalid
|
||||
result.HydrogenSamplesNotCurrent = hydrogen.NotCurrent
|
||||
var sourceResult AppendResult
|
||||
if hasSource && ShouldManageDataSource(identity) {
|
||||
if w.shouldTouchSource(identity, seenAt) {
|
||||
@@ -316,6 +356,11 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
|
||||
sourceResult.SourceTouchesSkippedUnmanaged = 1
|
||||
}
|
||||
samples, extractionResult, err := samplesFromEnvelopeWithResult(env, w.loc)
|
||||
extractionResult.HydrogenSamplesFound = result.HydrogenSamplesFound
|
||||
extractionResult.HydrogenSamplesWritten = result.HydrogenSamplesWritten
|
||||
extractionResult.HydrogenSamplesDuplicate = result.HydrogenSamplesDuplicate
|
||||
extractionResult.HydrogenSamplesInvalid = result.HydrogenSamplesInvalid
|
||||
extractionResult.HydrogenSamplesNotCurrent = result.HydrogenSamplesNotCurrent
|
||||
result = extractionResult
|
||||
result.SourceTouchesAttempted += sourceResult.SourceTouchesAttempted
|
||||
result.SourceTouchesWritten += sourceResult.SourceTouchesWritten
|
||||
@@ -1191,18 +1236,21 @@ func samplesFromEnvelopeWithResult(env envelope.FrameEnvelope, loc *time.Locatio
|
||||
result.SamplesAdjustedFutureEventTime = 1
|
||||
}
|
||||
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
|
||||
pureHydrogenActive, pureHydrogenModeKnown := PureHydrogenModeFromEnvelope(env)
|
||||
result.SamplesFound = 1
|
||||
return []MetricSample{{
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
StatDate: statDate,
|
||||
TotalMileageKM: totalMileage,
|
||||
EventTime: time.UnixMilli(eventMS).In(loc),
|
||||
SourceKey: sourceKey(env),
|
||||
Phone: strings.TrimSpace(env.Phone),
|
||||
DeviceID: strings.TrimSpace(env.DeviceID),
|
||||
SourceEndpoint: strings.TrimSpace(env.SourceEndpoint),
|
||||
PlatformName: strings.TrimSpace(env.PlatformName),
|
||||
VIN: vin,
|
||||
Protocol: env.Protocol,
|
||||
StatDate: statDate,
|
||||
TotalMileageKM: totalMileage,
|
||||
EventTime: time.UnixMilli(eventMS).In(loc),
|
||||
SourceKey: sourceKey(env),
|
||||
Phone: strings.TrimSpace(env.Phone),
|
||||
DeviceID: strings.TrimSpace(env.DeviceID),
|
||||
SourceEndpoint: strings.TrimSpace(env.SourceEndpoint),
|
||||
PlatformName: strings.TrimSpace(env.PlatformName),
|
||||
PureHydrogenActive: pureHydrogenActive,
|
||||
PureHydrogenModeKnown: pureHydrogenModeKnown,
|
||||
}}, result, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -118,6 +118,40 @@ func TestSamplesFromEnvelopeMapsProtocolMileageFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeCarriesPureHydrogenWorkMode(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
tests := []struct {
|
||||
name string
|
||||
protocol envelope.Protocol
|
||||
mode int
|
||||
}{
|
||||
{name: "gb32960 fuel cell engine running", protocol: envelope.ProtocolGB32960, mode: 2},
|
||||
{name: "yutong fuel cell driving", protocol: envelope.ProtocolYutongMQTT, mode: 11},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
fields := map[string]any{envelope.FieldFuelCellWorkMode: test.mode}
|
||||
if test.protocol == envelope.ProtocolGB32960 {
|
||||
fields["gb32960.vehicle.total_mileage_km"] = 12000.5
|
||||
} else {
|
||||
fields["yutong_mqtt.data.total_mileage"] = 12000500
|
||||
}
|
||||
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: test.protocol,
|
||||
VIN: "LNBVIN00000000001",
|
||||
EventTimeMS: time.Date(2026, 7, 23, 8, 30, 0, 0, loc).UnixMilli(),
|
||||
Fields: fields,
|
||||
}, loc)
|
||||
if err != nil || len(samples) != 1 {
|
||||
t.Fatalf("SamplesFromEnvelope() samples=%d error=%v", len(samples), err)
|
||||
}
|
||||
if !samples[0].PureHydrogenModeKnown || !samples[0].PureHydrogenActive {
|
||||
t.Fatalf("pure hydrogen evidence missing: %#v", samples[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSamplesFromEnvelopeFallsBackToReceivedTimeWhenEventTimeIsFuture(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
received := time.Date(2026, 7, 8, 23, 59, 0, 0, loc)
|
||||
@@ -1035,9 +1069,13 @@ func TestWriterAppendUsesCurrentSampleWhenNoHistoricalBaseline(t *testing.T) {
|
||||
float64(4123.9),
|
||||
float64(4123.9),
|
||||
float64(0),
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
eventTime,
|
||||
eventTime,
|
||||
false,
|
||||
false,
|
||||
QualityOK,
|
||||
QualityReasonCurrentDayFirst,
|
||||
).
|
||||
@@ -1115,9 +1153,13 @@ func TestWriterCachesMissingBaselineAfterSuccessfulFirstSample(t *testing.T) {
|
||||
float64(4123.9),
|
||||
float64(4123.9),
|
||||
float64(0),
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
firstTime,
|
||||
firstTime,
|
||||
false,
|
||||
false,
|
||||
QualityOK,
|
||||
QualityReasonCurrentDayFirst,
|
||||
).
|
||||
@@ -1161,9 +1203,13 @@ func TestWriterCachesMissingBaselineAfterSuccessfulFirstSample(t *testing.T) {
|
||||
float64(4123.9),
|
||||
float64(4124.4),
|
||||
approxFloat64{want: 0.5, tolerance: 0.000001},
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
firstTime,
|
||||
secondTime,
|
||||
false,
|
||||
false,
|
||||
QualityOK,
|
||||
QualityReasonCurrentDayFirst,
|
||||
).
|
||||
@@ -1356,9 +1402,13 @@ func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T
|
||||
float64(4100.8),
|
||||
float64(4123.9),
|
||||
approxFloat64{want: 23.1, tolerance: 0.000001},
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
previousTime,
|
||||
currentTime,
|
||||
false,
|
||||
false,
|
||||
QualityOK,
|
||||
QualityReasonHistorical,
|
||||
).
|
||||
@@ -1441,9 +1491,13 @@ func TestWriterAppendUsesOlderHistoricalSourceBaseline(t *testing.T) {
|
||||
float64(120672.0),
|
||||
float64(120788.0),
|
||||
approxFloat64{want: 116.0, tolerance: 0.000001},
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
historicalTime,
|
||||
currentTime,
|
||||
false,
|
||||
false,
|
||||
QualityOK,
|
||||
QualityReasonHistorical,
|
||||
).
|
||||
@@ -1776,9 +1830,13 @@ func TestWriterAppendMarksCandidateNoPreviousBaselineWhenPreviousBaselineMissing
|
||||
float64(4136.8),
|
||||
float64(4136.8),
|
||||
float64(0),
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
currentTime,
|
||||
currentTime,
|
||||
false,
|
||||
false,
|
||||
QualityOK,
|
||||
QualityReasonCurrentDayFirst,
|
||||
).
|
||||
@@ -1836,7 +1894,7 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
|
||||
t.Fatalf("Append() error = %v", err)
|
||||
}
|
||||
|
||||
schemaCalls := 4 + len(DailyMileageAlterSQL)
|
||||
schemaCalls := 6 + len(DailyMileageAlterSQL)
|
||||
if len(exec.calls) != schemaCalls+5 {
|
||||
t.Fatalf("exec calls = %d", len(exec.calls))
|
||||
}
|
||||
@@ -1845,6 +1903,8 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
|
||||
"CREATE TABLE IF NOT EXISTS vehicle_daily_gps_mileage_state",
|
||||
"CREATE TABLE IF NOT EXISTS vehicle_daily_mileage_source",
|
||||
"CREATE TABLE IF NOT EXISTS vehicle_daily_mileage",
|
||||
"CREATE TABLE IF NOT EXISTS vehicle_hydrogen_tank_capacity",
|
||||
"CREATE TABLE IF NOT EXISTS vehicle_open_hydrogen_stream_state",
|
||||
} {
|
||||
if !strings.Contains(exec.calls[i].query, want) {
|
||||
t.Fatalf("schema call %d = %s, want %s", i, exec.calls[i].query, want)
|
||||
@@ -2040,7 +2100,7 @@ func TestWriterCarriesPreviousYutongMileageIntoStationaryDay(t *testing.T) {
|
||||
if got := sourceWrite.args[11]; got != float64(0) {
|
||||
t.Fatalf("daily mileage = %#v, want explicit zero", got)
|
||||
}
|
||||
if got := sourceWrite.args[16]; got != QualityReasonStationaryCarry {
|
||||
if got := sourceWrite.args[20]; got != QualityReasonStationaryCarry {
|
||||
t.Fatalf("quality reason = %#v, want %q", got, QualityReasonStationaryCarry)
|
||||
}
|
||||
}
|
||||
@@ -2249,9 +2309,13 @@ func TestWriterRollsBackMileageTransactionWhenProjectionFails(t *testing.T) {
|
||||
float64(4123.9),
|
||||
float64(4123.9),
|
||||
float64(0),
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
eventTime,
|
||||
eventTime,
|
||||
false,
|
||||
false,
|
||||
QualityOK,
|
||||
QualityReasonCurrentDayFirst,
|
||||
).
|
||||
|
||||
149
go/vehicle-gateway/internal/stats/hydrogen_capacity.go
Normal file
149
go/vehicle-gateway/internal/stats/hydrogen_capacity.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const HydrogenTankCapacityTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_hydrogen_tank_capacity (
|
||||
vin VARCHAR(64) NOT NULL,
|
||||
source_vehicle_id BIGINT NOT NULL,
|
||||
source_model_id BIGINT NULL,
|
||||
plate_number VARCHAR(50) NOT NULL DEFAULT '',
|
||||
brand_name VARCHAR(500) NOT NULL DEFAULT '',
|
||||
model_name VARCHAR(500) NOT NULL DEFAULT '',
|
||||
tank_capacity_l DECIMAL(12,2) NOT NULL,
|
||||
active TINYINT(1) NOT NULL DEFAULT 1,
|
||||
source_updated_at DATETIME NULL,
|
||||
synced_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY (vin),
|
||||
KEY idx_hydrogen_capacity_model (source_model_id),
|
||||
KEY idx_hydrogen_capacity_active (active,vin)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`
|
||||
|
||||
type HydrogenCapacitySyncResult struct {
|
||||
Read int
|
||||
Written int
|
||||
Deactivated int64
|
||||
}
|
||||
|
||||
type hydrogenCapacityRow struct {
|
||||
VIN string
|
||||
SourceVehicleID int64
|
||||
SourceModelID sql.NullInt64
|
||||
Plate string
|
||||
Brand string
|
||||
Model string
|
||||
CapacityLiters float64
|
||||
SourceUpdatedAt sql.NullTime
|
||||
}
|
||||
|
||||
var mysqlIdentifier = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
|
||||
|
||||
func SyncHydrogenTankCapacities(ctx context.Context, db *sql.DB, sourceSchema string) (HydrogenCapacitySyncResult, error) {
|
||||
result := HydrogenCapacitySyncResult{}
|
||||
if db == nil {
|
||||
return result, fmt.Errorf("mysql database is required")
|
||||
}
|
||||
sourceSchema = strings.TrimSpace(sourceSchema)
|
||||
if !mysqlIdentifier.MatchString(sourceSchema) {
|
||||
return result, fmt.Errorf("invalid asset schema %q", sourceSchema)
|
||||
}
|
||||
query := fmt.Sprintf(`SELECT vi.id,vi.vehicle_model_id,UPPER(TRIM(vi.vin)),COALESCE(vi.plate_number,''),
|
||||
COALESCE(vm.brand,''),COALESCE(vm.model,''),vm.tank_capacity,
|
||||
CASE
|
||||
WHEN vi.update_time IS NULL THEN vm.update_time
|
||||
WHEN vm.update_time IS NULL THEN vi.update_time
|
||||
ELSE GREATEST(vi.update_time,vm.update_time)
|
||||
END
|
||||
FROM %s.vehicle_info vi
|
||||
JOIN %s.vehicle_model vm ON vm.id=vi.vehicle_model_id
|
||||
WHERE COALESCE(vi.del_flag,'0')='0' AND COALESCE(vm.del_flag,'0')='0'
|
||||
AND LENGTH(TRIM(vi.vin))=17 AND vm.tank_capacity>0`, sourceSchema, sourceSchema)
|
||||
rows, err := db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("read asset hydrogen capacities: %w", err)
|
||||
}
|
||||
byVIN := map[string]hydrogenCapacityRow{}
|
||||
for rows.Next() {
|
||||
var row hydrogenCapacityRow
|
||||
if err := rows.Scan(&row.SourceVehicleID, &row.SourceModelID, &row.VIN, &row.Plate, &row.Brand, &row.Model, &row.CapacityLiters, &row.SourceUpdatedAt); err != nil {
|
||||
rows.Close()
|
||||
return result, err
|
||||
}
|
||||
result.Read++
|
||||
if row.CapacityLiters <= 0 || row.CapacityLiters > 10000 {
|
||||
continue
|
||||
}
|
||||
if previous, exists := byVIN[row.VIN]; !exists || newerCapacityRow(row, previous) {
|
||||
byVIN[row.VIN] = row
|
||||
}
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
_, err = tx.ExecContext(ctx, `UPDATE vehicle_hydrogen_tank_capacity SET active=0 WHERE active=1`)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
const upsert = `INSERT INTO vehicle_hydrogen_tank_capacity(
|
||||
vin,source_vehicle_id,source_model_id,plate_number,brand_name,model_name,tank_capacity_l,active,source_updated_at,synced_at
|
||||
) VALUES(?,?,?,?,?,?,?,1,?,NOW(3))
|
||||
ON DUPLICATE KEY UPDATE source_vehicle_id=VALUES(source_vehicle_id),source_model_id=VALUES(source_model_id),
|
||||
plate_number=VALUES(plate_number),brand_name=VALUES(brand_name),model_name=VALUES(model_name),
|
||||
tank_capacity_l=VALUES(tank_capacity_l),active=1,source_updated_at=VALUES(source_updated_at),synced_at=NOW(3)`
|
||||
for _, row := range byVIN {
|
||||
if _, err := tx.ExecContext(ctx, upsert, row.VIN, row.SourceVehicleID, row.SourceModelID, row.Plate, row.Brand, row.Model, row.CapacityLiters, row.SourceUpdatedAt); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Written++
|
||||
}
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM vehicle_hydrogen_tank_capacity WHERE active=0`).Scan(&result.Deactivated); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func newerCapacityRow(left, right hydrogenCapacityRow) bool {
|
||||
if left.SourceUpdatedAt.Valid != right.SourceUpdatedAt.Valid {
|
||||
return left.SourceUpdatedAt.Valid
|
||||
}
|
||||
if left.SourceUpdatedAt.Valid && !left.SourceUpdatedAt.Time.Equal(right.SourceUpdatedAt.Time) {
|
||||
return left.SourceUpdatedAt.Time.After(right.SourceUpdatedAt.Time)
|
||||
}
|
||||
return left.SourceVehicleID > right.SourceVehicleID
|
||||
}
|
||||
|
||||
func LoadHydrogenTankCapacities(ctx context.Context, query Queryer) (map[string]float64, error) {
|
||||
if query == nil {
|
||||
return nil, fmt.Errorf("mysql queryer is required")
|
||||
}
|
||||
rows, err := query.QueryContext(ctx, `SELECT UPPER(TRIM(vin)),tank_capacity_l FROM vehicle_hydrogen_tank_capacity WHERE active=1 AND tank_capacity_l>0`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
capacities := map[string]float64{}
|
||||
for rows.Next() {
|
||||
var vin string
|
||||
var capacity float64
|
||||
if err := rows.Scan(&vin, &capacity); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(vin) == 17 && capacity > 0 && capacity <= 10000 {
|
||||
capacities[vin] = capacity
|
||||
}
|
||||
}
|
||||
return capacities, rows.Err()
|
||||
}
|
||||
72
go/vehicle-gateway/internal/stats/hydrogen_capacity_test.go
Normal file
72
go/vehicle-gateway/internal/stats/hydrogen_capacity_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
func TestSyncHydrogenTankCapacitiesCopiesAssetMasterData(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
updatedAt := time.Date(2026, 7, 21, 10, 0, 0, 0, time.Local)
|
||||
mock.ExpectQuery("FROM ln_asset_management\\.vehicle_info vi").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "vehicle_model_id", "vin", "plate", "brand", "model", "tank_capacity", "updated_at"}).
|
||||
AddRow(1001, 51, "LA9GG64L0NBAF4175", "浙F06618F", "feichi", "49吨牵引车头", 1400.0, updatedAt))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(regexp.QuoteMeta("UPDATE vehicle_hydrogen_tank_capacity SET active=0 WHERE active=1")).
|
||||
WillReturnResult(sqlmock.NewResult(0, 20))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO vehicle_hydrogen_tank_capacity(")).
|
||||
WithArgs("LA9GG64L0NBAF4175", int64(1001), sqlmock.AnyArg(), "浙F06618F", "feichi", "49吨牵引车头", 1400.0, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM vehicle_hydrogen_tank_capacity WHERE active=0")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0))
|
||||
mock.ExpectCommit()
|
||||
|
||||
result, err := SyncHydrogenTankCapacities(context.Background(), db, "ln_asset_management")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Read != 1 || result.Written != 1 || result.Deactivated != 0 {
|
||||
t.Fatalf("result=%+v", result)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncHydrogenTankCapacitiesRejectsUnsafeSchema(t *testing.T) {
|
||||
db, _, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := SyncHydrogenTankCapacities(context.Background(), db, "ln_asset_management;DROP"); err == nil {
|
||||
t.Fatal("unsafe schema accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadHydrogenTankCapacitiesBuildsVINMap(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT UPPER\\(TRIM\\(vin\\)\\),tank_capacity_l").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "capacity"}).
|
||||
AddRow("LA9GG64L0NBAF4175", 1400.0).
|
||||
AddRow("LB9A32A24R0LS1037", 380.0))
|
||||
capacities, err := LoadHydrogenTankCapacities(context.Background(), db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if capacities["LA9GG64L0NBAF4175"] != 1400 || capacities["LB9A32A24R0LS1037"] != 380 {
|
||||
t.Fatalf("capacities=%v", capacities)
|
||||
}
|
||||
}
|
||||
270
go/vehicle-gateway/internal/stats/hydrogen_stream.go
Normal file
270
go/vehicle-gateway/internal/stats/hydrogen_stream.go
Normal file
@@ -0,0 +1,270 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHydrogenNoiseKG = 0.05
|
||||
defaultHydrogenMaxDropKG = 20.0
|
||||
hydrogenMolarMassKGPerMol = 0.00201588
|
||||
universalGasConstant = 8.314472
|
||||
)
|
||||
|
||||
var hydrogenPressureFieldKeys = []string{
|
||||
"gb32960.fuel_cell.max_hydrogen_pressure_mpa",
|
||||
"fuel_cell_max_hydrogen_pressure_mpa",
|
||||
}
|
||||
|
||||
var hydrogenTemperatureFieldKeys = []string{
|
||||
"gb32960.fuel_cell.max_hydrogen_temperature_c",
|
||||
"fuel_cell_max_hydrogen_temperature_c",
|
||||
}
|
||||
|
||||
var hydrogenDensityA = [...]float64{0.05888460, -0.06136111, -0.002650473, 0.002731125, 0.001802374, -0.001150707, 0.00009588528, -0.0000001109040, 0.0000000001264403}
|
||||
var hydrogenDensityB = [...]float64{1.325, 1.87, 2.5, 2.8, 2.938, 3.14, 3.37, 3.75, 4.0}
|
||||
var hydrogenDensityC = [...]float64{1, 1, 2, 2, 2.42, 2.63, 3, 4, 5}
|
||||
|
||||
const HydrogenStreamStateTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_open_hydrogen_stream_state (
|
||||
vin VARCHAR(64) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
source_endpoint VARCHAR(128) NOT NULL DEFAULT '',
|
||||
first_mass_kg DECIMAL(18,3) NOT NULL,
|
||||
last_mass_kg DECIMAL(18,3) NOT NULL,
|
||||
cycle_min_mass_kg DECIMAL(18,3) NOT NULL,
|
||||
consumption_kg DECIMAL(18,3) NOT NULL DEFAULT 0,
|
||||
sample_count BIGINT UNSIGNED NOT NULL DEFAULT 1,
|
||||
refuel_count INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
abnormal_drop_count INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
tank_capacity_l DECIMAL(12,2) NOT NULL,
|
||||
first_pressure_mpa DECIMAL(10,3) NOT NULL,
|
||||
last_pressure_mpa DECIMAL(10,3) NOT NULL,
|
||||
first_temperature_c DECIMAL(10,3) NOT NULL,
|
||||
last_temperature_c DECIMAL(10,3) NOT NULL,
|
||||
calculation_method VARCHAR(32) NOT NULL DEFAULT 'PRESSURE_NIST',
|
||||
first_event_time DATETIME(3) NOT NULL,
|
||||
last_event_time DATETIME(3) NOT NULL,
|
||||
last_event_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||
quality_status VARCHAR(24) NOT NULL DEFAULT 'NO_DATA',
|
||||
quality_reason VARCHAR(255) NOT NULL DEFAULT '有效压力质量样本不足2条',
|
||||
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY (vin, stat_date, source_endpoint),
|
||||
KEY idx_hydrogen_stream_date_quality (stat_date, quality_status, vin)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`
|
||||
|
||||
type HydrogenStreamSample struct {
|
||||
VIN string
|
||||
Date string
|
||||
SourceEndpoint string
|
||||
EventID string
|
||||
EventTime time.Time
|
||||
MassKG float64
|
||||
TankCapacityLiters float64
|
||||
PressureMPa float64
|
||||
TemperatureC float64
|
||||
NoiseKG float64
|
||||
RefuelThresholdKG float64
|
||||
}
|
||||
|
||||
type HydrogenStreamResult struct {
|
||||
Found int
|
||||
Written int
|
||||
Duplicate int
|
||||
Invalid int
|
||||
NotCurrent int
|
||||
}
|
||||
|
||||
func HydrogenDensityKGPerM3(pressureMPa, temperatureC float64) (float64, bool) {
|
||||
temperatureK := temperatureC + 273.15
|
||||
if pressureMPa < 0 || pressureMPa > 70 || temperatureK < 220 || temperatureK > 1000 {
|
||||
return 0, false
|
||||
}
|
||||
z := 1.0
|
||||
for i := range hydrogenDensityA {
|
||||
z += hydrogenDensityA[i] * math.Pow(100/temperatureK, hydrogenDensityB[i]) * math.Pow(pressureMPa, hydrogenDensityC[i])
|
||||
}
|
||||
if z <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
molPerLiter := pressureMPa * 1000 / (universalGasConstant * temperatureK * z)
|
||||
return molPerLiter * hydrogenMolarMassKGPerMol * 1000, true
|
||||
}
|
||||
|
||||
func PressureHydrogenMassKG(pressureMPa, temperatureC, tankCapacityLiters float64) (float64, bool) {
|
||||
if tankCapacityLiters <= 0 || tankCapacityLiters > 10000 {
|
||||
return 0, false
|
||||
}
|
||||
density, ok := HydrogenDensityKGPerM3(pressureMPa, temperatureC)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
mass := density * tankCapacityLiters / 1000
|
||||
return mass, !math.IsNaN(mass) && !math.IsInf(mass, 0) && mass >= 0 && mass <= 500
|
||||
}
|
||||
|
||||
func HydrogenStreamSampleFromEnvelope(env envelope.FrameEnvelope, loc *time.Location, now time.Time, tankCapacityLiters float64) (HydrogenStreamSample, string, bool) {
|
||||
if env.Protocol != envelope.ProtocolGB32960 {
|
||||
return HydrogenStreamSample{}, "unsupported_protocol", false
|
||||
}
|
||||
vin := strings.ToUpper(strings.TrimSpace(env.VIN))
|
||||
if len(vin) != 17 {
|
||||
return HydrogenStreamSample{}, "invalid_vin", false
|
||||
}
|
||||
pressure, pressureOK := firstHydrogenNumber(env.Fields, hydrogenPressureFieldKeys)
|
||||
if !pressureOK {
|
||||
return HydrogenStreamSample{}, "missing_pressure", false
|
||||
}
|
||||
temperature, temperatureOK := firstHydrogenNumber(env.Fields, hydrogenTemperatureFieldKeys)
|
||||
if !temperatureOK {
|
||||
return HydrogenStreamSample{}, "missing_temperature", false
|
||||
}
|
||||
if tankCapacityLiters <= 0 || tankCapacityLiters > 10000 {
|
||||
return HydrogenStreamSample{}, "missing_tank_capacity", false
|
||||
}
|
||||
mass, ok := PressureHydrogenMassKG(pressure, temperature, tankCapacityLiters)
|
||||
if !ok {
|
||||
return HydrogenStreamSample{}, "invalid_pressure_mass", false
|
||||
}
|
||||
pressureStepMass, _ := PressureHydrogenMassKG(math.Max(0, pressure-0.2), temperature, tankCapacityLiters)
|
||||
noiseKG := math.Max(defaultHydrogenNoiseKG, mass-pressureStepMass)
|
||||
noiseKG = math.Min(noiseKG, 1.0)
|
||||
refuelThresholdKG := math.Max(1.0, mass*0.05)
|
||||
if loc == nil {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
eventMS, _, ok := envelope.NormalizedEventTimeMSWithReason(env)
|
||||
if !ok {
|
||||
return HydrogenStreamSample{}, "missing_event_time", false
|
||||
}
|
||||
eventTime := time.UnixMilli(eventMS).In(loc)
|
||||
if now.IsZero() {
|
||||
now = time.Now()
|
||||
}
|
||||
date := eventTime.Format("2006-01-02")
|
||||
if date != now.In(loc).Format("2006-01-02") {
|
||||
return HydrogenStreamSample{}, "not_current_date", false
|
||||
}
|
||||
return HydrogenStreamSample{
|
||||
VIN: vin, Date: date, SourceEndpoint: strings.TrimSpace(env.SourceEndpoint),
|
||||
EventID: env.StableEventID(), EventTime: eventTime, MassKG: mass,
|
||||
TankCapacityLiters: tankCapacityLiters, PressureMPa: pressure, TemperatureC: temperature,
|
||||
NoiseKG: noiseKG, RefuelThresholdKG: refuelThresholdKG,
|
||||
}, "", true
|
||||
}
|
||||
|
||||
func firstHydrogenNumber(fields map[string]any, keys []string) (float64, bool) {
|
||||
for _, key := range keys {
|
||||
if value, ok := telemetry.Number(fields, key); ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func AppendHydrogenStream(ctx context.Context, exec Execer, env envelope.FrameEnvelope, loc *time.Location, now time.Time, tankCapacityLiters float64) (HydrogenStreamResult, error) {
|
||||
sample, reason, ok := HydrogenStreamSampleFromEnvelope(env, loc, now, tankCapacityLiters)
|
||||
if !ok {
|
||||
result := HydrogenStreamResult{}
|
||||
switch reason {
|
||||
case "unsupported_protocol", "missing_pressure", "missing_temperature":
|
||||
return result, nil
|
||||
case "not_current_date":
|
||||
result.NotCurrent = 1
|
||||
default:
|
||||
result.Invalid = 1
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
result := HydrogenStreamResult{Found: 1}
|
||||
beginner, ok := exec.(txBeginner)
|
||||
if !ok {
|
||||
return result, sql.ErrTxDone
|
||||
}
|
||||
tx, err := beginner.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
write, err := tx.ExecContext(ctx, upsertHydrogenStreamStateSQL,
|
||||
sample.VIN, sample.Date, sample.SourceEndpoint, sample.MassKG, sample.MassKG, sample.MassKG,
|
||||
sample.TankCapacityLiters, sample.PressureMPa, sample.PressureMPa,
|
||||
sample.TemperatureC, sample.TemperatureC, sample.EventTime, sample.EventTime, sample.EventID,
|
||||
sample.NoiseKG, defaultHydrogenMaxDropKG, sample.RefuelThresholdKG,
|
||||
defaultHydrogenMaxDropKG, defaultHydrogenMaxDropKG, defaultHydrogenMaxDropKG,
|
||||
sample.RefuelThresholdKG, sample.NoiseKG, defaultHydrogenMaxDropKG,
|
||||
)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
affected, _ := write.RowsAffected()
|
||||
if affected == 0 {
|
||||
result.Duplicate = 1
|
||||
return result, tx.Commit()
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, projectHydrogenStreamDailySQL, sample.VIN, sample.Date); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Written = 1
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const upsertHydrogenStreamStateSQL = `
|
||||
INSERT INTO vehicle_open_hydrogen_stream_state(
|
||||
vin,stat_date,source_endpoint,first_mass_kg,last_mass_kg,cycle_min_mass_kg,consumption_kg,sample_count,
|
||||
refuel_count,abnormal_drop_count,tank_capacity_l,first_pressure_mpa,last_pressure_mpa,
|
||||
first_temperature_c,last_temperature_c,calculation_method,
|
||||
first_event_time,last_event_time,last_event_id,quality_status,quality_reason
|
||||
) VALUES(?,?,?,?,?,?,0,1,0,0,?,?,?,?,?,'PRESSURE_NIST',?,?,?,'NO_DATA','有效压力质量样本不足2条')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
consumption_kg = consumption_kg + IF(VALUES(last_event_time)>last_event_time
|
||||
AND cycle_min_mass_kg-VALUES(last_mass_kg)>? AND cycle_min_mass_kg-VALUES(last_mass_kg)<=?,
|
||||
cycle_min_mass_kg-VALUES(last_mass_kg),0),
|
||||
refuel_count = refuel_count + IF(VALUES(last_event_time)>last_event_time
|
||||
AND VALUES(last_mass_kg)-cycle_min_mass_kg>?,1,0),
|
||||
quality_status = IF(VALUES(last_event_time)<=last_event_time,quality_status,
|
||||
IF(abnormal_drop_count+IF(cycle_min_mass_kg-VALUES(last_mass_kg)>?,1,0)>0,'SUSPECT','OK')),
|
||||
quality_reason = IF(VALUES(last_event_time)<=last_event_time,quality_reason,
|
||||
IF(abnormal_drop_count+IF(cycle_min_mass_kg-VALUES(last_mass_kg)>?,1,0)>0,'存在超过阈值的异常下降','')),
|
||||
abnormal_drop_count = abnormal_drop_count + IF(VALUES(last_event_time)>last_event_time
|
||||
AND cycle_min_mass_kg-VALUES(last_mass_kg)>?,1,0),
|
||||
cycle_min_mass_kg = IF(VALUES(last_event_time)>last_event_time,
|
||||
IF(VALUES(last_mass_kg)-cycle_min_mass_kg>?,VALUES(last_mass_kg),
|
||||
IF(cycle_min_mass_kg-VALUES(last_mass_kg)>? AND cycle_min_mass_kg-VALUES(last_mass_kg)<=?,VALUES(last_mass_kg),cycle_min_mass_kg)),
|
||||
cycle_min_mass_kg),
|
||||
sample_count = sample_count + IF(VALUES(last_event_time)>last_event_time,1,0),
|
||||
tank_capacity_l = IF(VALUES(last_event_time)>last_event_time,VALUES(tank_capacity_l),tank_capacity_l),
|
||||
last_pressure_mpa = IF(VALUES(last_event_time)>last_event_time,VALUES(last_pressure_mpa),last_pressure_mpa),
|
||||
last_temperature_c = IF(VALUES(last_event_time)>last_event_time,VALUES(last_temperature_c),last_temperature_c),
|
||||
calculation_method = 'PRESSURE_NIST',
|
||||
last_mass_kg = IF(VALUES(last_event_time)>last_event_time,VALUES(last_mass_kg),last_mass_kg),
|
||||
last_event_id = IF(VALUES(last_event_time)>last_event_time,VALUES(last_event_id),last_event_id),
|
||||
last_event_time = GREATEST(last_event_time,VALUES(last_event_time))`
|
||||
|
||||
const projectHydrogenStreamDailySQL = `
|
||||
INSERT INTO vehicle_open_daily_energy(
|
||||
vin,stat_date,energy_type,source_endpoint,consumption_kg,unit,first_mass_kg,last_mass_kg,
|
||||
sample_count,refuel_count,quality_status,quality_reason,calculated_at
|
||||
)
|
||||
SELECT vin,stat_date,'HYDROGEN',source_endpoint,consumption_kg,'kg',first_mass_kg,last_mass_kg,
|
||||
sample_count,refuel_count,quality_status,quality_reason,NOW(3)
|
||||
FROM vehicle_open_hydrogen_stream_state
|
||||
WHERE vin=? AND stat_date=?
|
||||
ORDER BY CASE quality_status WHEN 'OK' THEN 0 WHEN 'SUSPECT' THEN 1 ELSE 2 END,
|
||||
sample_count DESC,source_endpoint ASC
|
||||
LIMIT 1
|
||||
ON DUPLICATE KEY UPDATE
|
||||
source_endpoint=VALUES(source_endpoint),consumption_kg=VALUES(consumption_kg),unit='kg',
|
||||
first_mass_kg=VALUES(first_mass_kg),last_mass_kg=VALUES(last_mass_kg),
|
||||
sample_count=VALUES(sample_count),refuel_count=VALUES(refuel_count),
|
||||
quality_status=VALUES(quality_status),quality_reason=VALUES(quality_reason),calculated_at=VALUES(calculated_at)`
|
||||
138
go/vehicle-gateway/internal/stats/hydrogen_stream_test.go
Normal file
138
go/vehicle-gateway/internal/stats/hydrogen_stream_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func pressureEnvelope(at time.Time) envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960, VIN: "LA9GG64L0NBAF4175", EventID: "event-1",
|
||||
EventTimeMS: at.UnixMilli(), SourceEndpoint: "10.0.0.1:9000",
|
||||
Fields: map[string]any{
|
||||
"gb32960.fuel_cell.max_hydrogen_pressure_mpa": 6.0,
|
||||
"gb32960.fuel_cell.max_hydrogen_temperature_c": 44.0,
|
||||
"gb32960.gd_fc_vehicle_info.hydrogen_mass_kg": 99.9,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrogenStreamSampleUsesPressureTemperatureAndCapacity(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
eventTime := time.Date(2026, 7, 21, 10, 30, 0, 0, loc)
|
||||
sample, reason, ok := HydrogenStreamSampleFromEnvelope(pressureEnvelope(eventTime), loc, eventTime.Add(time.Minute), 2050)
|
||||
if !ok || reason != "" {
|
||||
t.Fatalf("sample rejected: reason=%q", reason)
|
||||
}
|
||||
if sample.Date != "2026-07-21" || sample.EventID != "event-1" || sample.TankCapacityLiters != 2050 {
|
||||
t.Fatalf("unexpected sample: %#v", sample)
|
||||
}
|
||||
if math.Abs(sample.MassKG-9.092) > 0.01 {
|
||||
t.Fatalf("pressure mass = %.3fkg, want about 9.092kg", sample.MassKG)
|
||||
}
|
||||
if sample.MassKG == 99.9 {
|
||||
t.Fatal("reported Guangdong hydrogen mass must be ignored")
|
||||
}
|
||||
|
||||
_, reason, ok = HydrogenStreamSampleFromEnvelope(pressureEnvelope(eventTime), loc, eventTime.AddDate(0, 0, 1), 2050)
|
||||
if ok || reason != "not_current_date" {
|
||||
t.Fatalf("historical sample accepted: ok=%v reason=%q", ok, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrogenStreamRejectsMissingCapacity(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
eventTime := time.Date(2026, 7, 21, 10, 30, 0, 0, loc)
|
||||
_, reason, ok := HydrogenStreamSampleFromEnvelope(pressureEnvelope(eventTime), loc, eventTime, 0)
|
||||
if ok || reason != "missing_tank_capacity" {
|
||||
t.Fatalf("missing capacity accepted: ok=%v reason=%q", ok, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNISTHydrogenDensityValidationPoint(t *testing.T) {
|
||||
density, ok := HydrogenDensityKGPerM3(10, 26.85)
|
||||
if !ok || math.Abs(density-7.625) > 0.01 {
|
||||
t.Fatalf("density=%.6f ok=%v", density, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendHydrogenStreamPersistsPressureEvidenceAtomically(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
eventTime := time.Date(2026, 7, 21, 10, 30, 0, 0, loc)
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO vehicle_open_hydrogen_stream_state")).
|
||||
WithArgs("LA9GG64L0NBAF4175", "2026-07-21", "10.0.0.1:9000", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(),
|
||||
2050.0, 6.0, 6.0, 44.0, 44.0, eventTime, eventTime, "event-1",
|
||||
sqlmock.AnyArg(), 20.0, sqlmock.AnyArg(), 20.0, 20.0, 20.0,
|
||||
sqlmock.AnyArg(), sqlmock.AnyArg(), 20.0).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO vehicle_open_daily_energy")).
|
||||
WithArgs("LA9GG64L0NBAF4175", "2026-07-21").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
result, err := AppendHydrogenStream(context.Background(), db, pressureEnvelope(eventTime), loc, eventTime, 2050)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Found != 1 || result.Written != 1 {
|
||||
t.Fatalf("unexpected result: %#v", result)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterAppendWithResultUsesInMemoryCapacity(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
eventTime := time.Now().In(loc).Truncate(time.Millisecond)
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO vehicle_open_hydrogen_stream_state")).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO vehicle_open_daily_energy")).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
writer := NewWriter(db, loc)
|
||||
writer.hydrogenTankCapacities["LA9GG64L0NBAF4175"] = 2050
|
||||
result, err := writer.AppendWithResult(context.Background(), pressureEnvelope(eventTime))
|
||||
if err != nil {
|
||||
t.Fatalf("AppendWithResult() error = %v", err)
|
||||
}
|
||||
if result.HydrogenSamplesFound != 1 || result.HydrogenSamplesWritten != 1 {
|
||||
t.Fatalf("hydrogen counters lost: %+v", result)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrogenStreamSQLGuardsDuplicatesAndRecordsPressure(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"VALUES(last_event_time)>last_event_time",
|
||||
"cycle_min_mass_kg-VALUES(last_mass_kg)>?",
|
||||
"tank_capacity_l",
|
||||
"last_pressure_mpa",
|
||||
"last_temperature_c",
|
||||
"PRESSURE_NIST",
|
||||
} {
|
||||
if !regexp.MustCompile(regexp.QuoteMeta(want)).MatchString(upsertHydrogenStreamStateSQL) {
|
||||
t.Fatalf("stream upsert missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
120
go/vehicle-gateway/internal/stats/pure_hydrogen.go
Normal file
120
go/vehicle-gateway/internal/stats/pure_hydrogen.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const (
|
||||
gb32960FuelCellEngineWorkMode = 2
|
||||
yutongFuelCellRunningMode = 4
|
||||
yutongFuelCellDrivingMode = 11
|
||||
)
|
||||
|
||||
// PureHydrogenModeFromEnvelope returns whether the reported vehicle is in a
|
||||
// fuel-cell-only work mode and whether the source value is understood.
|
||||
//
|
||||
// The 32960 fuel-cell extension reports engine_work_state; production frames
|
||||
// use mode 2 for normal fuel-cell operation. Yutong reports the equivalent
|
||||
// state through TRIANGLE_STATE; production frames use 1 for inactive and 4/11
|
||||
// for active fuel-cell operating modes.
|
||||
func PureHydrogenModeFromEnvelope(env envelope.FrameEnvelope) (active bool, known bool) {
|
||||
value, exists := env.Fields[envelope.FieldFuelCellWorkMode]
|
||||
if !exists {
|
||||
return false, false
|
||||
}
|
||||
mode, ok := integerMode(value)
|
||||
if !ok {
|
||||
return false, false
|
||||
}
|
||||
switch env.Protocol {
|
||||
case envelope.ProtocolGB32960:
|
||||
switch mode {
|
||||
case gb32960FuelCellEngineWorkMode:
|
||||
return true, true
|
||||
case 0, 1:
|
||||
return false, true
|
||||
default:
|
||||
return false, false
|
||||
}
|
||||
case envelope.ProtocolYutongMQTT:
|
||||
switch mode {
|
||||
case yutongFuelCellRunningMode, yutongFuelCellDrivingMode:
|
||||
return true, true
|
||||
case 1:
|
||||
return false, true
|
||||
default:
|
||||
return false, false
|
||||
}
|
||||
default:
|
||||
return false, false
|
||||
}
|
||||
}
|
||||
|
||||
// PureHydrogenMileageDelta returns the odometer delta that can be attributed
|
||||
// to pure-hydrogen operation. Both endpoints must report a known active mode;
|
||||
// this deliberately leaves transitions and missing mode data unclassified.
|
||||
func PureHydrogenMileageDelta(
|
||||
previousKM float64,
|
||||
currentKM float64,
|
||||
previousActive bool,
|
||||
previousKnown bool,
|
||||
currentActive bool,
|
||||
currentKnown bool,
|
||||
) float64 {
|
||||
delta := currentKM - previousKM
|
||||
if !previousKnown || !previousActive || !currentKnown || !currentActive {
|
||||
return 0
|
||||
}
|
||||
if delta < 0 || delta > maxSelectedDailyMileageKM {
|
||||
return 0
|
||||
}
|
||||
return delta
|
||||
}
|
||||
|
||||
func integerMode(value any) (int, bool) {
|
||||
switch typed := value.(type) {
|
||||
case json.Number:
|
||||
parsed, err := strconv.Atoi(typed.String())
|
||||
return parsed, err == nil
|
||||
case string:
|
||||
parsed, err := strconv.Atoi(strings.TrimSpace(typed))
|
||||
return parsed, err == nil
|
||||
case int:
|
||||
return typed, true
|
||||
case int8:
|
||||
return int(typed), true
|
||||
case int16:
|
||||
return int(typed), true
|
||||
case int32:
|
||||
return int(typed), true
|
||||
case int64:
|
||||
return int(typed), true
|
||||
case uint:
|
||||
return int(typed), true
|
||||
case uint8:
|
||||
return int(typed), true
|
||||
case uint16:
|
||||
return int(typed), true
|
||||
case uint32:
|
||||
return int(typed), true
|
||||
case uint64:
|
||||
if typed > uint64(^uint(0)>>1) {
|
||||
return 0, false
|
||||
}
|
||||
return int(typed), true
|
||||
case float32:
|
||||
parsed := int(typed)
|
||||
return parsed, float32(parsed) == typed
|
||||
case float64:
|
||||
parsed := int(typed)
|
||||
return parsed, float64(parsed) == typed
|
||||
default:
|
||||
parsed, err := strconv.Atoi(strings.TrimSpace(fmt.Sprint(value)))
|
||||
return parsed, err == nil
|
||||
}
|
||||
}
|
||||
134
go/vehicle-gateway/internal/stats/pure_hydrogen_test.go
Normal file
134
go/vehicle-gateway/internal/stats/pure_hydrogen_test.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestPureHydrogenModeFromEnvelope(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env envelope.FrameEnvelope
|
||||
active bool
|
||||
known bool
|
||||
}{
|
||||
{
|
||||
name: "gb32960 fuel cell mode",
|
||||
env: envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, Fields: map[string]any{envelope.FieldFuelCellWorkMode: 2}},
|
||||
active: true,
|
||||
known: true,
|
||||
},
|
||||
{
|
||||
name: "gb32960 electric mode",
|
||||
env: envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, Fields: map[string]any{envelope.FieldFuelCellWorkMode: 1}},
|
||||
known: true,
|
||||
},
|
||||
{
|
||||
name: "yutong active mode",
|
||||
env: envelope.FrameEnvelope{Protocol: envelope.ProtocolYutongMQTT, Fields: map[string]any{envelope.FieldFuelCellWorkMode: json.Number("4")}},
|
||||
active: true,
|
||||
known: true,
|
||||
},
|
||||
{
|
||||
name: "yutong driving mode",
|
||||
env: envelope.FrameEnvelope{Protocol: envelope.ProtocolYutongMQTT, Fields: map[string]any{envelope.FieldFuelCellWorkMode: "11"}},
|
||||
active: true,
|
||||
known: true,
|
||||
},
|
||||
{
|
||||
name: "yutong inactive mode",
|
||||
env: envelope.FrameEnvelope{Protocol: envelope.ProtocolYutongMQTT, Fields: map[string]any{envelope.FieldFuelCellWorkMode: float64(1)}},
|
||||
known: true,
|
||||
},
|
||||
{
|
||||
name: "unknown yutong mode",
|
||||
env: envelope.FrameEnvelope{Protocol: envelope.ProtocolYutongMQTT, Fields: map[string]any{envelope.FieldFuelCellWorkMode: 9}},
|
||||
},
|
||||
{
|
||||
name: "unsupported protocol",
|
||||
env: envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Fields: map[string]any{envelope.FieldFuelCellWorkMode: 3}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
active, known := PureHydrogenModeFromEnvelope(test.env)
|
||||
if active != test.active || known != test.known {
|
||||
t.Fatalf("PureHydrogenModeFromEnvelope() = (%v, %v), want (%v, %v)", active, known, test.active, test.known)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPureHydrogenMileageDeltaRequiresContinuousKnownActiveMode(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
previousKM, currentKM float64
|
||||
previousActive, previousKnown bool
|
||||
currentActive, currentKnown bool
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "continuous active interval",
|
||||
previousKM: 100,
|
||||
currentKM: 106.5,
|
||||
previousActive: true,
|
||||
previousKnown: true,
|
||||
currentActive: true,
|
||||
currentKnown: true,
|
||||
want: 6.5,
|
||||
},
|
||||
{
|
||||
name: "transition into active is unclassified",
|
||||
previousKM: 100,
|
||||
currentKM: 106.5,
|
||||
previousKnown: true,
|
||||
currentActive: true,
|
||||
currentKnown: true,
|
||||
},
|
||||
{
|
||||
name: "unknown endpoint is unclassified",
|
||||
previousKM: 100,
|
||||
currentKM: 106.5,
|
||||
previousActive: true,
|
||||
previousKnown: true,
|
||||
currentActive: true,
|
||||
},
|
||||
{
|
||||
name: "odometer rollback is rejected",
|
||||
previousKM: 106.5,
|
||||
currentKM: 100,
|
||||
previousActive: true,
|
||||
previousKnown: true,
|
||||
currentActive: true,
|
||||
currentKnown: true,
|
||||
},
|
||||
{
|
||||
name: "implausible jump is rejected",
|
||||
previousKM: 100,
|
||||
currentKM: 2601,
|
||||
previousActive: true,
|
||||
previousKnown: true,
|
||||
currentActive: true,
|
||||
currentKnown: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got := PureHydrogenMileageDelta(
|
||||
test.previousKM,
|
||||
test.currentKM,
|
||||
test.previousActive,
|
||||
test.previousKnown,
|
||||
test.currentActive,
|
||||
test.currentKnown,
|
||||
)
|
||||
if got != test.want {
|
||||
t.Fatalf("PureHydrogenMileageDelta() = %v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -58,48 +58,51 @@ type MetricDiagnosisQuery struct {
|
||||
}
|
||||
|
||||
type MetricRow struct {
|
||||
VIN string `json:"vin"`
|
||||
StatDate string `json:"stat_date"`
|
||||
Protocol string `json:"protocol"`
|
||||
SourceID *int64 `json:"source_id,omitempty"`
|
||||
SourceIP string `json:"source_ip,omitempty"`
|
||||
LatestSourceEndpoint string `json:"latest_source_endpoint,omitempty"`
|
||||
PlatformName string `json:"platform_name,omitempty"`
|
||||
SourceCode string `json:"source_code,omitempty"`
|
||||
SourceKind string `json:"source_kind,omitempty"`
|
||||
DailyMileageKM float64 `json:"daily_mileage_km"`
|
||||
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
VIN string `json:"vin"`
|
||||
StatDate string `json:"stat_date"`
|
||||
Protocol string `json:"protocol"`
|
||||
SourceID *int64 `json:"source_id,omitempty"`
|
||||
SourceIP string `json:"source_ip,omitempty"`
|
||||
LatestSourceEndpoint string `json:"latest_source_endpoint,omitempty"`
|
||||
PlatformName string `json:"platform_name,omitempty"`
|
||||
SourceCode string `json:"source_code,omitempty"`
|
||||
SourceKind string `json:"source_kind,omitempty"`
|
||||
DailyMileageKM float64 `json:"daily_mileage_km"`
|
||||
PureHydrogenMileageKM float64 `json:"pure_hydrogen_mileage_km"`
|
||||
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type MetricSourceRow struct {
|
||||
VIN string `json:"vin"`
|
||||
StatDate string `json:"stat_date"`
|
||||
Protocol string `json:"protocol"`
|
||||
SourceKey string `json:"source_key"`
|
||||
SourceIP string `json:"source_ip"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
PlatformName string `json:"platform_name,omitempty"`
|
||||
SourceID *int64 `json:"source_id,omitempty"`
|
||||
SourceCode string `json:"source_code,omitempty"`
|
||||
SourceKind string `json:"source_kind,omitempty"`
|
||||
SourceEnabled *bool `json:"source_enabled,omitempty"`
|
||||
TrustPriority *int `json:"trust_priority,omitempty"`
|
||||
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
|
||||
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
|
||||
DailyMileageKM float64 `json:"daily_mileage_km"`
|
||||
SampleCount int64 `json:"sample_count"`
|
||||
FirstEventTime string `json:"first_event_time,omitempty"`
|
||||
LatestEventTime string `json:"latest_event_time,omitempty"`
|
||||
QualityStatus string `json:"quality_status"`
|
||||
QualityReason string `json:"quality_reason,omitempty"`
|
||||
IsSelected bool `json:"is_selected"`
|
||||
SelectionStatus string `json:"selection_status,omitempty"`
|
||||
SelectionReason string `json:"selection_reason,omitempty"`
|
||||
SelectionAction string `json:"selection_action,omitempty"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
VIN string `json:"vin"`
|
||||
StatDate string `json:"stat_date"`
|
||||
Protocol string `json:"protocol"`
|
||||
SourceKey string `json:"source_key"`
|
||||
SourceIP string `json:"source_ip"`
|
||||
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||
Phone string `json:"phone,omitempty"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
PlatformName string `json:"platform_name,omitempty"`
|
||||
SourceID *int64 `json:"source_id,omitempty"`
|
||||
SourceCode string `json:"source_code,omitempty"`
|
||||
SourceKind string `json:"source_kind,omitempty"`
|
||||
SourceEnabled *bool `json:"source_enabled,omitempty"`
|
||||
TrustPriority *int `json:"trust_priority,omitempty"`
|
||||
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
|
||||
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
|
||||
DailyMileageKM float64 `json:"daily_mileage_km"`
|
||||
PureHydrogenMileageKM float64 `json:"pure_hydrogen_mileage_km"`
|
||||
PureHydrogenSampleCount int64 `json:"pure_hydrogen_sample_count"`
|
||||
SampleCount int64 `json:"sample_count"`
|
||||
FirstEventTime string `json:"first_event_time,omitempty"`
|
||||
LatestEventTime string `json:"latest_event_time,omitempty"`
|
||||
QualityStatus string `json:"quality_status"`
|
||||
QualityReason string `json:"quality_reason,omitempty"`
|
||||
IsSelected bool `json:"is_selected"`
|
||||
SelectionStatus string `json:"selection_status,omitempty"`
|
||||
SelectionReason string `json:"selection_reason,omitempty"`
|
||||
SelectionAction string `json:"selection_action,omitempty"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type MetricDiagnosisRow struct {
|
||||
@@ -241,6 +244,7 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
|
||||
&sourceCode,
|
||||
&sourceKind,
|
||||
&row.DailyMileageKM,
|
||||
&row.PureHydrogenMileageKM,
|
||||
&latest,
|
||||
&updatedAt,
|
||||
); err != nil {
|
||||
@@ -303,7 +307,9 @@ func (r *MetricRepository) QuerySources(ctx context.Context, query MetricSourceQ
|
||||
&firstTotal,
|
||||
&latestTotal,
|
||||
&row.DailyMileageKM,
|
||||
&row.PureHydrogenMileageKM,
|
||||
&row.SampleCount,
|
||||
&row.PureHydrogenSampleCount,
|
||||
&firstEvent,
|
||||
&latestEvent,
|
||||
&row.QualityStatus,
|
||||
@@ -1108,7 +1114,7 @@ func buildMetricSQL(query MetricQuery) (string, []any) {
|
||||
where, args := buildMetricWhere(query)
|
||||
sqlText := `SELECT m.vin, m.stat_date, m.protocol, m.source_id,
|
||||
ds.source_ip, ds.latest_source_endpoint, ds.platform_name, ds.source_code, ds.source_kind,
|
||||
m.daily_mileage_km, m.latest_total_mileage_km, m.updated_at
|
||||
m.daily_mileage_km, m.pure_hydrogen_mileage_km, m.latest_total_mileage_km, m.updated_at
|
||||
FROM vehicle_daily_mileage m
|
||||
LEFT JOIN vehicle_data_source ds ON ds.id = m.source_id`
|
||||
if len(where) > 0 {
|
||||
@@ -1124,7 +1130,8 @@ func buildMetricSourceSQL(query MetricSourceQuery) (string, []any) {
|
||||
sqlText := `SELECT s.vin, s.stat_date, s.protocol, s.source_key, s.source_ip, s.source_endpoint, s.phone, s.device_id,
|
||||
COALESCE(NULLIF(TRIM(s.platform_name), ''), ds.platform_name) AS platform_name,
|
||||
ds.id, ds.source_code, ` + metricSourceKindSQL("s") + ` AS source_kind, ds.enabled, ds.trust_priority,
|
||||
s.first_total_mileage_km, s.latest_total_mileage_km, s.daily_mileage_km, s.sample_count,
|
||||
s.first_total_mileage_km, s.latest_total_mileage_km, s.daily_mileage_km, s.pure_hydrogen_mileage_km,
|
||||
s.sample_count, s.pure_hydrogen_sample_count,
|
||||
s.first_event_time, s.latest_event_time, s.quality_status, s.quality_reason, s.is_selected, s.updated_at
|
||||
FROM vehicle_daily_mileage_source s
|
||||
LEFT JOIN vehicle_data_source ds ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip`
|
||||
|
||||
@@ -32,10 +32,10 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "stat_date", "protocol", "source_id",
|
||||
"source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km",
|
||||
"latest_total_mileage_km", "updated_at",
|
||||
"pure_hydrogen_mileage_km", "latest_total_mileage_km", "updated_at",
|
||||
}).AddRow(
|
||||
"LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", 3,
|
||||
"115.231.168.135", "115.231.168.135:41561", "G7 平台", "G7S", "PLATFORM", 12.3,
|
||||
"115.231.168.135", "115.231.168.135:41561", "G7 平台", "G7S", "PLATFORM", 12.3, 0.0,
|
||||
12357.9, time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
|
||||
))
|
||||
|
||||
@@ -80,10 +80,10 @@ func TestMetricQueryReturnsSelectedSourceID(t *testing.T) {
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "stat_date", "protocol", "source_id",
|
||||
"source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km",
|
||||
"latest_total_mileage_km", "updated_at",
|
||||
"pure_hydrogen_mileage_km", "latest_total_mileage_km", "updated_at",
|
||||
}).AddRow(
|
||||
"LA9GG64L7PBAF4001", "2026-07-08", "JT808", 2,
|
||||
"117.132.194.31", "117.132.194.31:20471", "广安北斗", "guangan_beidou", "PLATFORM", 23.1,
|
||||
"117.132.194.31", "117.132.194.31:20471", "广安北斗", "guangan_beidou", "PLATFORM", 23.1, 0.0,
|
||||
4123.9, "2026-07-08 13:30:57",
|
||||
))
|
||||
|
||||
@@ -115,13 +115,13 @@ func TestMetricRepositoryQueriesDailyMetricSourcesWithEvidence(t *testing.T) {
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "stat_date", "protocol", "source_key", "source_ip", "source_endpoint", "phone", "device_id",
|
||||
"platform_name", "source_id", "source_code", "source_kind", "enabled", "trust_priority",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "daily_mileage_km", "sample_count",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "daily_mileage_km", "pure_hydrogen_mileage_km", "sample_count", "pure_hydrogen_sample_count",
|
||||
"first_event_time", "latest_event_time", "quality_status", "quality_reason", "is_selected", "updated_at",
|
||||
}).AddRow(
|
||||
"LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307795425@115.231.168.135", "115.231.168.135",
|
||||
"115.231.168.135:41561", "13307795425", nil,
|
||||
"信达", 5, "xinda", "PLATFORM", 1, 10,
|
||||
4100.8, 4123.9, 23.1, 128,
|
||||
4100.8, 4123.9, 23.1, 18.6, 128, 96,
|
||||
"2026-07-08 00:01:00", "2026-07-08 23:59:00", "OK", "historical_source_baseline", 1, "2026-07-08 23:59:10",
|
||||
))
|
||||
|
||||
@@ -856,10 +856,10 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "stat_date", "protocol", "source_id",
|
||||
"source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km",
|
||||
"latest_total_mileage_km", "updated_at",
|
||||
"pure_hydrogen_mileage_km", "latest_total_mileage_km", "updated_at",
|
||||
}).AddRow(
|
||||
"LB9A32A21R0LS1707", "2020-07-01", "GB32960", 1,
|
||||
"8.134.95.166", "8.134.95.166:32960", "现代 HTWO", "HYUNDAI", "PLATFORM", 0.0,
|
||||
"8.134.95.166", "8.134.95.166:32960", "现代 HTWO", "HYUNDAI", "PLATFORM", 0.0, 0.0,
|
||||
53490.9, "2026-07-01 22:28:25",
|
||||
))
|
||||
|
||||
@@ -900,13 +900,13 @@ func TestMetricHandlerReturnsDailyMetricSources(t *testing.T) {
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "stat_date", "protocol", "source_key", "source_ip", "source_endpoint", "phone", "device_id",
|
||||
"platform_name", "source_id", "source_code", "source_kind", "enabled", "trust_priority",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "daily_mileage_km", "sample_count",
|
||||
"first_total_mileage_km", "latest_total_mileage_km", "daily_mileage_km", "pure_hydrogen_mileage_km", "sample_count", "pure_hydrogen_sample_count",
|
||||
"first_event_time", "latest_event_time", "quality_status", "quality_reason", "is_selected", "updated_at",
|
||||
}).AddRow(
|
||||
"LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307795425@115.231.168.135", "115.231.168.135",
|
||||
"115.231.168.135:41561", "13307795425", nil,
|
||||
"信达", 5, "xinda", "PLATFORM", 1, 10,
|
||||
4100.8, 4123.9, 23.1, 128,
|
||||
4100.8, 4123.9, 23.1, 18.6, 128, 96,
|
||||
"2026-07-08 00:01:00", "2026-07-08 23:59:00", "OK", "historical_source_baseline", 1, "2026-07-08 23:59:10",
|
||||
))
|
||||
|
||||
@@ -1340,7 +1340,7 @@ func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "stat_date", "protocol", "source_id",
|
||||
"source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km",
|
||||
"latest_total_mileage_km", "updated_at",
|
||||
"pure_hydrogen_mileage_km", "latest_total_mileage_km", "updated_at",
|
||||
}))
|
||||
|
||||
handler := NewMetricHandler(NewMetricRepository(db))
|
||||
@@ -1372,10 +1372,10 @@ func TestMetricHandlerSkipsTotalCountByDefault(t *testing.T) {
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "stat_date", "protocol", "source_id",
|
||||
"source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km",
|
||||
"latest_total_mileage_km", "updated_at",
|
||||
"pure_hydrogen_mileage_km", "latest_total_mileage_km", "updated_at",
|
||||
}).AddRow(
|
||||
"LKLG7C4E3NA774736", "2026-07-02", "JT808", 3,
|
||||
"115.231.168.135", "115.231.168.135:41561", "G7 平台", "G7S", "PLATFORM", 12.3,
|
||||
"115.231.168.135", "115.231.168.135:41561", "G7 平台", "G7S", "PLATFORM", 12.3, 0.0,
|
||||
8805.1, "2026-07-02 23:59:59",
|
||||
))
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage (
|
||||
protocol VARCHAR(32) NOT NULL,
|
||||
source_id BIGINT NULL,
|
||||
daily_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0,
|
||||
pure_hydrogen_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (vin, stat_date, protocol),
|
||||
@@ -20,6 +21,7 @@ var DailyMileageAlterSQL = []string{
|
||||
"ALTER TABLE vehicle_data_source ADD KEY idx_protocol_source_code (protocol, source_code)",
|
||||
"ALTER TABLE vehicle_data_source ADD KEY idx_protocol_source_kind_seen (protocol, source_kind, latest_seen_at)",
|
||||
"ALTER TABLE vehicle_daily_mileage ADD COLUMN source_id BIGINT NULL AFTER protocol",
|
||||
"ALTER TABLE vehicle_daily_mileage ADD COLUMN pure_hydrogen_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0 AFTER daily_mileage_km",
|
||||
"ALTER TABLE vehicle_daily_mileage ADD KEY idx_source_id (source_id)",
|
||||
"ALTER TABLE vehicle_daily_mileage DROP COLUMN first_total_mileage_km",
|
||||
"ALTER TABLE vehicle_daily_mileage DROP COLUMN trusted_source_key",
|
||||
@@ -30,6 +32,10 @@ var DailyMileageAlterSQL = []string{
|
||||
"ALTER TABLE vehicle_daily_mileage_source ADD KEY idx_protocol_date_quality (protocol, stat_date, quality_status, vin)",
|
||||
"ALTER TABLE vehicle_daily_mileage_source ADD KEY idx_protocol_date_quality_reason (protocol, stat_date, quality_status, quality_reason, vin)",
|
||||
"ALTER TABLE vehicle_daily_mileage_source ADD KEY idx_source_ip_date (protocol, source_ip, stat_date)",
|
||||
"ALTER TABLE vehicle_daily_mileage_source ADD COLUMN pure_hydrogen_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0 AFTER daily_mileage_km",
|
||||
"ALTER TABLE vehicle_daily_mileage_source ADD COLUMN pure_hydrogen_sample_count BIGINT NOT NULL DEFAULT 0 AFTER sample_count",
|
||||
"ALTER TABLE vehicle_daily_mileage_source ADD COLUMN latest_pure_hydrogen_active TINYINT(1) NOT NULL DEFAULT 0 AFTER latest_event_time",
|
||||
"ALTER TABLE vehicle_daily_mileage_source ADD COLUMN latest_pure_hydrogen_mode_known TINYINT(1) NOT NULL DEFAULT 0 AFTER latest_pure_hydrogen_active",
|
||||
}
|
||||
|
||||
const DataSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_data_source (
|
||||
@@ -67,9 +73,13 @@ const DailyMileageSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mil
|
||||
first_total_mileage_km DECIMAL(18,3) NULL,
|
||||
latest_total_mileage_km DECIMAL(18,3) NULL,
|
||||
daily_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0,
|
||||
pure_hydrogen_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0,
|
||||
sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
pure_hydrogen_sample_count BIGINT NOT NULL DEFAULT 0,
|
||||
first_event_time DATETIME NULL,
|
||||
latest_event_time DATETIME NULL,
|
||||
latest_pure_hydrogen_active TINYINT(1) NOT NULL DEFAULT 0,
|
||||
latest_pure_hydrogen_mode_known TINYINT(1) NOT NULL DEFAULT 0,
|
||||
quality_status VARCHAR(32) NOT NULL DEFAULT 'OK',
|
||||
quality_reason VARCHAR(255) NULL,
|
||||
is_selected TINYINT(1) NOT NULL DEFAULT 0,
|
||||
|
||||
@@ -24,23 +24,27 @@ const (
|
||||
)
|
||||
|
||||
type SourceMileageSample struct {
|
||||
VIN string
|
||||
StatDate string
|
||||
Protocol envelope.Protocol
|
||||
SourceKey string
|
||||
SourceIP string
|
||||
SourceEndpoint string
|
||||
Phone string
|
||||
DeviceID string
|
||||
PlatformName string
|
||||
FirstTotalKM float64
|
||||
LatestTotalKM float64
|
||||
DailyKM float64
|
||||
SampleCount int64
|
||||
FirstEventTime time.Time
|
||||
LatestEventTime time.Time
|
||||
QualityStatus string
|
||||
QualityReason string
|
||||
VIN string
|
||||
StatDate string
|
||||
Protocol envelope.Protocol
|
||||
SourceKey string
|
||||
SourceIP string
|
||||
SourceEndpoint string
|
||||
Phone string
|
||||
DeviceID string
|
||||
PlatformName string
|
||||
FirstTotalKM float64
|
||||
LatestTotalKM float64
|
||||
DailyKM float64
|
||||
PureHydrogenMileageKM float64
|
||||
SampleCount int64
|
||||
PureHydrogenSampleCount int64
|
||||
FirstEventTime time.Time
|
||||
LatestEventTime time.Time
|
||||
LatestPureHydrogenActive bool
|
||||
LatestPureHydrogenModeKnown bool
|
||||
QualityStatus string
|
||||
QualityReason string
|
||||
}
|
||||
|
||||
func SourceKey(protocol envelope.Protocol, phone string, deviceID string, sourceIP string) string {
|
||||
@@ -80,26 +84,37 @@ func SourceKeyForSource(protocol envelope.Protocol, phone string, deviceID strin
|
||||
func SourceMileageSampleFromMetric(sample MetricSample, identity SourceIdentity) SourceMileageSample {
|
||||
eventTime := sample.EventTime
|
||||
return SourceMileageSample{
|
||||
VIN: sample.VIN,
|
||||
StatDate: sample.StatDate,
|
||||
Protocol: sample.Protocol,
|
||||
SourceKey: SourceKeyForSource(sample.Protocol, sample.Phone, sample.DeviceID, identity.SourceIP, identity.SourceKind, identity.SourceCode),
|
||||
SourceIP: identity.SourceIP,
|
||||
SourceEndpoint: identity.SourceEndpoint,
|
||||
Phone: sample.Phone,
|
||||
DeviceID: sample.DeviceID,
|
||||
PlatformName: firstNonEmpty(sample.PlatformName, identity.PlatformName),
|
||||
FirstTotalKM: sample.TotalMileageKM,
|
||||
LatestTotalKM: sample.TotalMileageKM,
|
||||
DailyKM: 0,
|
||||
SampleCount: 1,
|
||||
FirstEventTime: eventTime,
|
||||
LatestEventTime: eventTime,
|
||||
QualityStatus: QualityOK,
|
||||
QualityReason: "realtime_sample",
|
||||
VIN: sample.VIN,
|
||||
StatDate: sample.StatDate,
|
||||
Protocol: sample.Protocol,
|
||||
SourceKey: SourceKeyForSource(sample.Protocol, sample.Phone, sample.DeviceID, identity.SourceIP, identity.SourceKind, identity.SourceCode),
|
||||
SourceIP: identity.SourceIP,
|
||||
SourceEndpoint: identity.SourceEndpoint,
|
||||
Phone: sample.Phone,
|
||||
DeviceID: sample.DeviceID,
|
||||
PlatformName: firstNonEmpty(sample.PlatformName, identity.PlatformName),
|
||||
FirstTotalKM: sample.TotalMileageKM,
|
||||
LatestTotalKM: sample.TotalMileageKM,
|
||||
DailyKM: 0,
|
||||
PureHydrogenMileageKM: 0,
|
||||
SampleCount: 1,
|
||||
PureHydrogenSampleCount: boolCount(sample.PureHydrogenActive && sample.PureHydrogenModeKnown),
|
||||
FirstEventTime: eventTime,
|
||||
LatestEventTime: eventTime,
|
||||
LatestPureHydrogenActive: sample.PureHydrogenActive,
|
||||
LatestPureHydrogenModeKnown: sample.PureHydrogenModeKnown,
|
||||
QualityStatus: QualityOK,
|
||||
QualityReason: "realtime_sample",
|
||||
}
|
||||
}
|
||||
|
||||
func boolCount(value bool) int64 {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
@@ -203,9 +218,13 @@ func UpsertSourceMileage(ctx context.Context, exec Execer, sample SourceMileageS
|
||||
sample.FirstTotalKM,
|
||||
sample.LatestTotalKM,
|
||||
sample.DailyKM,
|
||||
sample.PureHydrogenMileageKM,
|
||||
sample.SampleCount,
|
||||
sample.PureHydrogenSampleCount,
|
||||
sample.FirstEventTime,
|
||||
sample.LatestEventTime,
|
||||
sample.LatestPureHydrogenActive,
|
||||
sample.LatestPureHydrogenModeKnown,
|
||||
sample.QualityStatus,
|
||||
sample.QualityReason,
|
||||
)
|
||||
@@ -422,18 +441,44 @@ const normalizePlatformMaxMileageSQL = `(` + normalizePlatformQualityWindowDaysS
|
||||
|
||||
const normalizePlatformOutsideDailyRangeSQL = `(` + normalizePlatformDailySQL + ` < 0 OR ` + normalizePlatformDailySQL + ` > ` + normalizePlatformMaxMileageSQL + `)`
|
||||
|
||||
const upsertSourcePureHydrogenIncrementSQL = `CASE
|
||||
WHEN latest_event_time IS NOT NULL
|
||||
AND VALUES(latest_event_time) > latest_event_time
|
||||
AND latest_pure_hydrogen_mode_known = 1
|
||||
AND latest_pure_hydrogen_active = 1
|
||||
AND VALUES(latest_pure_hydrogen_mode_known) = 1
|
||||
AND VALUES(latest_pure_hydrogen_active) = 1
|
||||
AND VALUES(latest_total_mileage_km) >= latest_total_mileage_km
|
||||
AND VALUES(latest_total_mileage_km) - latest_total_mileage_km <= ` + maxSelectedDailyMileageKMSQL + `
|
||||
THEN VALUES(latest_total_mileage_km) - latest_total_mileage_km
|
||||
ELSE 0
|
||||
END`
|
||||
|
||||
const upsertSourceMileageSQL = `
|
||||
INSERT INTO vehicle_daily_mileage_source
|
||||
(vin, stat_date, protocol, source_key, source_ip, source_endpoint, phone, device_id, platform_name,
|
||||
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count,
|
||||
first_event_time, latest_event_time, quality_status, quality_reason)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, pure_hydrogen_mileage_km,
|
||||
sample_count, pure_hydrogen_sample_count, first_event_time, latest_event_time,
|
||||
latest_pure_hydrogen_active, latest_pure_hydrogen_mode_known, quality_status, quality_reason)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
source_ip = VALUES(source_ip),
|
||||
source_endpoint = VALUES(source_endpoint),
|
||||
phone = VALUES(phone),
|
||||
device_id = VALUES(device_id),
|
||||
platform_name = COALESCE(NULLIF(TRIM(VALUES(platform_name)), ''), platform_name),
|
||||
pure_hydrogen_mileage_km = pure_hydrogen_mileage_km + ` + upsertSourcePureHydrogenIncrementSQL + `,
|
||||
pure_hydrogen_sample_count = pure_hydrogen_sample_count + VALUES(pure_hydrogen_sample_count),
|
||||
latest_pure_hydrogen_active = CASE
|
||||
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time
|
||||
THEN VALUES(latest_pure_hydrogen_active)
|
||||
ELSE latest_pure_hydrogen_active
|
||||
END,
|
||||
latest_pure_hydrogen_mode_known = CASE
|
||||
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time
|
||||
THEN VALUES(latest_pure_hydrogen_mode_known)
|
||||
ELSE latest_pure_hydrogen_mode_known
|
||||
END,
|
||||
first_total_mileage_km = ` + upsertSourceMergedFirstTotalSQL + `,
|
||||
latest_total_mileage_km = ` + upsertSourceMergedLatestTotalSQL + `,
|
||||
daily_mileage_km = ` + upsertSourceMergedDailySQL + `,
|
||||
@@ -456,8 +501,9 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
const normalizePlatformSourceMileageInsertSQL = `
|
||||
INSERT INTO vehicle_daily_mileage_source
|
||||
(vin, stat_date, protocol, source_key, source_ip, source_endpoint, phone, device_id, platform_name,
|
||||
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count,
|
||||
first_event_time, latest_event_time, quality_status, quality_reason)
|
||||
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, pure_hydrogen_mileage_km,
|
||||
sample_count, pure_hydrogen_sample_count, first_event_time, latest_event_time,
|
||||
latest_pure_hydrogen_active, latest_pure_hydrogen_mode_known, quality_status, quality_reason)
|
||||
SELECT
|
||||
s.vin,
|
||||
s.stat_date,
|
||||
@@ -471,9 +517,13 @@ SELECT
|
||||
` + normalizePlatformFirstTotalSQL + ` AS first_total_mileage_km,
|
||||
` + normalizePlatformLatestTotalSQL + ` AS latest_total_mileage_km,
|
||||
` + normalizePlatformDailySQL + ` AS daily_mileage_km,
|
||||
SUM(s.pure_hydrogen_mileage_km) AS pure_hydrogen_mileage_km,
|
||||
SUM(s.sample_count) AS sample_count,
|
||||
SUM(s.pure_hydrogen_sample_count) AS pure_hydrogen_sample_count,
|
||||
MIN(s.first_event_time) AS first_event_time,
|
||||
MAX(s.latest_event_time) AS latest_event_time,
|
||||
CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_pure_hydrogen_active ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS UNSIGNED) AS latest_pure_hydrogen_active,
|
||||
CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_pure_hydrogen_mode_known ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS UNSIGNED) AS latest_pure_hydrogen_mode_known,
|
||||
CASE
|
||||
WHEN ` + normalizePlatformOutsideDailyRangeSQL + ` THEN '` + QualityInvalidDelta + `'
|
||||
ELSE '` + QualityOK + `'
|
||||
@@ -530,6 +580,16 @@ ON DUPLICATE KEY UPDATE
|
||||
phone = COALESCE(NULLIF(TRIM(VALUES(phone)), ''), phone),
|
||||
device_id = COALESCE(NULLIF(TRIM(VALUES(device_id)), ''), device_id),
|
||||
platform_name = COALESCE(NULLIF(TRIM(VALUES(platform_name)), ''), platform_name),
|
||||
pure_hydrogen_mileage_km = VALUES(pure_hydrogen_mileage_km),
|
||||
pure_hydrogen_sample_count = VALUES(pure_hydrogen_sample_count),
|
||||
latest_pure_hydrogen_active = CASE
|
||||
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(latest_pure_hydrogen_active)
|
||||
ELSE latest_pure_hydrogen_active
|
||||
END,
|
||||
latest_pure_hydrogen_mode_known = CASE
|
||||
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(latest_pure_hydrogen_mode_known)
|
||||
ELSE latest_pure_hydrogen_mode_known
|
||||
END,
|
||||
first_total_mileage_km = ` + upsertSourceMergedFirstTotalSQL + `,
|
||||
latest_total_mileage_km = ` + upsertSourceMergedLatestTotalSQL + `,
|
||||
daily_mileage_km = ` + upsertSourceMergedDailySQL + `,
|
||||
@@ -673,13 +733,14 @@ const selectedSourceIdentitySampleCountSQL = `(
|
||||
|
||||
const projectDailyMileageSQL = `
|
||||
INSERT INTO vehicle_daily_mileage
|
||||
(vin, stat_date, protocol, source_id, daily_mileage_km, latest_total_mileage_km)
|
||||
(vin, stat_date, protocol, source_id, daily_mileage_km, pure_hydrogen_mileage_km, latest_total_mileage_km)
|
||||
SELECT
|
||||
s.vin,
|
||||
s.stat_date,
|
||||
s.protocol,
|
||||
ds.id,
|
||||
s.daily_mileage_km,
|
||||
LEAST(s.pure_hydrogen_mileage_km, s.daily_mileage_km),
|
||||
s.latest_total_mileage_km
|
||||
FROM vehicle_daily_mileage_source s
|
||||
LEFT JOIN vehicle_data_source ds
|
||||
@@ -688,6 +749,7 @@ WHERE s.vin = ?
|
||||
AND s.stat_date = ?
|
||||
AND s.protocol = ?
|
||||
AND s.quality_status = '` + QualityOK + `'
|
||||
AND s.latest_total_mileage_km IS NOT NULL
|
||||
AND s.daily_mileage_km BETWEEN 0 AND (? * GREATEST(1, DATEDIFF(DATE(s.latest_event_time), DATE(s.first_event_time))))
|
||||
AND (ds.id IS NULL OR ds.enabled = 1 OR (
|
||||
COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN'
|
||||
@@ -710,6 +772,7 @@ LIMIT 1
|
||||
ON DUPLICATE KEY UPDATE
|
||||
source_id = VALUES(source_id),
|
||||
daily_mileage_km = VALUES(daily_mileage_km),
|
||||
pure_hydrogen_mileage_km = VALUES(pure_hydrogen_mileage_km),
|
||||
latest_total_mileage_km = VALUES(latest_total_mileage_km),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`
|
||||
|
||||
@@ -176,10 +176,10 @@ func TestUpsertSourceMileageTruncatesSubsecondEventTimesAtDayBoundary(t *testing
|
||||
if len(exec.calls) != 1 {
|
||||
t.Fatalf("exec calls = %d, want 1", len(exec.calls))
|
||||
}
|
||||
if got := exec.calls[0].args[13]; got != previous.Truncate(time.Second) {
|
||||
if got := exec.calls[0].args[15]; got != previous.Truncate(time.Second) {
|
||||
t.Fatalf("first event arg = %v, want %v", got, previous.Truncate(time.Second))
|
||||
}
|
||||
if got := exec.calls[0].args[14]; got != current.Truncate(time.Second) {
|
||||
if got := exec.calls[0].args[16]; got != current.Truncate(time.Second) {
|
||||
t.Fatalf("latest event arg = %v, want %v", got, current.Truncate(time.Second))
|
||||
}
|
||||
}
|
||||
@@ -211,6 +211,27 @@ func TestUpsertSourceMileageUsesEventTimeBoundaries(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertSourceMileageAccumulatesOnlyContinuousPureHydrogenIntervals(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"pure_hydrogen_mileage_km = pure_hydrogen_mileage_km + CASE",
|
||||
"VALUES(latest_event_time) > latest_event_time",
|
||||
"latest_pure_hydrogen_mode_known = 1",
|
||||
"latest_pure_hydrogen_active = 1",
|
||||
"VALUES(latest_pure_hydrogen_mode_known) = 1",
|
||||
"VALUES(latest_pure_hydrogen_active) = 1",
|
||||
"VALUES(latest_total_mileage_km) - latest_total_mileage_km",
|
||||
} {
|
||||
if !strings.Contains(upsertSourceMileageSQL, want) {
|
||||
t.Fatalf("pure hydrogen accumulation SQL missing %q:\n%s", want, upsertSourceMileageSQL)
|
||||
}
|
||||
}
|
||||
pureUpdate := strings.Index(upsertSourceMileageSQL, "pure_hydrogen_mileage_km = pure_hydrogen_mileage_km + CASE")
|
||||
latestTotalUpdate := strings.Index(upsertSourceMileageSQL, "latest_total_mileage_km = CASE")
|
||||
if pureUpdate < 0 || latestTotalUpdate < 0 || pureUpdate > latestTotalUpdate {
|
||||
t.Fatalf("pure hydrogen increment must use the previous odometer before MySQL updates latest_total_mileage_km:\n%s", upsertSourceMileageSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertSourceMileageRechecksMergedDailyRange(t *testing.T) {
|
||||
if maxSelectedDailyMileageKMSQL != "2500" || maxNegativeMileageJitterKMSQL != "1" {
|
||||
t.Fatalf("SQL mileage limits drifted from Go quality constants: selected=%s jitter=%s", maxSelectedDailyMileageKMSQL, maxNegativeMileageJitterKMSQL)
|
||||
@@ -515,6 +536,7 @@ func TestProjectDailyMileageSelectsCandidateAndMarksSource(t *testing.T) {
|
||||
"WHEN 'UNKNOWN' THEN 2",
|
||||
"ds.trust_priority",
|
||||
"s.quality_status = '" + QualityOK + "'",
|
||||
"s.latest_total_mileage_km IS NOT NULL",
|
||||
"COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN'",
|
||||
"AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')",
|
||||
"AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')",
|
||||
|
||||
@@ -225,6 +225,26 @@ func TestDailyMileageSourceSchemaIncludesQueryIndexes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDailyMileageSchemaIncludesPureHydrogenEvidence(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"pure_hydrogen_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0",
|
||||
} {
|
||||
if !strings.Contains(DailyMileageTableSQL, want) {
|
||||
t.Fatalf("daily mileage schema missing %q:\n%s", want, DailyMileageTableSQL)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"pure_hydrogen_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0",
|
||||
"pure_hydrogen_sample_count BIGINT NOT NULL DEFAULT 0",
|
||||
"latest_pure_hydrogen_active TINYINT(1) NOT NULL DEFAULT 0",
|
||||
"latest_pure_hydrogen_mode_known TINYINT(1) NOT NULL DEFAULT 0",
|
||||
} {
|
||||
if !strings.Contains(DailyMileageSourceTableSQL, want) {
|
||||
t.Fatalf("daily mileage source schema missing %q:\n%s", want, DailyMileageSourceTableSQL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func containsStatement(statements []string, fragment string) bool {
|
||||
for _, statement := range statements {
|
||||
if strings.Contains(statement, fragment) {
|
||||
|
||||
Reference in New Issue
Block a user