lingniu
d6b957dfc8
Add monitoring metrics, numeric sorting, report-time display, vehicle number labels, and map layout fixes. Include accumulated mileage, hydrogen, OneOS authentication, open platform, ingestion, deployment, and UI updates. Exclude temporary files, generated reports, and credentials. Validate frontend 632 tests and production build, API packages, vehicle-map tests, and release installer tests. Gateway suite has one pre-existing date-sensitive identity throttling test failure reproduced on the parent commit.
105 lines
3.6 KiB
Go
105 lines
3.6 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"log/slog"
|
|
"sort"
|
|
"strings"
|
|
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
|
)
|
|
|
|
type backfillSourceID struct {
|
|
protocol envelope.Protocol
|
|
ip string
|
|
}
|
|
|
|
// Resolve forwarding endpoints before grouping RAW or looking up durable daily
|
|
// boundaries. Normalizing only after writing cannot repair an IP-key lookup
|
|
// that has already missed the newer @PLATFORM boundary.
|
|
func loadBackfillSourceCatalog(ctx context.Context, db *sql.DB) (map[backfillSourceID]stats.SourceIdentity, error) {
|
|
rows, err := db.QueryContext(ctx, `SELECT protocol, source_ip,
|
|
COALESCE(source_code, ''), COALESCE(platform_name, ''), source_kind
|
|
FROM vehicle_data_source WHERE protocol = 'JT808'`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
catalog := map[backfillSourceID]stats.SourceIdentity{}
|
|
for rows.Next() {
|
|
var protocol string
|
|
var source stats.SourceIdentity
|
|
if err := rows.Scan(&protocol, &source.SourceIP, &source.SourceCode, &source.PlatformName, &source.SourceKind); err != nil {
|
|
return nil, err
|
|
}
|
|
source.Protocol = envelope.Protocol(strings.TrimSpace(protocol))
|
|
source.SourceIP = stats.NormalizeSourceIP(source.SourceIP)
|
|
source.SourceCode = strings.TrimSpace(source.SourceCode)
|
|
source.PlatformName = strings.TrimSpace(source.PlatformName)
|
|
source.SourceKind = strings.ToUpper(strings.TrimSpace(source.SourceKind))
|
|
if source.SourceIP != "" {
|
|
catalog[backfillSourceID{source.Protocol, source.SourceIP}] = source
|
|
}
|
|
}
|
|
return catalog, rows.Err()
|
|
}
|
|
|
|
func backfillSourceMetadata(cfg config, protocol envelope.Protocol, endpoint string) (string, string, string) {
|
|
if source, ok := cfg.SourceCatalog[backfillSourceID{protocol, stats.NormalizeSourceIP(endpoint)}]; ok {
|
|
return source.SourceCode, source.PlatformName, source.SourceKind
|
|
}
|
|
return knownPlatformSourceMetadata(protocol, endpoint)
|
|
}
|
|
|
|
func mergeDailySourceRows(existing, incoming dailySourceLast) dailySourceLast {
|
|
latest := existing
|
|
if incoming.TS.After(existing.TS) {
|
|
latest = incoming
|
|
}
|
|
if incoming.FirstTS.Before(existing.FirstTS) {
|
|
latest.FirstTS, latest.FirstTotalKM = incoming.FirstTS, incoming.FirstTotalKM
|
|
} else {
|
|
latest.FirstTS, latest.FirstTotalKM = existing.FirstTS, existing.FirstTotalKM
|
|
}
|
|
latest.RawSampleCount = existing.RawSampleCount + incoming.RawSampleCount
|
|
return latest
|
|
}
|
|
|
|
func backfillVINPredicate(cfg config, column string) string {
|
|
if len(cfg.VINs) == 0 {
|
|
return ""
|
|
}
|
|
values := make([]string, 0, len(cfg.VINs))
|
|
for _, vin := range cfg.VINs {
|
|
values = append(values, "'"+quote(vin)+"'")
|
|
}
|
|
return column + " IN (" + strings.Join(values, ",") + ")"
|
|
}
|
|
|
|
func appendBackfillVINScope(query string, cfg config, column string) string {
|
|
if predicate := backfillVINPredicate(cfg, column); predicate != "" {
|
|
query += "\n AND " + predicate
|
|
}
|
|
return query
|
|
}
|
|
|
|
func logBackfillCandidates(aggregates map[string]*metricAgg) {
|
|
keys := make([]string, 0, len(aggregates))
|
|
for key := range aggregates {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, key := range keys {
|
|
a := aggregates[key]
|
|
slog.Info("backfill candidate", "vin", a.VIN, "date", a.Date, "protocol", a.Protocol,
|
|
"source_key", a.SourceKey, "platform", a.PlatformName,
|
|
"baseline_time", a.FirstEventTime.Format("2006-01-02 15:04:05"), "baseline_km", a.FirstKM,
|
|
"end_time", a.LatestEventTime.Format("2006-01-02 15:04:05"), "end_km", a.LatestKM,
|
|
"daily_km", fmt.Sprintf("%.3f", stats.DailyMileageFromDayBoundary(a.FirstKM, a.LatestKM)),
|
|
"samples", a.Count, "quality", a.QualityStatus)
|
|
}
|
|
}
|