feat: sync vehicle monitoring and accumulated platform updates

Add monitoring metrics, numeric sorting, report-time display, vehicle number labels, and map layout fixes. Include accumulated mileage, hydrogen, OneOS authentication, open platform, ingestion, deployment, and UI updates.

Exclude temporary files, generated reports, and credentials. Validate frontend 632 tests and production build, API packages, vehicle-map tests, and release installer tests. Gateway suite has one pre-existing date-sensitive identity throttling test failure reproduced on the parent commit.
This commit is contained in:
lingniu
2026-10-26 18:05:59 +08:00
parent 8b32ae2988
commit d6b957dfc8
290 changed files with 28238 additions and 1647 deletions
+7 -6
View File
@@ -66,12 +66,13 @@ func main() {
protocols := []gateway.TCPProtocol{
{
Protocol: envelope.ProtocolGB32960,
Addr: env("GB32960_TCP_ADDR", ":32960"),
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Authenticate: gb32960Authenticator,
Respond: gb32960.AutoResponse,
Protocol: envelope.ProtocolGB32960,
Addr: env("GB32960_TCP_ADDR", ":32960"),
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
ParseForPlatform: gb32960.ParseFrameForPlatform,
Authenticate: gb32960Authenticator,
Respond: gb32960.AutoResponse,
},
{
Protocol: envelope.ProtocolJT808,
@@ -14,6 +14,7 @@ import (
"os"
"sort"
"strconv"
"strings"
"time"
)
@@ -28,10 +29,13 @@ type change struct {
Total string `json:"latest_total_mileage_km"`
LastTime string `json:"latest_event_time"`
Daily string `json:"daily_mileage_km"`
Status string `json:"quality_status"`
Selected int `json:"is_selected"`
} `json:"before"`
Baseline struct {
Total string `json:"total"`
Time string `json:"time"`
Source string `json:"source_key"`
Total string `json:"total"`
Time string `json:"time"`
} `json:"baseline"`
Daily string `json:"daily"`
Status string `json:"status"`
@@ -42,6 +46,7 @@ func main() {
input := flag.String("input", "", "reviewed JSON plan")
apply := flag.Bool("apply", false, "apply plan and projections in one transaction")
manifest := flag.String("backup-manifest", "", "verified backup manifest required for apply")
selectedOnly := flag.Bool("selected-projections-only", false, "preserve final results when only unselected candidates change")
flag.Parse()
data, err := os.ReadFile(*input)
must(err)
@@ -67,24 +72,44 @@ func main() {
affected := map[string]change{}
for _, c := range changes {
b := c.Before
total, stamp, found, err := stats.LookupLatestSourceBaselineBefore(ctx, tx, b.VIN, b.Date, envelope.Protocol(b.Protocol), b.Source)
baselineSource := b.Source
if c.Baseline.Source != "" && c.Baseline.Source != b.Source {
// Historical IP keys and current platform keys may describe the
// same terminal. Verify that identity against the registry before
// accepting an explicitly reviewed alias boundary.
currentCanonical, err := canonicalRepairSource(ctx, tx, b.Protocol, b.Source)
must(err)
baselineCanonical, err := canonicalRepairSource(ctx, tx, b.Protocol, c.Baseline.Source)
must(err)
if currentCanonical != baselineCanonical {
panic("repair cannot mix independent source counters")
}
baselineSource = c.Baseline.Source
}
total, stamp, found, err := stats.LookupLatestSourceBaselineBefore(ctx, tx, b.VIN, b.Date, envelope.Protocol(b.Protocol), baselineSource)
must(err)
expected, err := strconv.ParseFloat(c.Baseline.Total, 64)
must(err)
if !found || math.Abs(total-expected) > 0.0001 || stamp.Format("2006-01-02 15:04:05") != c.Baseline.Time {
panic(fmt.Sprintf("baseline changed: %s %s", b.VIN, b.Date))
}
oldStatus := b.Status
if oldStatus == "" {
oldStatus = stats.QualityOK
}
result, err := tx.ExecContext(ctx, `UPDATE vehicle_daily_mileage_source SET first_total_mileage_km=?,first_event_time=?,daily_mileage_km=?,quality_status=?,quality_reason=?
WHERE vin=? AND stat_date=? AND protocol=? AND source_key=?
AND first_total_mileage_km <=> ? AND first_event_time <=> ? AND latest_total_mileage_km=?
AND latest_event_time=? AND daily_mileage_km=? AND quality_status='OK'`, c.Baseline.Total, c.Baseline.Time, c.Daily, c.Status, c.Reason, b.VIN, b.Date, b.Protocol, b.Source, b.First, b.FirstTime, b.Total, b.LastTime, b.Daily)
AND latest_event_time=? AND daily_mileage_km=? AND quality_status=?`, c.Baseline.Total, c.Baseline.Time, c.Daily, c.Status, c.Reason, b.VIN, b.Date, b.Protocol, b.Source, b.First, b.FirstTime, b.Total, b.LastTime, b.Daily, oldStatus)
must(err)
n, err := result.RowsAffected()
must(err)
if n != 1 {
panic(fmt.Sprintf("concurrent change: %s %s %s", b.VIN, b.Date, b.Source))
}
affected[b.Date+"|"+b.VIN+"|"+b.Protocol] = c
if !*selectedOnly || b.Selected == 1 {
affected[b.Date+"|"+b.VIN+"|"+b.Protocol] = c
}
}
keys := make([]string, 0, len(affected))
for key := range affected {
@@ -98,6 +123,43 @@ func main() {
must(tx.Commit())
must(json.NewEncoder(os.Stdout).Encode(map[string]any{"updatedSources": len(changes), "reprojectedDays": len(keys), "committed": true}))
}
func canonicalRepairSource(ctx context.Context, db stats.Queryer, protocol, key string) (string, error) {
if protocol != "JT808" {
return "", fmt.Errorf("source alias repair is only supported for JT808")
}
if strings.Contains(key, "@PLATFORM:") || strings.HasSuffix(key, "@DIRECT") {
return key, nil
}
at := strings.LastIndex(key, "@")
if at < 0 || !strings.HasPrefix(key, protocol+":") {
return "", fmt.Errorf("unsupported legacy source key")
}
rows, err := db.QueryContext(ctx, `SELECT COALESCE(source_code,''),source_kind
FROM vehicle_data_source WHERE protocol=? AND source_ip=?`, protocol, key[at+1:])
if err != nil {
return "", err
}
defer rows.Close()
if !rows.Next() {
return "", fmt.Errorf("source registry is missing the legacy IP")
}
var code, kind string
if err := rows.Scan(&code, &kind); err != nil {
return "", err
}
if err := rows.Err(); err != nil {
return "", err
}
if kind == "PLATFORM" && strings.TrimSpace(code) != "" {
return key[:at] + "@PLATFORM:" + strings.TrimSpace(code), nil
}
if kind == "DIRECT" {
return key[:at] + "@DIRECT", nil
}
return "", fmt.Errorf("legacy source has no verified platform or direct-terminal identity")
}
func must(err error) {
if err != nil {
panic(err)
@@ -0,0 +1,53 @@
package main
import (
"context"
"testing"
"github.com/DATA-DOG/go-sqlmock"
)
func TestCanonicalRepairSourceRequiresVerifiedRegistryIdentity(t *testing.T) {
for _, tc := range []struct {
name, key, code, kind, want string
missing, wantError bool
}{
{name: "platform", key: "JT808:41456413939@117.132.196.192", code: "guangan_beidou", kind: "PLATFORM", want: "JT808:41456413939@PLATFORM:guangan_beidou"},
{name: "direct", key: "JT808:41456413939@117.132.196.192", kind: "DIRECT", want: "JT808:41456413939@DIRECT"},
{name: "unknown", key: "JT808:41456413939@117.132.196.192", kind: "UNKNOWN", wantError: true},
{name: "missing", key: "JT808:41456413939@117.132.196.192", missing: true, wantError: true},
{name: "platform-without-code", key: "JT808:41456413939@117.132.196.192", kind: "PLATFORM", wantError: true},
} {
t.Run(tc.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
rows := sqlmock.NewRows([]string{"code", "kind"})
if !tc.missing {
rows.AddRow(tc.code, tc.kind)
}
mock.ExpectQuery("SELECT COALESCE").WithArgs("JT808", "117.132.196.192").WillReturnRows(rows)
got, err := canonicalRepairSource(context.Background(), db, "JT808", tc.key)
if (err != nil) != tc.wantError || got != tc.want {
t.Fatalf("got=%q err=%v want=%q wantError=%v", got, err, tc.want, tc.wantError)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
})
}
}
func TestCanonicalRepairSourceKeepsIndependentCounters(t *testing.T) {
for _, key := range []string{"JT808:terminal-a@PLATFORM:g7s", "JT808:terminal-b@PLATFORM:g7s", "JT808:terminal-a@PLATFORM:dongfang_beidou"} {
got, err := canonicalRepairSource(context.Background(), nil, "JT808", key)
if err != nil || got != key {
t.Fatalf("independent identity changed: %s %v", got, err)
}
}
if _, err := canonicalRepairSource(context.Background(), nil, "GB32960", "GB32960:vin@PLATFORM:g7s"); err == nil {
t.Fatal("alias repair must not silently change other protocol rules")
}
}
@@ -0,0 +1,180 @@
// Apply reviewed event-time boundary corrections with a verified backup.
package main
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"math"
"os"
"sort"
"strconv"
"strings"
"time"
_ "github.com/go-sql-driver/mysql"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
type row struct {
VIN string `json:"vin"`
Date string `json:"stat_date"`
Protocol string `json:"protocol"`
Source string `json:"source_key"`
First string `json:"first_total_mileage_km"`
Last string `json:"latest_total_mileage_km"`
FirstTime string `json:"first_event_time"`
LastTime string `json:"latest_event_time"`
Daily string `json:"daily_mileage_km"`
Status string `json:"quality_status"`
Reason string `json:"quality_reason"`
Count int64 `json:"sample_count"`
}
type change struct {
Before row `json:"before"`
After row `json:"after"`
}
func main() {
input := flag.String("input", "", "reviewed event boundary plan")
manifest := flag.String("backup-manifest", "", "verified backup manifest")
apply := flag.Bool("apply", false, "apply changes atomically")
flag.Parse()
data, err := os.ReadFile(*input)
must(err)
var plan []change
must(json.Unmarshal(data, &plan))
loc := time.FixedZone("Asia/Shanghai", 8*3600)
for _, c := range plan {
must(validateChange(c, loc))
}
if !*apply {
fmt.Printf("planned=%d dry_run=true\n", len(plan))
return
}
var backup struct {
File string `json:"backup_file"`
SHA string `json:"backup_sha256"`
PlanSHA string `json:"plan_sha256"`
Verified bool `json:"gzip_and_json_verified"`
}
b, err := os.ReadFile(*manifest)
must(err)
must(json.Unmarshal(b, &backup))
digest := sha256.Sum256(data)
if !backup.Verified || hex.EncodeToString(digest[:]) != backup.PlanSHA {
panic("reviewed plan is not the backed-up plan")
}
compressed, err := os.ReadFile(backup.File)
must(err)
digest = sha256.Sum256(compressed)
if hex.EncodeToString(digest[:]) != backup.SHA {
panic("backup checksum mismatch")
}
db, err := sql.Open("mysql", os.Getenv("MYSQL_DSN"))
must(err)
defer db.Close()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
tx, err := db.BeginTx(ctx, nil)
must(err)
defer tx.Rollback()
sort.Slice(plan, func(i, j int) bool {
a, b := plan[i].Before, plan[j].Before
return a.Date+a.VIN+a.Source < b.Date+b.VIN+b.Source
})
targets := map[string]row{}
for _, c := range plan {
b, a := c.Before, c.After
res, err := tx.ExecContext(ctx, `UPDATE vehicle_daily_mileage_source SET first_total_mileage_km=?,first_event_time=?,latest_total_mileage_km=?,latest_event_time=?,daily_mileage_km=?,quality_status=?,quality_reason=?
WHERE vin=? AND stat_date=? AND protocol='JT808' AND source_key=?
AND first_total_mileage_km <=> ? AND first_event_time <=> ? AND latest_total_mileage_km=? AND latest_event_time=?
AND daily_mileage_km=? AND quality_status=?`, a.First, a.FirstTime, a.Last, a.LastTime, a.Daily, a.Status, a.Reason,
b.VIN, b.Date, b.Source, b.First, b.FirstTime, b.Last, b.LastTime, b.Daily, b.Status)
must(err)
n, err := res.RowsAffected()
must(err)
if n != 1 {
panic(fmt.Sprintf("concurrent change: %s %s %s", b.VIN, b.Date, b.Source))
}
targets[b.Date+"|"+b.VIN] = b
}
keys := make([]string, 0, len(targets))
for k := range targets {
keys = append(keys, k)
}
sort.Strings(keys)
projected, preserved := 0, 0
for _, k := range keys {
b := targets[k]
rows, err := tx.QueryContext(ctx, `SELECT source_ip,COALESCE(quality_reason,'') FROM vehicle_daily_mileage_source WHERE vin=? AND stat_date=? AND protocol='JT808' AND is_selected=1 FOR UPDATE`, b.VIN, b.Date)
must(err)
manual := false
for rows.Next() {
var ip, reason string
must(rows.Scan(&ip, &reason))
manual = manual || strings.HasPrefix(ip, "manual") || strings.HasPrefix(ip, "legacy-mysql") || strings.HasPrefix(reason, "manual")
}
must(rows.Err())
must(rows.Close())
if manual {
preserved++
continue
}
must(stats.ProjectDailyMileage(ctx, tx, b.VIN, b.Date, envelope.ProtocolJT808))
projected++
}
must(tx.Commit())
must(json.NewEncoder(os.Stdout).Encode(map[string]any{"committed": true, "updatedSources": len(plan), "reprojectedDays": projected, "preservedManualDays": preserved}))
}
func validateChange(c change, loc *time.Location) error {
b, a := c.Before, c.After
if b.Protocol != "JT808" || b.VIN == "" || b.Source == "" {
return fmt.Errorf("invalid GPS identity")
}
first, err := strconv.ParseFloat(a.First, 64)
if err != nil {
return err
}
last, err := strconv.ParseFloat(a.Last, 64)
if err != nil {
return err
}
daily, err := strconv.ParseFloat(a.Daily, 64)
if err != nil {
return err
}
firstTime, err := time.ParseInLocation("2006-01-02 15:04:05", a.FirstTime, loc)
if err != nil {
return err
}
lastTime, err := time.ParseInLocation("2006-01-02 15:04:05", a.LastTime, loc)
if err != nil {
return err
}
oldEnd, err := time.ParseInLocation("2006-01-02 15:04:05", b.LastTime, loc)
if err != nil {
return err
}
if lastTime.Before(oldEnd) || firstTime.After(lastTime) || lastTime.Format("2006-01-02") != b.Date || first <= 0 || last <= 0 {
return fmt.Errorf("invalid event-time boundary %s %s", b.VIN, b.Date)
}
sample := stats.SourceMileageSample{DailyKM: last - first, FirstEventTime: firstTime, LatestEventTime: lastTime, QualityStatus: stats.QualityOK, QualityReason: a.Reason}
stats.ApplyMileageQualityRules(&sample)
if math.IsNaN(daily) || math.IsInf(daily, 0) || math.Abs(sample.DailyKM-daily) > 0.0001 || sample.QualityStatus != a.Status {
return fmt.Errorf("incorrect reviewed mileage %s %s", b.VIN, b.Date)
}
return nil
}
func must(err error) {
if err != nil {
panic(err)
}
}
@@ -0,0 +1,27 @@
package main
import (
"testing"
"time"
)
func TestReviewedEventTimeRepairRequiresConsistentForwardTime(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
c := change{Before: row{VIN: "LGAX2BG42H1003112", Date: "2026-10-21", Protocol: "JT808", Source: "JT808:terminal@PLATFORM:forwarder", LastTime: "2026-10-21 08:59:34"},
After: row{First: "44124.600", Last: "44402.100", FirstTime: "2026-10-20 23:29:53", LastTime: "2026-10-21 23:59:55", Daily: "277.500", Status: "OK", Reason: "historical_source_baseline"}}
if err := validateChange(c, loc); err != nil {
t.Fatal(err)
}
for _, mutate := range []func(*change){
func(c *change) { c.After.LastTime = "2026-10-21 08:00:00" },
func(c *change) { c.After.Daily = "353.800" },
func(c *change) { c.Before.Protocol = "GB32960" },
func(c *change) { c.After.FirstTime = "2026-10-22 00:00:00" },
} {
bad := c
mutate(&bad)
if err := validateChange(bad, loc); err == nil {
t.Fatalf("unsafe reviewed plan accepted: %+v", bad)
}
}
}
@@ -75,6 +75,28 @@ func main() {
os.Exit(1)
}
}
if loaded, err := writer.ReloadSourceIdentities(ctx); err != nil {
logger.Error("source identity snapshot load failed", "error", err)
os.Exit(1)
} else {
registry.SetGauge("vehicle_stat_source_identity_snapshot_entries", nil, float64(loaded))
}
go func() {
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if loaded, err := writer.ReloadSourceIdentities(ctx); err != nil {
logger.Warn("source identity snapshot refresh failed; retaining previous snapshot", "error", err)
} else {
registry.SetGauge("vehicle_stat_source_identity_snapshot_entries", nil, float64(loaded))
}
}
}
}()
if cfg.HydrogenCapacitySyncEnabled {
syncHydrogenCapacities(ctx, logger, registry, db, writer, cfg)
} else if loaded, err := writer.ReloadHydrogenTankCapacities(ctx); err != nil {
@@ -0,0 +1,185 @@
package main
import (
"context"
"database/sql"
"fmt"
"log/slog"
"math"
"sort"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
const customerHydrogenAlgorithm = "MODE_INTERVAL_V1"
type customerHydrogenDay struct {
VIN, Date, Protocol, SourceKey string
HydrogenKM, ObservedKM, ClassifiedKM float64
Samples, KnownSamples, Intervals, Rejected int64
First, Last time.Time
PreviousKM float64
}
func addCustomerHydrogenSample(days map[string]*customerHydrogenDay, s stats.MetricSample) {
if s.Protocol != envelope.ProtocolGB32960 && s.Protocol != envelope.ProtocolYutongMQTT {
return
}
if math.IsNaN(s.TotalMileageKM) || math.IsInf(s.TotalMileageKM, 0) || s.TotalMileageKM < 0 {
return
}
key := s.VIN + "/" + s.StatDate + "/" + string(s.Protocol) + "/" + s.SourceKey
d := days[key]
if d == nil {
d = &customerHydrogenDay{VIN: s.VIN, Date: s.StatDate, Protocol: string(s.Protocol), SourceKey: s.SourceKey, First: s.EventTime, Last: s.EventTime, PreviousKM: s.TotalMileageKM, Samples: 1}
if s.PureHydrogenModeKnown {
d.KnownSamples = 1
}
days[key] = d
return
}
// Do not count duplicate or out-of-order reports as extra evidence.
if !s.EventTime.After(d.Last) {
return
}
elapsed := s.EventTime.Sub(d.Last)
delta := s.TotalMileageKM - d.PreviousKM
d.Samples++
if s.PureHydrogenModeKnown {
d.KnownSamples++
}
if delta < -0.2 || delta > math.Max(0.5, elapsed.Hours()*130) || elapsed > 15*time.Minute {
d.Rejected++
} else {
delta = math.Max(0, delta)
d.ObservedKM += delta
d.Intervals++
if s.PureHydrogenModeKnown {
d.ClassifiedKM += delta
if s.PureHydrogenActive {
d.HydrogenKM += delta
}
}
}
d.Last = s.EventTime
d.PreviousKM = s.TotalMileageKM
}
// This path reads raw work-mode evidence and only writes the dedicated customer
// monitor projection. It never resets/reprojects the existing GPS/daily tables.
func runCustomerHydrogenBackfill(ctx context.Context, mysqlDB, td *sql.DB, cfg config) error {
return runCustomerHydrogenBackfillWithLease(ctx, mysqlDB, td, cfg, "")
}
func runCustomerHydrogenBackfillWithLease(ctx context.Context, mysqlDB, td *sql.DB, cfg config, owner string) error {
if cfg.Reset {
return fmt.Errorf("hydrogen-only backfill does not allow reset")
}
if cfg.Limit > 0 && !cfg.DryRun {
return fmt.Errorf("limited scan cannot publish a daily result")
}
for _, p := range cfg.Protocols {
if p != envelope.ProtocolGB32960 && p != envelope.ProtocolYutongMQTT {
return fmt.Errorf("hydrogen-only backfill accepts GB32960,YUTONG_MQTT only")
}
}
rows, err := queryRawFrames(ctx, td, cfg)
if err != nil {
return err
}
defer rows.Close()
days := map[string]*customerHydrogenDay{}
var scanned, skipped int64
for rows.Next() {
scanned++
row, err := scanRawFrame(rows)
if err != nil {
return err
}
text := strings.TrimSpace(row.ParsedJSON)
if isChunkManifest(text) {
text, err = readChunkedPayload(ctx, td, cfg.TDengineDatabase, row.EventID, "parsed_fields")
if err != nil {
return fmt.Errorf("read complete mode evidence: %w", err)
}
}
fields := fieldsForStats(row.Protocol, row.VIN, text)
if len(fields) == 0 && row.Protocol == envelope.ProtocolYutongMQTT {
fields = fieldsForStats(row.Protocol, row.VIN, row.RawText)
}
samples, err := stats.SamplesFromEnvelope(envelope.FrameEnvelope{EventID: row.EventID, Protocol: row.Protocol, MessageID: row.MessageID, VIN: row.VIN, Phone: row.Phone, DeviceID: row.DeviceID, SourceEndpoint: row.SourceEndpoint, EventTimeMS: row.EventTimeMS, ReceivedAtMS: row.ReceivedAtMS, Fields: fields, ParsedFields: fields, ParseStatus: envelope.ParseOK}, cfg.Location)
if err != nil {
return err
}
if len(samples) == 0 {
skipped++
}
for _, s := range samples {
if s.StatDate >= cfg.DateFrom && s.StatDate <= cfg.DateTo {
addCustomerHydrogenSample(days, s)
}
}
if cfg.ProgressEvery > 0 && scanned%cfg.ProgressEvery == 0 {
slog.Info("customer hydrogen scan", "scanned", scanned, "source_days", len(days), "skipped", skipped)
}
}
if err = rows.Err(); err != nil {
return err
}
var known, samples int64
var hydrogen float64
for _, d := range days {
known += d.KnownSamples
samples += d.Samples
hydrogen += d.HydrogenKM
}
slog.Info("customer hydrogen evidence", "scanned", scanned, "samples", samples, "known_mode_samples", known, "source_days", len(days), "hydrogen_km_across_sources", hydrogen, "dry_run", cfg.DryRun)
if cfg.DryRun {
return nil
}
return publishCustomerHydrogenDays(ctx, mysqlDB, days, cfg.DateFrom, owner)
}
func publishCustomerHydrogenDays(ctx context.Context, mysqlDB *sql.DB, days map[string]*customerHydrogenDay, date, owner string) error {
// Publish only after a complete successful scan. Upserts are deterministic;
// failures roll back the entire projection update.
tx, err := mysqlDB.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if owner != "" {
// Fence publication with the same row lock used to claim jobs. A stale
// worker must not overwrite results even before its next heartbeat.
var valid int
if err = tx.QueryRowContext(ctx, `SELECT 1 FROM platform_customer_hydrogen_job WHERE stat_date=? AND lease_owner=? AND status='running' AND lease_until>NOW(3) FOR UPDATE`, date, owner).Scan(&valid); err != nil {
return fmt.Errorf("customer hydrogen publication lease lost: %w", err)
}
}
keys := make([]string, 0, len(days))
for k := range days {
keys = append(keys, k)
}
sort.Strings(keys)
for start := 0; start < len(keys); start += 200 {
end := start + 200
if end > len(keys) {
end = len(keys)
}
values := []string{}
args := []any{}
for _, key := range keys[start:end] {
d := days[key]
values = append(values, "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,NOW(3))")
args = append(args, d.VIN, d.Date, d.Protocol, d.SourceKey, d.HydrogenKM, d.ObservedKM, d.ClassifiedKM, d.Samples, d.KnownSamples, d.Intervals, d.Rejected, d.First, d.Last, customerHydrogenAlgorithm)
}
_, err = tx.ExecContext(ctx, `INSERT INTO platform_customer_hydrogen_daily(vin,stat_date,protocol,source_key,hydrogen_km,observed_km,classified_km,sample_count,known_sample_count,interval_count,rejected_intervals,first_event_time,last_event_time,algorithm_version,calculated_at) VALUES `+strings.Join(values, ",")+` ON DUPLICATE KEY UPDATE hydrogen_km=VALUES(hydrogen_km),observed_km=VALUES(observed_km),classified_km=VALUES(classified_km),sample_count=VALUES(sample_count),known_sample_count=VALUES(known_sample_count),interval_count=VALUES(interval_count),rejected_intervals=VALUES(rejected_intervals),first_event_time=VALUES(first_event_time),last_event_time=VALUES(last_event_time),algorithm_version=VALUES(algorithm_version),calculated_at=VALUES(calculated_at)`, args...)
if err != nil {
return err
}
}
return tx.Commit()
}
@@ -0,0 +1,53 @@
package main
import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
"testing"
"time"
)
func TestCustomerHydrogenIntervalsClassifyElectricZeroAndUnknown(t *testing.T) {
start := time.Date(2026, 9, 29, 8, 0, 0, 0, time.UTC)
days := map[string]*customerHydrogenDay{}
sample := func(min int, km float64, known, active bool) stats.MetricSample {
return stats.MetricSample{VIN: "VIN", StatDate: "2026-09-29", Protocol: envelope.ProtocolGB32960, SourceKey: "source", EventTime: start.Add(time.Duration(min) * time.Minute), TotalMileageKM: km, PureHydrogenModeKnown: known, PureHydrogenActive: active}
}
for _, s := range []stats.MetricSample{sample(0, 100, true, false), sample(5, 105, true, false), sample(10, 110, true, true), sample(15, 115, false, false), sample(15, 115, true, true), sample(60, 130, true, true), sample(65, 140, true, true)} {
addCustomerHydrogenSample(days, s)
}
for _, d := range days {
if d.HydrogenKM != 15 || d.ObservedKM != 25 || d.ClassifiedKM != 20 || d.Samples != 6 || d.KnownSamples != 5 || d.Rejected != 1 {
t.Fatalf("incorrect evidence %+v", d)
}
}
zero := map[string]*customerHydrogenDay{}
addCustomerHydrogenSample(zero, sample(0, 100, true, false))
addCustomerHydrogenSample(zero, sample(5, 105, true, false))
for _, d := range zero {
if d.HydrogenKM != 0 || d.ClassifiedKM != 5 || d.KnownSamples != 2 {
t.Fatal("observed electric-only day not distinguished from missing")
}
}
}
func TestCustomerHydrogenSeparatesSourcesDaysAndRejectsJumps(t *testing.T) {
days := map[string]*customerHydrogenDay{}
s := stats.MetricSample{VIN: "VIN", StatDate: "2026-09-29", Protocol: envelope.ProtocolGB32960, SourceKey: "A", EventTime: time.Now(), TotalMileageKM: 100, PureHydrogenModeKnown: true, PureHydrogenActive: true}
addCustomerHydrogenSample(days, s)
s.EventTime = s.EventTime.Add(time.Minute)
s.TotalMileageKM = 1100
addCustomerHydrogenSample(days, s)
s.SourceKey = "B"
s.TotalMileageKM = 100
addCustomerHydrogenSample(days, s)
s.StatDate = "2026-09-30"
addCustomerHydrogenSample(days, s)
if len(days) != 3 {
t.Fatal("source/day evidence mixed")
}
for _, d := range days {
if d.HydrogenKM != 0 {
t.Fatal("odometer jump counted as hydrogen")
}
}
}
@@ -0,0 +1,113 @@
package main
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
)
func runCustomerHydrogenWorker(parent context.Context, mysqlDB, td *sql.DB, cfg config) error {
ctx, stop := signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM)
defer stop()
maxJobs := envInt("CUSTOMER_HYDROGEN_MAX_JOBS", 0)
completedJobs := 0
for ctx.Err() == nil {
date, owner, err := claimCustomerHydrogenDay(ctx, mysqlDB)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
slog.Error("customer hydrogen claim failed", "error", err)
}
if err != nil {
select {
case <-ctx.Done():
return nil
case <-time.After(5 * time.Second):
continue
}
}
dayCtx, cancel := context.WithTimeout(ctx, 2*time.Hour)
heartbeatDone := make(chan struct{})
go func() {
defer close(heartbeatDone)
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-dayCtx.Done():
return
case <-ticker.C:
result, e := mysqlDB.ExecContext(dayCtx, `UPDATE platform_customer_hydrogen_job SET lease_until=DATE_ADD(NOW(3),INTERVAL 5 MINUTE) WHERE stat_date=? AND lease_owner=? AND status='running'`, date, owner)
if e != nil {
cancel()
return
}
n, e := result.RowsAffected()
if e != nil || n != 1 {
cancel()
return
}
}
}
}()
task := cfg
task.DateFrom = date
task.DateTo = date
task.Limit = 0
task.DryRun = false
task.Reset = false
task.VINs = nil
slog.Info("customer hydrogen job started", "date", date)
err = runCustomerHydrogenBackfillWithLease(dayCtx, mysqlDB, td, task, owner)
cancel()
<-heartbeatDone
status, message := "complete", ""
if err != nil {
status, message = "failed", "原始报文计算失败,请检查任务日志"
slog.Error("customer hydrogen job failed", "date", date, "error", err)
}
// A cancelled/expired worker cannot acknowledge another worker's lease.
finishCtx, finishCancel := context.WithTimeout(context.Background(), 5*time.Second)
_, saveErr := mysqlDB.ExecContext(finishCtx, `UPDATE platform_customer_hydrogen_job SET status=?,error_message=?,completed_at=IF(?='complete',NOW(3),completed_at),lease_until=NULL,lease_owner='' WHERE stat_date=? AND lease_owner=?`, status, message, status, date, owner)
finishCancel()
if saveErr != nil {
slog.Error("customer hydrogen job completion failed", "date", date, "error", saveErr)
}
completedJobs++
if maxJobs > 0 && completedJobs >= maxJobs {
return nil
}
}
return nil
}
func claimCustomerHydrogenDay(ctx context.Context, db *sql.DB) (string, string, error) {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return "", "", err
}
defer tx.Rollback()
var date string
err = tx.QueryRowContext(ctx, `SELECT DATE_FORMAT(stat_date,'%Y-%m-%d') FROM platform_customer_hydrogen_job WHERE status='pending' OR (status='running' AND lease_until<NOW(3)) OR (status='failed' AND attempts<3 AND started_at<DATE_SUB(NOW(3),INTERVAL 1 MINUTE)) ORDER BY requested_at ASC,stat_date DESC LIMIT 1 FOR UPDATE`).Scan(&date)
if err != nil {
return "", "", err
}
var bytes [16]byte
if _, err = rand.Read(bytes[:]); err != nil {
return "", "", err
}
owner := hex.EncodeToString(bytes[:])
_, err = tx.ExecContext(ctx, `UPDATE platform_customer_hydrogen_job SET status='running',started_at=NOW(3),lease_until=DATE_ADD(NOW(3),INTERVAL 5 MINUTE),lease_owner=?,attempts=attempts+1,error_message='' WHERE stat_date=?`, owner, date)
if err != nil {
return "", "", err
}
if err = tx.Commit(); err != nil {
return "", "", err
}
return date, owner, nil
}
@@ -0,0 +1,119 @@
package main
import (
"context"
"database/sql"
"errors"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestCustomerHydrogenJobClaimUsesTransactionAndOwner(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
mock.ExpectBegin()
mock.ExpectQuery("SELECT DATE_FORMAT.*FOR UPDATE").WillReturnRows(sqlmock.NewRows([]string{"date"}).AddRow("2026-09-29"))
mock.ExpectExec("UPDATE platform_customer_hydrogen_job").WithArgs(sqlmock.AnyArg(), "2026-09-29").WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
date, owner, err := claimCustomerHydrogenDay(context.Background(), db)
if err != nil || date != "2026-09-29" || len(owner) != 32 {
t.Fatalf("invalid claim %s %s %v", date, owner, err)
}
if err = mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestCustomerHydrogenBoundedWorkerPublishesOneJobThenExits(t *testing.T) {
t.Setenv("CUSTOMER_HYDROGEN_MAX_JOBS", "1")
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
td, raw, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer td.Close()
mock.ExpectBegin()
mock.ExpectQuery("SELECT DATE_FORMAT.*FOR UPDATE").WillReturnRows(sqlmock.NewRows([]string{"date"}).AddRow("2026-09-29"))
mock.ExpectExec("UPDATE platform_customer_hydrogen_job").WithArgs(sqlmock.AnyArg(), "2026-09-29").WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
raw.ExpectQuery("SELECT").WillReturnRows(sqlmock.NewRows([]string{"event_id"}))
mock.ExpectBegin()
mock.ExpectQuery("SELECT 1.*FOR UPDATE").WithArgs("2026-09-29", sqlmock.AnyArg()).WillReturnRows(sqlmock.NewRows([]string{"valid"}).AddRow(1))
mock.ExpectCommit()
mock.ExpectExec("UPDATE platform_customer_hydrogen_job SET status").WithArgs("complete", "", "complete", "2026-09-29", sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(0, 1))
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
err = runCustomerHydrogenWorker(ctx, db, td, config{Location: time.UTC, TDengineDatabase: "vehicle_ingest", Protocols: []envelope.Protocol{envelope.ProtocolGB32960}})
if err != nil || ctx.Err() != nil {
t.Fatalf("bounded worker did not exit after publication: %v / %v", err, ctx.Err())
}
if err = mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
if err = raw.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestCustomerHydrogenStaleWorkerCannotPublish(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
mock.ExpectBegin()
mock.ExpectQuery("SELECT 1.*lease_until>NOW.*FOR UPDATE").WithArgs("2026-09-29", "old-owner").WillReturnRows(sqlmock.NewRows([]string{"valid"}))
mock.ExpectRollback()
days := map[string]*customerHydrogenDay{"sample": {VIN: "VIN", Date: "2026-09-29", HydrogenKM: 10}}
if err = publishCustomerHydrogenDays(context.Background(), db, days, "2026-09-29", "old-owner"); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("stale worker must fail before writing: %v", err)
}
if err = mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestCustomerHydrogenPublicationLocksCurrentLease(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
mock.ExpectBegin()
mock.ExpectQuery("SELECT 1.*lease_until>NOW.*FOR UPDATE").WithArgs("2026-09-29", "current-owner").WillReturnRows(sqlmock.NewRows([]string{"valid"}).AddRow(1))
mock.ExpectExec("INSERT INTO platform_customer_hydrogen_daily").WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
days := map[string]*customerHydrogenDay{"sample": {VIN: "VIN", Date: "2026-09-29", HydrogenKM: 10}}
if err = publishCustomerHydrogenDays(context.Background(), db, days, "2026-09-29", "current-owner"); err != nil {
t.Fatal(err)
}
if err = mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestCustomerHydrogenIdleWorkerDoesNotCreateJobs(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
mock.ExpectBegin()
mock.ExpectQuery("SELECT DATE_FORMAT.*FOR UPDATE").WillReturnRows(sqlmock.NewRows([]string{"date"}))
mock.ExpectRollback()
if _, _, err = claimCustomerHydrogenDay(context.Background(), db); !errors.Is(err, sql.ErrNoRows) {
t.Fatal(err)
}
if err = mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
@@ -0,0 +1,88 @@
package main
import (
"context"
"database/sql"
"errors"
"fmt"
"log/slog"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
var errNoUsableSourceBoundary = errors.New("source has no usable mileage boundary")
// FIRST/LAST are ordered by TDengine's storage timestamp, which represents
// receipt order. Keep the fast aggregate path, but independently verify both
// ends against extrema of the vehicle event timestamp and fetch the matching
// complete row whenever late delivery has moved either end.
type sourceBoundaryEvidence struct {
vin, phone, deviceID, endpoint string
firstTS, lastTS time.Time
firstParsed, firstRaw string
lastParsed, lastRaw string
count, minimumMS, maximumMS int64
}
func correctSourceEventBoundaries(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, where string, includeFirst bool, e *sourceBoundaryEvidence) error {
_, firstUsable := mileageFromEvidence(protocol, e.vin, e.firstParsed, e.firstRaw, e.firstTS, cfg.Location)
_, lastUsable := mileageFromEvidence(protocol, e.vin, e.lastParsed, e.lastRaw, e.lastTS, cfg.Location)
if e.lastTS.UnixMilli() != e.maximumMS || !lastUsable {
stamp, parsed, raw, err := querySourceEventBoundary(ctx, db, cfg, protocol, where, *e, e.maximumMS, false)
if err != nil {
if errors.Is(err, sql.ErrNoRows) && !firstUsable && !lastUsable {
return errNoUsableSourceBoundary
}
return err
}
slog.Info("corrected last source event boundary", "vin", e.vin, "endpoint", e.endpoint, "storage_last", e.lastTS, "event_last", stamp)
e.lastTS, e.lastParsed, e.lastRaw = stamp, parsed, raw
}
if !includeFirst {
e.firstTS, e.firstParsed, e.firstRaw = e.lastTS, e.lastParsed, e.lastRaw
return nil
}
if e.firstTS.UnixMilli() != e.minimumMS || !firstUsable {
stamp, parsed, raw, err := querySourceEventBoundary(ctx, db, cfg, protocol, where, *e, e.minimumMS, true)
if err != nil {
return err
}
slog.Info("corrected first source event boundary", "vin", e.vin, "endpoint", e.endpoint, "storage_first", e.firstTS, "event_first", stamp)
e.firstTS, e.firstParsed, e.firstRaw = stamp, parsed, raw
}
return nil
}
func querySourceEventBoundary(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, where string, e sourceBoundaryEvidence, eventMS int64, first bool) (time.Time, string, string, error) {
// An equal event timestamp is resolved consistently by newest storage ts.
// Never obtain the timestamp and payload through independent aggregates.
comparison, order := "<=", "DESC"
if first {
comparison, order = ">=", "ASC"
}
query := fmt.Sprintf(`SELECT event_time, parsed_json, raw_text
FROM %s.raw_frames
WHERE %s AND vin = '%s' AND phone = '%s' AND device_id = '%s'
AND source_endpoint = '%s' AND CAST(event_time AS BIGINT) %s %d
ORDER BY event_time %s, ts DESC`, ident(cfg.TDengineDatabase), where, quote(e.vin), quote(e.phone), quote(e.deviceID), quote(e.endpoint), comparison, eventMS, order)
rows, err := db.QueryContext(ctx, query)
if err != nil {
return time.Time{}, "", "", fmt.Errorf("resolve event-time boundary %s/%s/%d: %w", e.vin, e.endpoint, eventMS, err)
}
defer rows.Close()
for rows.Next() {
var stamp time.Time
var parsed, raw string
if err := rows.Scan(&stamp, &parsed, &raw); err != nil {
return time.Time{}, "", "", err
}
if _, ok := mileageFromEvidence(protocol, e.vin, parsed, raw, stamp, cfg.Location); ok {
return stamp, parsed, raw, nil
}
}
if err := rows.Err(); err != nil {
return time.Time{}, "", "", err
}
return time.Time{}, "", "", sql.ErrNoRows
}
@@ -0,0 +1,121 @@
package main
import (
"context"
"fmt"
"math"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestDailySourceEndUsesVehicleEventTimeAfterLateDelivery(t *testing.T) {
for _, tc := range []struct {
vin, phone, ip, code string
lateHour, lateMin int
endHour, endMin int
lateKM, endKM float64
}{
{"LGAX2BG42H1003112", "40409727033", "122.152.221.156", "dongfang_beidou", 8, 59, 23, 59, 44478.4, 44402.1},
{"LA9HE60A0PBAF4002", "13307765803", "115.231.168.135", "xinda", 20, 46, 21, 14, 6194.8, 6195.1},
} {
t.Run(tc.vin, func(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
db.SetMaxOpenConns(1)
loc := time.FixedZone("Asia/Shanghai", 8*3600)
first := time.Date(2026, 10, 21, 0, 0, 0, 0, loc)
late := time.Date(2026, 10, 21, tc.lateHour, tc.lateMin, 0, 0, loc)
last := time.Date(2026, 10, 21, tc.endHour, tc.endMin, 0, 0, loc)
parsed := func(km float64) string { return fmt.Sprintf(`{"jt808.location.total_mileage_km":%f}`, km) }
mock.ExpectQuery("(?s)SELECT vin.*MIN\\(CAST\\(event_time AS BIGINT\\)\\).*MAX\\(CAST\\(event_time AS BIGINT\\)\\)").
WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "first", "first_parsed", "first_raw", "last", "last_parsed", "last_raw", "count", "minimum", "maximum"}).
AddRow(tc.vin, tc.phone, "", tc.ip+":1234", first, parsed(tc.endKM-100), "", late, parsed(tc.lateKM), "", 100, first.UnixMilli(), last.UnixMilli())).RowsWillBeClosed()
mock.ExpectQuery(fmt.Sprintf("(?s)SELECT event_time, parsed_json, raw_text.*phone = '%s'.*CAST\\(event_time AS BIGINT\\) <= %d.*ORDER BY event_time DESC, ts DESC", tc.phone, last.UnixMilli())).
WillReturnRows(sqlmock.NewRows([]string{"event_time", "parsed_json", "raw_text"}).AddRow(last, parsed(tc.endKM), ""))
cfg := config{TDengineDatabase: "lingniu_vehicle_ts", Location: loc}
result, err := queryDailyLastSourceRows(context.Background(), db, cfg, envelope.ProtocolJT808, "2026-10-21")
if err != nil {
t.Fatal(err)
}
got := result[tc.vin][0]
if !got.TS.Equal(last) || math.Abs(got.TotalKM-tc.endKM) > 0.0001 || got.RawSampleCount != 100 {
t.Fatalf("late delivery replaced the vehicle day end: %+v", got)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
})
}
}
func TestDailySourceFirstAlsoUsesVehicleEventTime(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
first := time.Date(2026, 10, 21, 8, 0, 0, 0, loc)
storageFirst := first.Add(time.Hour)
last := first.Add(10 * time.Hour)
mock.ExpectQuery("(?s)SELECT vin.*MIN\\(CAST").WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "first", "first_parsed", "first_raw", "last", "last_parsed", "last_raw", "count", "minimum", "maximum"}).
AddRow("VIN", "phone", "", "1.2.3.4:123", storageFirst, `{"jt808.location.total_mileage_km":101}`, "", last, `{"jt808.location.total_mileage_km":110}`, "", 5, first.UnixMilli(), last.UnixMilli()))
mock.ExpectQuery(fmt.Sprintf("(?s)SELECT event_time.*CAST\\(event_time AS BIGINT\\) >= %d", first.UnixMilli())).WillReturnRows(sqlmock.NewRows([]string{"event", "parsed", "raw"}).AddRow(first, `{"jt808.location.total_mileage_km":100}`, ""))
got, err := queryDailyLastSourceRows(context.Background(), db, config{TDengineDatabase: "db", Location: loc}, envelope.ProtocolJT808, "2026-10-21")
if err != nil {
t.Fatal(err)
}
if got["VIN"][0].FirstTotalKM != 100 || !got["VIN"][0].FirstTS.Equal(first) {
t.Fatalf("wrong first event: %+v", got)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestMissingVehicleEventBoundaryFailsBeforeWriting(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
stamp := time.Date(2026, 10, 20, 20, 0, 0, 0, time.UTC)
mock.ExpectQuery("(?s)SELECT vin.*MAX\\(CAST").WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "last", "parsed", "raw", "count", "maximum"}).AddRow("VIN", "phone", "", "1.2.3.4:123", stamp, `{"jt808.location.total_mileage_km":100}`, "", 2, stamp.Add(time.Hour).UnixMilli()))
mock.ExpectQuery("SELECT event_time").WillReturnRows(sqlmock.NewRows([]string{"event", "parsed", "raw"}))
if _, err := queryPreviousLastSourceRows(context.Background(), db, config{TDengineDatabase: "db", Location: time.UTC}, envelope.ProtocolJT808, "2026-10-21"); err == nil {
t.Fatal("missing event boundary silently accepted")
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestZeroMileageTailDoesNotHideLastUsableVehicleEvent(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
first := time.Date(2026, 10, 21, 8, 0, 0, 0, loc)
last := first.Add(time.Hour)
zero := last.Add(time.Hour)
mock.ExpectQuery("(?s)SELECT vin.*MIN\\(CAST").WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "first", "first_parsed", "first_raw", "last", "last_parsed", "last_raw", "count", "minimum", "maximum"}).AddRow("VIN", "phone", "", "1.2.3.4:123", first, `{"jt808.location.total_mileage_km":100}`, "", zero, `{"jt808.location.total_mileage_km":0}`, "", 3, first.UnixMilli(), zero.UnixMilli()))
mock.ExpectQuery("(?s)SELECT event_time.*ORDER BY event_time DESC, ts DESC").WillReturnRows(sqlmock.NewRows([]string{"event", "parsed", "raw"}).AddRow(zero, `{"jt808.location.total_mileage_km":0}`, "").AddRow(last, `{"jt808.location.total_mileage_km":110}`, ""))
got, err := queryDailyLastSourceRows(context.Background(), db, config{TDengineDatabase: "db", Location: loc}, envelope.ProtocolJT808, "2026-10-21")
if err != nil {
t.Fatal(err)
}
if got["VIN"][0].TotalKM != 110 || !got["VIN"][0].TS.Equal(last) {
t.Fatalf("non-positive frame replaced usable end: %+v", got)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
+112 -32
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
@@ -43,6 +44,7 @@ type config struct {
ListHydrogenVINs bool
ListActiveVINs bool
Location *time.Location
SourceCatalog map[backfillSourceID]stats.SourceIdentity
}
type rawFrameRow struct {
@@ -150,9 +152,27 @@ func main() {
return
}
schemaWriter := stats.NewWriter(mysqlDB, cfg.Location)
if err := schemaWriter.EnsureSchema(ctx); err != nil {
fail("ensure schema", err)
if envBool("CUSTOMER_HYDROGEN_WORKER", false) {
if err := runCustomerHydrogenWorker(ctx, mysqlDB, td, cfg); err != nil {
fail("customer hydrogen worker", err)
}
return
}
if envBool("BACKFILL_HYDROGEN_ONLY", false) {
if err := runCustomerHydrogenBackfill(ctx, mysqlDB, td, cfg); err != nil {
fail("customer hydrogen backfill", err)
}
return
}
cfg.SourceCatalog, err = loadBackfillSourceCatalog(ctx, mysqlDB)
if err != nil {
fail("load backfill source catalog", err)
}
if !cfg.DryRun {
schemaWriter := stats.NewWriter(mysqlDB, cfg.Location)
if err := schemaWriter.EnsureSchema(ctx); err != nil {
fail("ensure schema", err)
}
}
if cfg.Reset && !cfg.DryRun {
deleted, err := resetStats(ctx, mysqlDB, cfg)
@@ -178,6 +198,9 @@ func main() {
if err != nil {
fail("cleanup sub-threshold GPS mileage candidates", err)
}
if cfg.Debug {
logBackfillCandidates(aggregates)
}
var written int64
var normalized int
if !cfg.DryRun {
@@ -250,6 +273,7 @@ func main() {
ParsedFields: fields,
ParseStatus: envelope.ParseOK,
}
env.SourceCode, env.PlatformName, env.SourceKind = backfillSourceMetadata(cfg, row.Protocol, row.SourceEndpoint)
samples, err := stats.SamplesFromEnvelope(env, cfg.Location)
if err != nil {
slog.Warn("skip invalid sample", "event_id", row.EventID, "error", err)
@@ -347,6 +371,23 @@ func activeModeSampleCount(sample stats.MetricSample) int64 {
}
func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg, batchSize int) (int64, error) {
gps, other := map[string]*metricAgg{}, map[string]*metricAgg{}
for key, agg := range aggregates {
if agg.Protocol == envelope.ProtocolJT808 {
gps[key] = agg
} else {
other[key] = agg
}
}
written, err := writeProtectedGPSAggregates(ctx, db, gps)
if err != nil {
return written, err
}
n, err := writeLegacyAggregates(ctx, db, other, batchSize)
return written + n, err
}
func writeLegacyAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg, batchSize int) (int64, error) {
if len(aggregates) == 0 {
return 0, nil
}
@@ -429,11 +470,25 @@ func normalizeHistoricalPlatformSources(ctx context.Context, db *sql.DB, cfg con
continue
}
for _, date := range dates {
count, err := stats.NormalizePlatformSourceMileageForDate(ctx, db, date, protocol)
if len(cfg.VINs) > 0 {
for _, vin := range cfg.VINs {
if err := normalizeProtectedGPSDay(ctx, db, vin, date); err != nil {
return normalized, err
}
normalized++
}
continue
}
vins, err := stats.PlatformSourceMileageLegacyVINs(ctx, db, date, protocol)
if err != nil {
return normalized, err
}
normalized += count
for _, vin := range vins {
if err := normalizeProtectedGPSDay(ctx, db, vin, date); err != nil {
return normalized, err
}
normalized++
}
}
}
return normalized, nil
@@ -592,7 +647,8 @@ FROM vehicle_daily_mileage_source
WHERE stat_date >= ? AND stat_date <= ?
AND quality_reason = ?
AND daily_mileage_km >= 0 AND daily_mileage_km < ?
AND protocol IN (` + strings.Join(placeholders, ",") + `)
AND protocol IN (` + strings.Join(placeholders, ",") + `)`
query = appendBackfillVINScope(query, cfg, "vin") + `
GROUP BY vin, stat_date, protocol
ORDER BY stat_date, protocol, vin`
rows, err := db.QueryContext(ctx, query, args...)
@@ -697,6 +753,9 @@ func addProtocolGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sq
"latitude BETWEEN -90 AND 90",
"NOT (longitude = 0 AND latitude = 0)",
}
if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
where = append(where, predicate)
}
if protocol != envelope.ProtocolJT808 {
where = append(where, "speed_kmh > 0")
}
@@ -1060,13 +1119,17 @@ func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, proto
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
FIRST(event_time), FIRST(parsed_json), FIRST(raw_text),
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*),
MIN(CAST(event_time AS BIGINT)), MAX(CAST(event_time AS BIGINT))
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
return querySourceRows(ctx, db, cfg, protocol, sqlText, true)
return querySourceRows(ctx, db, cfg, protocol, sqlText, true, strings.Join(where, " AND "))
}
func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
@@ -1085,12 +1148,15 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*), MAX(CAST(event_time AS BIGINT))
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
return querySourceRows(ctx, db, cfg, protocol, sqlText, false, strings.Join(where, " AND "))
}
func backfillBeforePredicates(cfg config, eventDateExclusive string) []string {
@@ -1141,6 +1207,7 @@ WHERE l.protocol = ?
if cfg.StationaryCarry {
args = append(args, date)
}
sqlText = appendBackfillVINScope(sqlText, cfg, "l.vin")
rows, err := db.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
@@ -1167,7 +1234,7 @@ WHERE l.protocol = ?
if protocol == envelope.ProtocolYutongMQTT {
deviceID = vin
}
sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
sourceCode, platformName, sourceKind := backfillSourceMetadata(cfg, protocol, sourceEndpoint)
sourceKey := normalizedSourceKeyForSource(string(protocol), "", deviceID, sourceEndpoint, sourceKind, sourceCode)
if sourceKey == "" {
continue
@@ -1194,37 +1261,46 @@ WHERE l.protocol = ?
return out, nil
}
func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool) (map[string][]dailySourceLast, error) {
func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool, boundaryWhere string) (map[string][]dailySourceLast, error) {
rows, err := db.QueryContext(ctx, sqlText)
if err != nil {
return nil, err
}
defer rows.Close()
latestBySource := map[string]dailySourceLast{}
var evidence []sourceBoundaryEvidence
for rows.Next() {
var vin string
var phone string
var deviceID string
var sourceEndpoint string
var firstTS time.Time
var firstParsedJSON string
var firstRawText string
var ts time.Time
var parsedJSON string
var rawText string
var rawSampleCount int64
var e sourceBoundaryEvidence
if includeFirst {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &firstRawText, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
if err := rows.Scan(&e.vin, &e.phone, &e.deviceID, &e.endpoint, &e.firstTS, &e.firstParsed, &e.firstRaw, &e.lastTS, &e.lastParsed, &e.lastRaw, &e.count, &e.minimumMS, &e.maximumMS); err != nil {
return nil, err
}
} else {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
if err := rows.Scan(&e.vin, &e.phone, &e.deviceID, &e.endpoint, &e.lastTS, &e.lastParsed, &e.lastRaw, &e.count, &e.maximumMS); err != nil {
return nil, err
}
firstTS = ts
firstParsedJSON = parsedJSON
firstRawText = rawText
e.minimumMS = e.maximumMS
}
evidence = append(evidence, e)
}
if err := rows.Err(); err != nil {
return nil, err
}
// Release the aggregate cursor before resolving exceptional event-time
// boundaries; this also works with a single TDengine connection.
if err := rows.Close(); err != nil {
return nil, err
}
latestBySource := map[string]dailySourceLast{}
for _, e := range evidence {
if err := correctSourceEventBoundaries(ctx, db, cfg, protocol, boundaryWhere, includeFirst, &e); err != nil {
if errors.Is(err, errNoUsableSourceBoundary) {
continue
}
return nil, err
}
vin, phone, deviceID, sourceEndpoint := e.vin, e.phone, e.deviceID, e.endpoint
firstTS, firstParsedJSON, firstRawText := e.firstTS, e.firstParsed, e.firstRaw
ts, parsedJSON, rawText, rawSampleCount := e.lastTS, e.lastParsed, e.lastRaw, e.count
latestTotalKM, ok := mileageFromEvidence(protocol, vin, parsedJSON, rawText, ts, cfg.Location)
if !ok {
continue
@@ -1235,7 +1311,7 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
firstTotalKM = parsedFirst
}
}
sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
sourceCode, platformName, sourceKind := backfillSourceMetadata(cfg, protocol, sourceEndpoint)
sourceKey := normalizedSourceKeyForSource(string(protocol), phone, deviceID, sourceEndpoint, sourceKind, sourceCode)
if sourceKey == "" {
continue
@@ -1256,9 +1332,10 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
RawSampleCount: rawSampleCount,
}
key := row.VIN + "|" + row.SourceKey
if existing, ok := latestBySource[key]; !ok || row.TS.After(existing.TS) {
latestBySource[key] = row
if existing, ok := latestBySource[key]; ok {
row = mergeDailySourceRows(existing, row)
}
latestBySource[key] = row
}
if err := rows.Err(); err != nil {
return nil, err
@@ -1731,6 +1808,9 @@ func resetStats(ctx context.Context, db *sql.DB, cfg config) (int64, error) {
}
clauses = append(clauses, "protocol IN ("+strings.Join(placeholders, ",")+")")
}
if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
clauses = append(clauses, predicate)
}
var deleted int64
for _, table := range []string{"vehicle_daily_mileage_source", "vehicle_daily_mileage"} {
result, err := db.ExecContext(ctx, "DELETE FROM "+table+" WHERE "+strings.Join(clauses, " AND "), args...)
@@ -302,23 +302,21 @@ func TestBuildLastDiffAggregatesRetainsBoundaryAfterEmptyDays(t *testing.T) {
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
currentRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
})
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MIN_EVENT_MS", "MAX_EVENT_MS"})
}
previousRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
})
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MAX_EVENT_MS"})
}
mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(previousRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", int64(4)))
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", int64(4), dayOneTS.UnixMilli(), dayOneTS.UnixMilli()))
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", int64(6)))
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", int64(6), dayThreeTS.UnixMilli(), dayThreeTS.UnixMilli()))
aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
@@ -632,20 +630,7 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
"LMRKH9AC6R1004108",
"mqtt://yutong/ytforward/shln/3",
firstTS,
`{"yutong_mqtt.data.latitude":"30.0"}`,
`{"data":{"TOTAL_MILEAGE":65422000}}`,
lastTS,
`{"yutong_mqtt.data.latitude":"30.1"}`,
`{"data":{"TOTAL_MILEAGE":65423000}}`,
int64(42),
))
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MIN_EVENT_MS", "MAX_EVENT_MS"}).AddRow("LMRKH9AC6R1004108", "", "LMRKH9AC6R1004108", "mqtt://yutong/ytforward/shln/3", firstTS, `{"yutong_mqtt.data.latitude":"30.0"}`, `{"data":{"TOTAL_MILEAGE":65422000}}`, lastTS, `{"yutong_mqtt.data.latitude":"30.1"}`, `{"data":{"TOTAL_MILEAGE":65423000}}`, int64(42), firstTS.UnixMilli(), lastTS.UnixMilli()))
rows, err := queryDailyLastSourceRows(context.Background(), db, config{
TDengineDatabase: "lingniu_vehicle_ts",
@@ -677,17 +662,7 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
"LMRKH9AC6R1004108",
"mqtt://yutong/ytforward/shln/3",
lastTS,
`{"yutong_mqtt.data.latitude":"30.0"}`,
`{"data":{"TOTAL_MILEAGE":65377000}}`,
int64(31),
))
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MAX_EVENT_MS"}).AddRow("LMRKH9AC6R1004108", "", "LMRKH9AC6R1004108", "mqtt://yutong/ytforward/shln/3", lastTS, `{"yutong_mqtt.data.latitude":"30.0"}`, `{"data":{"TOTAL_MILEAGE":65377000}}`, int64(31), lastTS.UnixMilli()))
rows, err := queryPreviousLastSourceRows(context.Background(), db, config{
TDengineDatabase: "lingniu_vehicle_ts",
@@ -735,17 +710,7 @@ func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone
utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC7R1004098",
"",
"LMRKH9AC7R1004098",
"mqtt://yutong/ytforward/shln/4",
utcInstant,
`{"yutong_mqtt.data.total_mileage":"41249000"}`,
"",
int64(1),
))
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MAX_EVENT_MS"}).AddRow("LMRKH9AC7R1004098", "", "LMRKH9AC7R1004098", "mqtt://yutong/ytforward/shln/4", utcInstant, `{"yutong_mqtt.data.total_mileage":"41249000"}`, "", int64(1), utcInstant.UnixMilli()))
rows, err := queryPreviousLastSourceRows(context.Background(), db, config{
TDengineDatabase: "lingniu_vehicle_ts",
@@ -866,106 +831,6 @@ func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) {
}
}
func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.MatchExpectationsInOrder(true)
projNow := time.Date(2026, 7, 8, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600))
aggregates := map[string]*metricAgg{
"LA9GG64L7PBAF4001|2026-07-08|JT808|JT808:13307765812@115.231.168.135": {
VIN: "LA9GG64L7PBAF4001",
Date: "2026-07-08",
Protocol: envelope.ProtocolJT808,
FirstKM: 4100,
LatestKM: 4101,
Count: 1,
SourceKey: "JT808:13307765812@115.231.168.135",
Phone: "13307765812",
SourceEndpoint: "115.231.168.135:20215",
FirstEventTime: projNow,
LatestEventTime: projNow,
QualityStatus: stats.QualityInvalidDelta,
QualityReason: "outside_daily_range",
},
}
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage_source WHERE vin = \? AND stat_date = \? AND protocol = \?`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage WHERE vin = \? AND stat_date = \? AND protocol = \?`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO vehicle_data_source`).
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
"JT808:13307765812@115.231.168.135",
"115.231.168.135",
"115.231.168.135:20215",
"13307765812",
"",
"",
float64(4100),
float64(4101),
float64(1),
float64(0),
int64(1),
int64(0),
projNow,
projNow,
false,
false,
stats.QualityInvalidDelta,
"outside_daily_range",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectBegin()
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`).
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(2500)).
WillReturnResult(sqlmock.NewResult(1, 0))
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
int64(2500),
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
).
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage`).
WithArgs(
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
"LA9GG64L7PBAF4001",
"2026-07-08",
"JT808",
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
written, err := writeAggregates(context.Background(), db, aggregates, 500)
if err != nil {
t.Fatalf("writeAggregates() error = %v", err)
}
if written != 1 {
t.Fatalf("written = %d, want 1", written)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestWriteAggregatesSkipsBlankSourceBeforeClearingTarget(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
@@ -0,0 +1,239 @@
package main
import (
"context"
"database/sql"
"log/slog"
"sort"
"strings"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
// Rebuild each GPS vehicle-day atomically. Retained RAW is not guaranteed to
// contain every source or a later sample already accepted by the live writer.
// Preserve those durable candidates instead of deleting the complete day.
func writeProtectedGPSAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg) (int64, error) {
groups := map[string][]*metricAgg{}
for _, agg := range aggregates {
if strings.TrimSpace(stats.NormalizeSourceIP(agg.SourceEndpoint)) == "" {
continue
}
key := agg.VIN + "|" + agg.Date
groups[key] = append(groups[key], agg)
}
keys := make([]string, 0, len(groups))
for key := range groups {
keys = append(keys, key)
}
sort.Strings(keys)
var written int64
for _, key := range keys {
group := groups[key]
sort.Slice(group, func(i, j int) bool { return group[i].SourceKey < group[j].SourceKey })
n, err := writeProtectedGPSDay(ctx, db, group)
if err != nil {
return written, err
}
written += n
}
return written, nil
}
func writeProtectedGPSDay(ctx context.Context, db *sql.DB, group []*metricAgg) (int64, error) {
first := group[0]
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return 0, err
}
defer tx.Rollback()
existing, selectedManual, err := lockGPSCandidates(ctx, tx, first.VIN, first.Date)
if err != nil {
return 0, err
}
var written int64
for _, agg := range group {
candidate := backfillCandidate(agg)
if old, ok := existing[candidate.SourceKey]; ok {
if isManualGPSCandidate(old) {
continue
}
candidate = mergeProtectedGPSCandidate(old, candidate)
}
identity := stats.SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: candidate.SourceIP,
SourceEndpoint: candidate.SourceEndpoint, SourceCode: agg.SourceCode, PlatformName: agg.PlatformName, SourceKind: agg.SourceKind}
if candidate.QualityReason != stats.QualityReasonGPSCoordinate {
if err := stats.UpsertDataSource(ctx, tx, identity, candidate.LatestEventTime); err != nil {
return 0, err
}
}
if err := replaceBackfillCandidate(ctx, tx, candidate); err != nil {
return 0, err
}
written++
}
// An explicitly selected imported report is a business override. Keep that
// final result while still correcting the underlying automatic candidates.
if !selectedManual {
if err := stats.ProjectDailyMileage(ctx, tx, first.VIN, first.Date, envelope.ProtocolJT808); err != nil {
return 0, err
}
}
if err := tx.Commit(); err != nil {
return 0, err
}
return written, nil
}
func normalizeProtectedGPSDay(ctx context.Context, db *sql.DB, vin, date string) error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
_, manual, err := lockGPSCandidates(ctx, tx, vin, date)
if err != nil {
return err
}
if manual {
return tx.Commit()
}
if err := stats.NormalizePlatformSourceMileage(ctx, tx, vin, date, envelope.ProtocolJT808); err != nil {
return err
}
if err := stats.ProjectDailyMileage(ctx, tx, vin, date, envelope.ProtocolJT808); err != nil {
return err
}
return tx.Commit()
}
func isManualGPSCandidate(c stats.SourceMileageSample) bool {
return strings.HasPrefix(c.QualityReason, "manual") || strings.HasPrefix(c.SourceIP, "manual") || strings.HasPrefix(c.SourceIP, "legacy-mysql")
}
func backfillCandidate(agg *metricAgg) stats.SourceMileageSample {
c := stats.SourceMileageSample{
VIN: agg.VIN, StatDate: agg.Date, Protocol: agg.Protocol, SourceKey: agg.SourceKey,
SourceIP: stats.NormalizeSourceIP(agg.SourceEndpoint), SourceEndpoint: agg.SourceEndpoint,
Phone: agg.Phone, DeviceID: agg.DeviceID, PlatformName: agg.PlatformName,
FirstTotalKM: agg.FirstKM, LatestTotalKM: agg.LatestKM,
DailyKM: stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM),
PureHydrogenMileageKM: agg.PureHydrogenMileageKM, SampleCount: agg.Count,
PureHydrogenSampleCount: agg.PureHydrogenSampleCount,
FirstEventTime: agg.FirstEventTime, LatestEventTime: agg.LatestEventTime,
LatestPureHydrogenActive: agg.LatestPureHydrogenActive, LatestPureHydrogenModeKnown: agg.LatestPureHydrogenModeKnown,
QualityStatus: agg.QualityStatus, QualityReason: agg.QualityReason,
}
stats.ApplyMileageQualityRules(&c)
return c
}
func mergeProtectedGPSCandidate(old, next stats.SourceMileageSample) stats.SourceMileageSample {
if next.QualityReason == stats.QualityReasonGPSCoordinate || old.QualityReason == stats.QualityReasonGPSCoordinate {
if old.LatestEventTime.After(next.LatestEventTime) {
return old
}
return next
}
if old.LatestEventTime.After(next.LatestEventTime) {
slog.Info("preserved newer durable GPS end", "vin", next.VIN, "date", next.StatDate,
"source_key", next.SourceKey, "raw_end", next.LatestEventTime, "durable_end", old.LatestEventTime)
next.LatestEventTime, next.LatestTotalKM = old.LatestEventTime, old.LatestTotalKM
next.SourceEndpoint = old.SourceEndpoint
next.LatestPureHydrogenActive, next.LatestPureHydrogenModeKnown = old.LatestPureHydrogenActive, old.LatestPureHydrogenModeKnown
}
// Prefer the newest prior-day boundary; otherwise retain the earliest
// current-day sample. Always move the timestamp and odometer together.
if preferDurableGPSFirst(old.FirstEventTime, next.FirstEventTime, next.StatDate) {
next.FirstEventTime, next.FirstTotalKM = old.FirstEventTime, old.FirstTotalKM
}
if old.SampleCount > next.SampleCount {
next.SampleCount = old.SampleCount
}
if old.PureHydrogenSampleCount > next.PureHydrogenSampleCount {
next.PureHydrogenSampleCount = old.PureHydrogenSampleCount
}
if old.PureHydrogenMileageKM > next.PureHydrogenMileageKM {
next.PureHydrogenMileageKM = old.PureHydrogenMileageKM
}
next.DailyKM = stats.DailyMileageFromDayBoundary(next.FirstTotalKM, next.LatestTotalKM)
next.QualityStatus = stats.QualityOK
if next.FirstEventTime.Format("2006-01-02") < next.StatDate {
next.QualityReason = stats.QualityReasonHistorical
} else {
next.QualityReason = stats.QualityReasonCurrentDayFirst
}
stats.ApplyMileageQualityRules(&next)
return next
}
func preferDurableGPSFirst(old, next time.Time, date string) bool {
if old.IsZero() {
return false
}
if next.IsZero() {
return true
}
oldBefore, nextBefore := old.Format("2006-01-02") < date, next.Format("2006-01-02") < date
if oldBefore != nextBefore {
return oldBefore
}
if oldBefore {
return old.After(next)
}
return old.Before(next)
}
func lockGPSCandidates(ctx context.Context, tx *sql.Tx, vin, date string) (map[string]stats.SourceMileageSample, bool, error) {
rows, err := tx.QueryContext(ctx, `SELECT source_key,source_ip,COALESCE(source_endpoint,''),COALESCE(phone,''),COALESCE(device_id,''),COALESCE(platform_name,''),
first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,pure_hydrogen_mileage_km,sample_count,pure_hydrogen_sample_count,
first_event_time,latest_event_time,latest_pure_hydrogen_active,latest_pure_hydrogen_mode_known,quality_status,COALESCE(quality_reason,''),is_selected
FROM vehicle_daily_mileage_source WHERE vin=? AND stat_date=? AND protocol='JT808' FOR UPDATE`, vin, date)
if err != nil {
return nil, false, err
}
defer rows.Close()
out := map[string]stats.SourceMileageSample{}
manual := false
for rows.Next() {
c := stats.SourceMileageSample{VIN: vin, StatDate: date, Protocol: envelope.ProtocolJT808}
var first, last sql.NullTime
var firstKM, lastKM sql.NullFloat64
var selected bool
if err := rows.Scan(&c.SourceKey, &c.SourceIP, &c.SourceEndpoint, &c.Phone, &c.DeviceID, &c.PlatformName,
&firstKM, &lastKM, &c.DailyKM, &c.PureHydrogenMileageKM, &c.SampleCount, &c.PureHydrogenSampleCount,
&first, &last, &c.LatestPureHydrogenActive, &c.LatestPureHydrogenModeKnown, &c.QualityStatus, &c.QualityReason, &selected); err != nil {
return nil, false, err
}
c.FirstEventTime, c.LatestEventTime = first.Time, last.Time
c.FirstTotalKM, c.LatestTotalKM = firstKM.Float64, lastKM.Float64
out[c.SourceKey] = c
if selected && isManualGPSCandidate(c) {
manual = true
}
}
return out, manual, rows.Err()
}
func replaceBackfillCandidate(ctx context.Context, tx *sql.Tx, c stats.SourceMileageSample) error {
_, err := tx.ExecContext(ctx, `INSERT INTO vehicle_daily_mileage_source
(vin,stat_date,protocol,source_key,source_ip,source_endpoint,phone,device_id,platform_name,
first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,pure_hydrogen_mileage_km,
sample_count,pure_hydrogen_sample_count,first_event_time,latest_event_time,
latest_pure_hydrogen_active,latest_pure_hydrogen_mode_known,quality_status,quality_reason)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE source_ip=VALUES(source_ip),source_endpoint=VALUES(source_endpoint),phone=VALUES(phone),device_id=VALUES(device_id),
platform_name=VALUES(platform_name),first_total_mileage_km=VALUES(first_total_mileage_km),latest_total_mileage_km=VALUES(latest_total_mileage_km),
daily_mileage_km=VALUES(daily_mileage_km),pure_hydrogen_mileage_km=VALUES(pure_hydrogen_mileage_km),
sample_count=VALUES(sample_count),pure_hydrogen_sample_count=VALUES(pure_hydrogen_sample_count),
first_event_time=VALUES(first_event_time),latest_event_time=VALUES(latest_event_time),
latest_pure_hydrogen_active=VALUES(latest_pure_hydrogen_active),latest_pure_hydrogen_mode_known=VALUES(latest_pure_hydrogen_mode_known),
quality_status=VALUES(quality_status),quality_reason=VALUES(quality_reason),updated_at=CURRENT_TIMESTAMP`,
c.VIN, c.StatDate, string(c.Protocol), c.SourceKey, c.SourceIP, c.SourceEndpoint, c.Phone, c.DeviceID, c.PlatformName,
c.FirstTotalKM, c.LatestTotalKM, c.DailyKM, c.PureHydrogenMileageKM, c.SampleCount, c.PureHydrogenSampleCount,
c.FirstEventTime.Truncate(time.Second), c.LatestEventTime.Truncate(time.Second),
c.LatestPureHydrogenActive, c.LatestPureHydrogenModeKnown, c.QualityStatus, c.QualityReason)
return err
}
@@ -0,0 +1,132 @@
package main
import (
"context"
"database/sql"
"database/sql/driver"
"errors"
"os"
"reflect"
"testing"
"time"
mysql "github.com/go-sql-driver/mysql"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
// A lost session must never reconnect without its temporary shadow tables.
type oneSessionConnector struct {
driver.Connector
used bool
}
func (c *oneSessionConnector) Connect(ctx context.Context) (driver.Conn, error) {
if c.used {
return nil, errors.New("temporary-table integration session cannot reconnect")
}
c.used = true
return c.Connector.Connect(ctx)
}
func TestProtectedGPSWriteMySQLTemporaryTables(t *testing.T) {
dsn := os.Getenv("BACKFILL_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("set BACKFILL_TEST_MYSQL_DSN for isolated temporary-table integration")
}
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal("invalid integration DSN")
}
cfg.ParseTime = true
cfg.Loc = time.FixedZone("Asia/Shanghai", 8*3600)
connector, err := mysql.NewConnector(cfg)
if err != nil {
t.Fatal("integration connector unavailable")
}
db := sql.OpenDB(&oneSessionConnector{Connector: connector})
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(1)
defer db.Close()
ctx := context.Background()
for _, table := range []string{"vehicle_daily_mileage_source", "vehicle_daily_mileage", "vehicle_data_source"} {
if _, err := db.ExecContext(ctx, "CREATE TEMPORARY TABLE gps_fixture_"+table+" LIKE "+table); err != nil {
t.Fatal(err)
}
if _, err := db.ExecContext(ctx, "CREATE TEMPORARY TABLE "+table+" LIKE gps_fixture_"+table); err != nil {
t.Fatal(err)
}
}
loc := cfg.Loc
prev := time.Date(2026, 10, 20, 23, 59, 0, 0, loc)
end := time.Date(2026, 10, 21, 23, 59, 0, 0, loc)
const vin = "LTESTGPSPROTECT001"
key := "JT808:fixture@PLATFORM:g7s"
if _, err := db.Exec(`INSERT INTO vehicle_daily_mileage_source
(vin,stat_date,protocol,source_key,source_ip,source_endpoint,phone,platform_name,first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,sample_count,first_event_time,latest_event_time,quality_status,quality_reason)
VALUES (?,'2026-10-21','JT808',?,'1.2.3.4','1.2.3.4:1','fixture','G7s',100,120,20,200,?,?,'OK','historical_source_baseline'),
(?,'2026-10-21','JT808','JT808:other@PLATFORM:other','2.3.4.5','2.3.4.5:2','other','Other',100,101,1,3,?,?,'OK','historical_source_baseline')`, vin, key, prev, end, vin, prev, end.Add(-time.Minute)); err != nil {
t.Fatal(err)
}
// Keep the existing manual final selected. This exercises the protected
// write path without MySQL's temporary-table self-join restriction in the
// independent projection query (covered separately by replay verification).
if _, err := db.Exec(`INSERT INTO vehicle_daily_mileage_source (vin,stat_date,protocol,source_key,source_ip,daily_mileage_km,sample_count,quality_status,quality_reason,is_selected)
VALUES (?,'2026-10-21','JT808','manual-report','manual-import',88,1,'OK','manual_gps_report',1)`, vin); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`INSERT INTO vehicle_daily_mileage (vin,stat_date,protocol,source_id,daily_mileage_km) VALUES (?,'2026-10-21','JT808',0,88)`, vin); err != nil {
t.Fatal(err)
}
agg := &metricAgg{VIN: vin, Date: "2026-10-21", Protocol: envelope.ProtocolJT808, SourceKey: key, Phone: "fixture", SourceEndpoint: "1.2.3.4:3", SourceCode: "g7s", SourceKind: "PLATFORM", PlatformName: "G7s", FirstKM: 90, LatestKM: 115, Count: 100, FirstEventTime: prev.Add(-24 * time.Hour), LatestEventTime: end.Add(-time.Hour), QualityStatus: "OK", QualityReason: "historical_source_baseline"}
read := func() []any {
var first, last, daily float64
var count int
var firstTS, lastTS time.Time
if err := db.QueryRow(`SELECT first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,sample_count,first_event_time,latest_event_time FROM vehicle_daily_mileage_source WHERE vin=? AND source_key=?`, vin, key).Scan(&first, &last, &daily, &count, &firstTS, &lastTS); err != nil {
t.Fatal(err)
}
return []any{first, last, daily, count, firstTS, lastTS}
}
var before []any
for run := 0; run < 2; run++ {
if _, err := writeAggregates(ctx, db, map[string]*metricAgg{"fixture": agg}, 500); err != nil {
t.Fatal(err)
}
got := read()
if got[0] != float64(100) || got[1] != float64(120) || got[2] != float64(20) || got[3] != 200 {
t.Fatalf("durable data overwritten: %+v", got)
}
if run == 1 && !reflect.DeepEqual(before, got) {
t.Fatalf("repeated backfill changed business values: before=%+v after=%+v", before, got)
}
before = got
}
var sources int
if err := db.QueryRow("SELECT COUNT(*) FROM vehicle_daily_mileage_source WHERE vin=?", vin).Scan(&sources); err != nil {
t.Fatal(err)
}
if sources != 3 {
t.Fatalf("missing durable alternate source: %d", sources)
}
// Newer event with a lower counter wins by time, not MAX(odometer).
agg.FirstKM, agg.FirstEventTime = 100, prev
agg.LatestKM, agg.LatestEventTime = 119, end.Add(time.Second)
if _, err := writeAggregates(ctx, db, map[string]*metricAgg{"fixture": agg}, 500); err != nil {
t.Fatal(err)
}
if got := read(); got[1] != float64(119) || got[2] != float64(19) {
t.Fatalf("newer event rejected: %+v", got)
}
agg.LatestKM, agg.LatestEventTime = 121, end.Add(2*time.Second)
if _, err := writeAggregates(ctx, db, map[string]*metricAgg{"fixture": agg}, 500); err != nil {
t.Fatal(err)
}
var final float64
if err := db.QueryRow("SELECT daily_mileage_km FROM vehicle_daily_mileage WHERE vin=?", vin).Scan(&final); err != nil {
t.Fatal(err)
}
if final != 88 {
t.Fatalf("manual final overwritten: %v", final)
}
t.Log("temporary-table scenarios passed: incomplete RAW, alternate source retention, repeated backfill, event ordering, manual final; production rows modified=0")
}
@@ -0,0 +1,105 @@
package main
import (
"context"
"errors"
"reflect"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
func TestDurableGPSBoundariesSurviveIncompleteRawAndRepeatedBackfill(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
previous := time.Date(2026, 10, 20, 23, 59, 0, 0, loc)
last := time.Date(2026, 10, 21, 23, 59, 0, 0, loc)
old := stats.SourceMileageSample{VIN: "VIN", StatDate: "2026-10-21", Protocol: envelope.ProtocolJT808, SourceKey: "JT808:phone@PLATFORM:g7s", SourceEndpoint: "1.2.3.4:1", FirstEventTime: previous, FirstTotalKM: 100, LatestEventTime: last, LatestTotalKM: 120, DailyKM: 20, SampleCount: 200, QualityStatus: stats.QualityOK, QualityReason: stats.QualityReasonHistorical}
partial := old
partial.FirstEventTime, partial.FirstTotalKM = previous.Add(-24*time.Hour), 90
partial.LatestEventTime, partial.LatestTotalKM = last.Add(-time.Hour), 115
partial.SampleCount = 100
for i := 0; i < 3; i++ {
got := mergeProtectedGPSCandidate(old, partial)
if !reflect.DeepEqual(got, old) {
t.Fatalf("run %d replaced durable boundaries or double-counted: got=%+v old=%+v", i, got, old)
}
old = got
}
newer := partial
newer.FirstEventTime, newer.FirstTotalKM = previous, 100
newer.LatestEventTime, newer.LatestTotalKM = last.Add(time.Minute), 119
got := mergeProtectedGPSCandidate(old, newer)
if got.LatestTotalKM != 119 || got.DailyKM != 19 {
t.Fatalf("event ordering incorrectly replaced with maximum odometer: %+v", got)
}
}
func TestBackfillCurrentDayFirstBoundaryRetainsEarliestEvent(t *testing.T) {
day := time.Date(2026, 10, 21, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600))
if !preferDurableGPSFirst(day, day.Add(time.Hour), "2026-10-21") {
t.Fatal("earliest current-day event lost")
}
if preferDurableGPSFirst(day.Add(-48*time.Hour), day.Add(-24*time.Hour), "2026-10-21") {
t.Fatal("older previous-day baseline won")
}
}
func gpsLockedRows() *sqlmock.Rows {
return sqlmock.NewRows([]string{"source_key", "source_ip", "endpoint", "phone", "device", "platform", "first_km", "last_km", "daily_km", "hydrogen_km", "count", "hydrogen_count", "first_time", "last_time", "active", "known", "status", "reason", "selected"})
}
func TestGPSBackfillKeepsManualFinalAndRollsBackPartialWrites(t *testing.T) {
for _, fail := range []bool{false, true} {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
stamp := time.Date(2026, 10, 21, 23, 59, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600))
agg := &metricAgg{VIN: "VIN", Date: "2026-10-21", Protocol: envelope.ProtocolJT808, SourceKey: "JT808:phone@PLATFORM:g7s", SourceEndpoint: "1.2.3.4:123", FirstKM: 100, LatestKM: 120, FirstEventTime: stamp.Add(-24 * time.Hour), LatestEventTime: stamp, Count: 5, QualityStatus: stats.QualityOK, QualityReason: stats.QualityReasonHistorical}
mock.ExpectBegin()
mock.ExpectQuery("(?s)SELECT source_key.*FOR UPDATE").WithArgs("VIN", "2026-10-21").WillReturnRows(gpsLockedRows().AddRow("manual-report", "manual-import", "", "", "", "G7s", nil, nil, 88, 0, 1, 0, nil, nil, false, false, "OK", "manual_gps_report", true))
mock.ExpectExec("INSERT INTO vehicle_data_source").WillReturnResult(sqlmock.NewResult(0, 1))
if fail {
mock.ExpectExec("INSERT INTO vehicle_daily_mileage_source").WillReturnError(errors.New("write failed"))
mock.ExpectRollback()
} else {
mock.ExpectExec("(?s)INSERT INTO vehicle_daily_mileage_source.*sample_count=VALUES\\(sample_count\\)").WillReturnResult(sqlmock.NewResult(0, 1))
// No clearing other candidates, source-count increment or final
// projection is permitted over a selected manual report.
mock.ExpectCommit()
}
n, err := writeAggregates(context.Background(), db, map[string]*metricAgg{"target": agg}, 500)
if fail && (err == nil || n != 0) {
t.Fatalf("failed target committed: n=%d err=%v", n, err)
}
if !fail && (err != nil || n != 1) {
t.Fatalf("manual override write: n=%d err=%v", n, err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
db.Close()
}
}
func TestGPSNormalizationDoesNotRewriteSelectedManualEvidence(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
mock.ExpectBegin()
mock.ExpectQuery("(?s)SELECT source_key.*FOR UPDATE").WithArgs("VIN", "2026-07-01").WillReturnRows(gpsLockedRows().AddRow("manual-report", "manual-import", "", "", "", "G7s", nil, nil, 88, 0, 1, 0, nil, nil, false, false, "OK", "manual_gps_report", true))
// Do not normalize/delete the selected manual source, lose its selection
// flag, or reproject over the manual final result.
mock.ExpectCommit()
if err := normalizeProtectedGPSDay(context.Background(), db, "VIN", "2026-07-01"); err != nil {
t.Fatal(err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
@@ -0,0 +1,104 @@
package main
import (
"context"
"database/sql"
"fmt"
"log/slog"
"sort"
"strings"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
type backfillSourceID struct {
protocol envelope.Protocol
ip string
}
// Resolve forwarding endpoints before grouping RAW or looking up durable daily
// boundaries. Normalizing only after writing cannot repair an IP-key lookup
// that has already missed the newer @PLATFORM boundary.
func loadBackfillSourceCatalog(ctx context.Context, db *sql.DB) (map[backfillSourceID]stats.SourceIdentity, error) {
rows, err := db.QueryContext(ctx, `SELECT protocol, source_ip,
COALESCE(source_code, ''), COALESCE(platform_name, ''), source_kind
FROM vehicle_data_source WHERE protocol = 'JT808'`)
if err != nil {
return nil, err
}
defer rows.Close()
catalog := map[backfillSourceID]stats.SourceIdentity{}
for rows.Next() {
var protocol string
var source stats.SourceIdentity
if err := rows.Scan(&protocol, &source.SourceIP, &source.SourceCode, &source.PlatformName, &source.SourceKind); err != nil {
return nil, err
}
source.Protocol = envelope.Protocol(strings.TrimSpace(protocol))
source.SourceIP = stats.NormalizeSourceIP(source.SourceIP)
source.SourceCode = strings.TrimSpace(source.SourceCode)
source.PlatformName = strings.TrimSpace(source.PlatformName)
source.SourceKind = strings.ToUpper(strings.TrimSpace(source.SourceKind))
if source.SourceIP != "" {
catalog[backfillSourceID{source.Protocol, source.SourceIP}] = source
}
}
return catalog, rows.Err()
}
func backfillSourceMetadata(cfg config, protocol envelope.Protocol, endpoint string) (string, string, string) {
if source, ok := cfg.SourceCatalog[backfillSourceID{protocol, stats.NormalizeSourceIP(endpoint)}]; ok {
return source.SourceCode, source.PlatformName, source.SourceKind
}
return knownPlatformSourceMetadata(protocol, endpoint)
}
func mergeDailySourceRows(existing, incoming dailySourceLast) dailySourceLast {
latest := existing
if incoming.TS.After(existing.TS) {
latest = incoming
}
if incoming.FirstTS.Before(existing.FirstTS) {
latest.FirstTS, latest.FirstTotalKM = incoming.FirstTS, incoming.FirstTotalKM
} else {
latest.FirstTS, latest.FirstTotalKM = existing.FirstTS, existing.FirstTotalKM
}
latest.RawSampleCount = existing.RawSampleCount + incoming.RawSampleCount
return latest
}
func backfillVINPredicate(cfg config, column string) string {
if len(cfg.VINs) == 0 {
return ""
}
values := make([]string, 0, len(cfg.VINs))
for _, vin := range cfg.VINs {
values = append(values, "'"+quote(vin)+"'")
}
return column + " IN (" + strings.Join(values, ",") + ")"
}
func appendBackfillVINScope(query string, cfg config, column string) string {
if predicate := backfillVINPredicate(cfg, column); predicate != "" {
query += "\n AND " + predicate
}
return query
}
func logBackfillCandidates(aggregates map[string]*metricAgg) {
keys := make([]string, 0, len(aggregates))
for key := range aggregates {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
a := aggregates[key]
slog.Info("backfill candidate", "vin", a.VIN, "date", a.Date, "protocol", a.Protocol,
"source_key", a.SourceKey, "platform", a.PlatformName,
"baseline_time", a.FirstEventTime.Format("2006-01-02 15:04:05"), "baseline_km", a.FirstKM,
"end_time", a.LatestEventTime.Format("2006-01-02 15:04:05"), "end_km", a.LatestKM,
"daily_km", fmt.Sprintf("%.3f", stats.DailyMileageFromDayBoundary(a.FirstKM, a.LatestKM)),
"samples", a.Count, "quality", a.QualityStatus)
}
}
@@ -0,0 +1,183 @@
package main
import (
"context"
"errors"
"fmt"
"math"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
)
func TestBackfillPlatformBoundarySurvivesRawGapAndRepeatedRuns(t *testing.T) {
for _, tc := range []struct {
vin, terminal, ip, code string
old, previous, end float64
}{
{"LMRKH9AC9R1004099", "64341233471", "115.159.85.149", "g7s", 62749.7, 64432.7, 65136},
{"LMRKH9AC7R1004120", "64341233712", "115.159.85.149", "g7s", 51126.2, 52704.4, 53614.1},
{"LMRKH9AC3R1004129", "64341233795", "122.152.221.156", "dongfang_beidou", 57481.7, 59072.7, 59434.6},
} {
t.Run(tc.vin, func(t *testing.T) {
mysqlDB, mysql, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer mysqlDB.Close()
tdDB, td, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer tdDB.Close()
mysql.ExpectQuery("SELECT protocol, source_ip").WillReturnRows(sqlmock.NewRows([]string{"protocol", "ip", "code", "name", "kind"}).
AddRow("JT808", tc.ip, tc.code, tc.code, "PLATFORM"))
catalog, err := loadBackfillSourceCatalog(context.Background(), mysqlDB)
if err != nil {
t.Fatal(err)
}
loc := time.FixedZone("Asia/Shanghai", 8*3600)
oldTS := time.Date(2026, 10, 16, 15, 26, 0, 0, loc)
previousTS := time.Date(2026, 10, 18, 23, 59, 0, 0, loc)
firstTS := time.Date(2026, 10, 19, 11, 0, 0, 0, loc)
endTS := time.Date(2026, 10, 19, 23, 59, 0, 0, loc)
parsed := func(km float64) string { return fmt.Sprintf(`{"jt808.location.total_mileage_km":%f}`, km) }
key := "JT808:" + tc.terminal + "@PLATFORM:" + tc.code
cfg := config{DateFrom: "2026-10-19", DateTo: "2026-10-19", Protocols: []envelope.Protocol{envelope.ProtocolJT808},
VINs: []string{tc.vin}, TDengineDatabase: "lingniu_vehicle_ts", Location: loc, SourceCatalog: catalog}
// Run twice against the same durable October 18 boundary. A replay must
// neither use October 16 RAW nor subtract today's already-written result.
for run := 0; run < 2; run++ {
td.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-10-19 00:00:00'.*vin IN \\('" + tc.vin + "'\\)").
WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "last", "parsed", "raw", "count", "MAX_EVENT_MS"}).
AddRow(tc.vin, tc.terminal, "", tc.ip+":1000", oldTS, parsed(tc.old), "", 10, oldTS.UnixMilli()))
td.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-10-19 00:00:00'.*vin IN \\('" + tc.vin + "'\\)").
WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "first", "first_parsed", "first_raw", "last", "last_parsed", "last_raw", "count", "MIN_EVENT_MS", "MAX_EVENT_MS"}).
AddRow(tc.vin, tc.terminal, "", tc.ip+":2000", firstTS, parsed(tc.previous+1), "", firstTS.Add(time.Hour), parsed(tc.previous+10), "", 100, firstTS.UnixMilli(), firstTS.Add(time.Hour).UnixMilli()).
AddRow(tc.vin, tc.terminal, "", tc.ip+":3000", firstTS.Add(2*time.Hour), parsed(tc.previous+11), "", endTS, parsed(tc.end), "", 200, firstTS.Add(2*time.Hour).UnixMilli(), endTS.UnixMilli()))
mysql.ExpectQuery("SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source").
WithArgs(tc.vin, "2026-10-19", "JT808", key).
WillReturnRows(sqlmock.NewRows([]string{"total", "time"}).AddRow(tc.previous, previousTS))
aggregates, err := buildLastDiffAggregates(context.Background(), mysqlDB, tdDB, cfg)
if err != nil {
t.Fatal(err)
}
a := aggregates[tc.vin+"|2026-10-19|JT808|"+key]
if len(aggregates) != 1 || a == nil || a.FirstKM != tc.previous || !a.FirstEventTime.Equal(previousTS) || a.Count != 300 {
t.Fatalf("run %d: wrong boundary or connection grouping: %+v", run, a)
}
if math.Abs((a.LatestKM-a.FirstKM)-(tc.end-tc.previous)) > 0.0001 || a.SourceCode != tc.code {
t.Fatalf("run %d: recounted historical mileage: %+v", run, a)
}
}
if err := mysql.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
if err := td.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
})
}
}
func TestBackfillSourceCatalogPreservesIndependentAndUnknownCounters(t *testing.T) {
cfg := config{SourceCatalog: map[backfillSourceID]stats.SourceIdentity{
{envelope.ProtocolJT808, "1.2.3.4"}: {SourceCode: "registered", SourceKind: "PLATFORM"},
{envelope.ProtocolJT808, "2.3.4.5"}: {SourceKind: "DIRECT"},
}}
key := func(protocol envelope.Protocol, terminal, endpoint string) string {
code, _, kind := backfillSourceMetadata(cfg, protocol, endpoint)
return normalizedSourceKeyForSource(string(protocol), terminal, "", endpoint, kind, code)
}
for _, tc := range []struct{ terminal, endpoint, want string }{
{"terminal-a", "1.2.3.4:123", "JT808:terminal-a@PLATFORM:registered"},
{"terminal-b", "1.2.3.4:456", "JT808:terminal-b@PLATFORM:registered"},
{"terminal-a", "2.3.4.5:123", "JT808:terminal-a@DIRECT"},
{"terminal-a", "3.4.5.6:123", "JT808:terminal-a@3.4.5.6"},
} {
if got := key(envelope.ProtocolJT808, tc.terminal, tc.endpoint); got != tc.want {
t.Fatalf("key=%s want=%s", got, tc.want)
}
}
if got := key(envelope.ProtocolGB32960, "terminal-a", "1.2.3.4:123"); strings.Contains(got, "PLATFORM") {
t.Fatalf("JT808 metadata leaked into GB32960: %s", got)
}
}
func TestBackfillSourceCatalogFailureIsNotIgnored(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
mock.ExpectQuery("SELECT protocol, source_ip").WillReturnError(errors.New("database unavailable"))
if _, err := loadBackfillSourceCatalog(context.Background(), db); err == nil {
t.Fatal("missing source catalog must fail before recomputation")
}
}
func TestBackfillVINScopeIncludesCleanupAndRealtimeFallback(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
cfg := config{VINs: []string{"ONLY-THIS-VIN"}, DateFrom: "2026-10-19", DateTo: "2026-10-19", Protocols: []envelope.Protocol{envelope.ProtocolJT808}}
mock.ExpectQuery("(?s)FROM vehicle_daily_mileage_source.*AND vin IN \\('ONLY-THIS-VIN'\\).*GROUP BY").
WithArgs(cfg.DateFrom, cfg.DateTo, stats.QualityReasonGPSCoordinate, stats.GPSMinimumDailyDistanceKM, "JT808").
WillReturnRows(sqlmock.NewRows([]string{"vin", "date", "protocol", "count"}))
if _, _, err := cleanupSubthresholdGPSMileageCandidates(context.Background(), db, cfg, true); err != nil {
t.Fatal(err)
}
mock.ExpectQuery("(?s)FROM vehicle_realtime_location.*AND l.vin IN \\('ONLY-THIS-VIN'\\)").
WithArgs("YUTONG_MQTT", cfg.DateFrom, cfg.DateFrom).
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "time", "total"}))
if _, err := queryRealtimeLocationLastRows(context.Background(), db, cfg, envelope.ProtocolYutongMQTT, cfg.DateFrom); err != nil {
t.Fatal(err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestBackfillScopedNormalizationDoesNotScanOtherVehicles(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer db.Close()
vin, date := "ONLY-THIS-VIN", "2026-10-19"
mock.ExpectBegin()
mock.ExpectQuery("(?s)SELECT source_key.*FOR UPDATE").WithArgs(vin, date).WillReturnRows(gpsLockedRows())
mock.ExpectExec("INSERT INTO vehicle_daily_mileage_source").WithArgs(vin, date, "JT808").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("DELETE s").WithArgs(vin, date, "JT808").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("INSERT INTO vehicle_daily_mileage").WithArgs(vin, date, "JT808", int64(2500)).WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("UPDATE vehicle_daily_mileage_source s").WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("DELETE FROM vehicle_daily_mileage").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectCommit()
_, err = normalizeHistoricalPlatformSources(context.Background(), db, config{VINs: []string{vin}, DateFrom: date, DateTo: date, Protocols: []envelope.Protocol{envelope.ProtocolJT808}})
if err != nil {
t.Fatal(err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestMergeDailySourceRowsIsIndependentOfConnectionOrder(t *testing.T) {
first := dailySourceLast{FirstTS: time.Unix(100, 0), TS: time.Unix(200, 0), FirstTotalKM: 10, TotalKM: 20, RawSampleCount: 4}
last := dailySourceLast{FirstTS: time.Unix(300, 0), TS: time.Unix(400, 0), FirstTotalKM: 30, TotalKM: 40, RawSampleCount: 6}
for _, merged := range []dailySourceLast{mergeDailySourceRows(first, last), mergeDailySourceRows(last, first)} {
if merged.FirstTotalKM != 10 || merged.TotalKM != 40 || merged.RawSampleCount != 10 {
t.Fatalf("lost connection boundary: %+v", merged)
}
}
// Scope quoting is shared by SQL paths, including escaped input.
if got := backfillVINPredicate(config{VINs: []string{"A'B"}}, "vin"); got != "vin IN ('A''B')" {
t.Fatalf("unexpected predicate: %s", got)
}
}