feat: sync vehicle monitoring and accumulated platform updates
Add monitoring metrics, numeric sorting, report-time display, vehicle number labels, and map layout fixes. Include accumulated mileage, hydrogen, OneOS authentication, open platform, ingestion, deployment, and UI updates. Exclude temporary files, generated reports, and credentials. Validate frontend 632 tests and production build, API packages, vehicle-map tests, and release installer tests. Gateway suite has one pre-existing date-sensitive identity throttling test failure reproduced on the parent commit.
This commit is contained in:
@@ -66,12 +66,13 @@ func main() {
|
||||
|
||||
protocols := []gateway.TCPProtocol{
|
||||
{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: env("GB32960_TCP_ADDR", ":32960"),
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
Authenticate: gb32960Authenticator,
|
||||
Respond: gb32960.AutoResponse,
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
Addr: env("GB32960_TCP_ADDR", ":32960"),
|
||||
Extract: gb32960.ExtractFrames,
|
||||
Parse: gb32960.ParseFrame,
|
||||
ParseForPlatform: gb32960.ParseFrameForPlatform,
|
||||
Authenticate: gb32960Authenticator,
|
||||
Respond: gb32960.AutoResponse,
|
||||
},
|
||||
{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -28,10 +29,13 @@ type change struct {
|
||||
Total string `json:"latest_total_mileage_km"`
|
||||
LastTime string `json:"latest_event_time"`
|
||||
Daily string `json:"daily_mileage_km"`
|
||||
Status string `json:"quality_status"`
|
||||
Selected int `json:"is_selected"`
|
||||
} `json:"before"`
|
||||
Baseline struct {
|
||||
Total string `json:"total"`
|
||||
Time string `json:"time"`
|
||||
Source string `json:"source_key"`
|
||||
Total string `json:"total"`
|
||||
Time string `json:"time"`
|
||||
} `json:"baseline"`
|
||||
Daily string `json:"daily"`
|
||||
Status string `json:"status"`
|
||||
@@ -42,6 +46,7 @@ func main() {
|
||||
input := flag.String("input", "", "reviewed JSON plan")
|
||||
apply := flag.Bool("apply", false, "apply plan and projections in one transaction")
|
||||
manifest := flag.String("backup-manifest", "", "verified backup manifest required for apply")
|
||||
selectedOnly := flag.Bool("selected-projections-only", false, "preserve final results when only unselected candidates change")
|
||||
flag.Parse()
|
||||
data, err := os.ReadFile(*input)
|
||||
must(err)
|
||||
@@ -67,24 +72,44 @@ func main() {
|
||||
affected := map[string]change{}
|
||||
for _, c := range changes {
|
||||
b := c.Before
|
||||
total, stamp, found, err := stats.LookupLatestSourceBaselineBefore(ctx, tx, b.VIN, b.Date, envelope.Protocol(b.Protocol), b.Source)
|
||||
baselineSource := b.Source
|
||||
if c.Baseline.Source != "" && c.Baseline.Source != b.Source {
|
||||
// Historical IP keys and current platform keys may describe the
|
||||
// same terminal. Verify that identity against the registry before
|
||||
// accepting an explicitly reviewed alias boundary.
|
||||
currentCanonical, err := canonicalRepairSource(ctx, tx, b.Protocol, b.Source)
|
||||
must(err)
|
||||
baselineCanonical, err := canonicalRepairSource(ctx, tx, b.Protocol, c.Baseline.Source)
|
||||
must(err)
|
||||
if currentCanonical != baselineCanonical {
|
||||
panic("repair cannot mix independent source counters")
|
||||
}
|
||||
baselineSource = c.Baseline.Source
|
||||
}
|
||||
total, stamp, found, err := stats.LookupLatestSourceBaselineBefore(ctx, tx, b.VIN, b.Date, envelope.Protocol(b.Protocol), baselineSource)
|
||||
must(err)
|
||||
expected, err := strconv.ParseFloat(c.Baseline.Total, 64)
|
||||
must(err)
|
||||
if !found || math.Abs(total-expected) > 0.0001 || stamp.Format("2006-01-02 15:04:05") != c.Baseline.Time {
|
||||
panic(fmt.Sprintf("baseline changed: %s %s", b.VIN, b.Date))
|
||||
}
|
||||
oldStatus := b.Status
|
||||
if oldStatus == "" {
|
||||
oldStatus = stats.QualityOK
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE vehicle_daily_mileage_source SET first_total_mileage_km=?,first_event_time=?,daily_mileage_km=?,quality_status=?,quality_reason=?
|
||||
WHERE vin=? AND stat_date=? AND protocol=? AND source_key=?
|
||||
AND first_total_mileage_km <=> ? AND first_event_time <=> ? AND latest_total_mileage_km=?
|
||||
AND latest_event_time=? AND daily_mileage_km=? AND quality_status='OK'`, c.Baseline.Total, c.Baseline.Time, c.Daily, c.Status, c.Reason, b.VIN, b.Date, b.Protocol, b.Source, b.First, b.FirstTime, b.Total, b.LastTime, b.Daily)
|
||||
AND latest_event_time=? AND daily_mileage_km=? AND quality_status=?`, c.Baseline.Total, c.Baseline.Time, c.Daily, c.Status, c.Reason, b.VIN, b.Date, b.Protocol, b.Source, b.First, b.FirstTime, b.Total, b.LastTime, b.Daily, oldStatus)
|
||||
must(err)
|
||||
n, err := result.RowsAffected()
|
||||
must(err)
|
||||
if n != 1 {
|
||||
panic(fmt.Sprintf("concurrent change: %s %s %s", b.VIN, b.Date, b.Source))
|
||||
}
|
||||
affected[b.Date+"|"+b.VIN+"|"+b.Protocol] = c
|
||||
if !*selectedOnly || b.Selected == 1 {
|
||||
affected[b.Date+"|"+b.VIN+"|"+b.Protocol] = c
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(affected))
|
||||
for key := range affected {
|
||||
@@ -98,6 +123,43 @@ func main() {
|
||||
must(tx.Commit())
|
||||
must(json.NewEncoder(os.Stdout).Encode(map[string]any{"updatedSources": len(changes), "reprojectedDays": len(keys), "committed": true}))
|
||||
}
|
||||
|
||||
func canonicalRepairSource(ctx context.Context, db stats.Queryer, protocol, key string) (string, error) {
|
||||
if protocol != "JT808" {
|
||||
return "", fmt.Errorf("source alias repair is only supported for JT808")
|
||||
}
|
||||
if strings.Contains(key, "@PLATFORM:") || strings.HasSuffix(key, "@DIRECT") {
|
||||
return key, nil
|
||||
}
|
||||
at := strings.LastIndex(key, "@")
|
||||
if at < 0 || !strings.HasPrefix(key, protocol+":") {
|
||||
return "", fmt.Errorf("unsupported legacy source key")
|
||||
}
|
||||
rows, err := db.QueryContext(ctx, `SELECT COALESCE(source_code,''),source_kind
|
||||
FROM vehicle_data_source WHERE protocol=? AND source_ip=?`, protocol, key[at+1:])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
if !rows.Next() {
|
||||
return "", fmt.Errorf("source registry is missing the legacy IP")
|
||||
}
|
||||
var code, kind string
|
||||
if err := rows.Scan(&code, &kind); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if kind == "PLATFORM" && strings.TrimSpace(code) != "" {
|
||||
return key[:at] + "@PLATFORM:" + strings.TrimSpace(code), nil
|
||||
}
|
||||
if kind == "DIRECT" {
|
||||
return key[:at] + "@DIRECT", nil
|
||||
}
|
||||
return "", fmt.Errorf("legacy source has no verified platform or direct-terminal identity")
|
||||
}
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
func TestCanonicalRepairSourceRequiresVerifiedRegistryIdentity(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, key, code, kind, want string
|
||||
missing, wantError bool
|
||||
}{
|
||||
{name: "platform", key: "JT808:41456413939@117.132.196.192", code: "guangan_beidou", kind: "PLATFORM", want: "JT808:41456413939@PLATFORM:guangan_beidou"},
|
||||
{name: "direct", key: "JT808:41456413939@117.132.196.192", kind: "DIRECT", want: "JT808:41456413939@DIRECT"},
|
||||
{name: "unknown", key: "JT808:41456413939@117.132.196.192", kind: "UNKNOWN", wantError: true},
|
||||
{name: "missing", key: "JT808:41456413939@117.132.196.192", missing: true, wantError: true},
|
||||
{name: "platform-without-code", key: "JT808:41456413939@117.132.196.192", kind: "PLATFORM", wantError: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
rows := sqlmock.NewRows([]string{"code", "kind"})
|
||||
if !tc.missing {
|
||||
rows.AddRow(tc.code, tc.kind)
|
||||
}
|
||||
mock.ExpectQuery("SELECT COALESCE").WithArgs("JT808", "117.132.196.192").WillReturnRows(rows)
|
||||
got, err := canonicalRepairSource(context.Background(), db, "JT808", tc.key)
|
||||
if (err != nil) != tc.wantError || got != tc.want {
|
||||
t.Fatalf("got=%q err=%v want=%q wantError=%v", got, err, tc.want, tc.wantError)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalRepairSourceKeepsIndependentCounters(t *testing.T) {
|
||||
for _, key := range []string{"JT808:terminal-a@PLATFORM:g7s", "JT808:terminal-b@PLATFORM:g7s", "JT808:terminal-a@PLATFORM:dongfang_beidou"} {
|
||||
got, err := canonicalRepairSource(context.Background(), nil, "JT808", key)
|
||||
if err != nil || got != key {
|
||||
t.Fatalf("independent identity changed: %s %v", got, err)
|
||||
}
|
||||
}
|
||||
if _, err := canonicalRepairSource(context.Background(), nil, "GB32960", "GB32960:vin@PLATFORM:g7s"); err == nil {
|
||||
t.Fatal("alias repair must not silently change other protocol rules")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
// Apply reviewed event-time boundary corrections with a verified backup.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
type row struct {
|
||||
VIN string `json:"vin"`
|
||||
Date string `json:"stat_date"`
|
||||
Protocol string `json:"protocol"`
|
||||
Source string `json:"source_key"`
|
||||
First string `json:"first_total_mileage_km"`
|
||||
Last string `json:"latest_total_mileage_km"`
|
||||
FirstTime string `json:"first_event_time"`
|
||||
LastTime string `json:"latest_event_time"`
|
||||
Daily string `json:"daily_mileage_km"`
|
||||
Status string `json:"quality_status"`
|
||||
Reason string `json:"quality_reason"`
|
||||
Count int64 `json:"sample_count"`
|
||||
}
|
||||
type change struct {
|
||||
Before row `json:"before"`
|
||||
After row `json:"after"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
input := flag.String("input", "", "reviewed event boundary plan")
|
||||
manifest := flag.String("backup-manifest", "", "verified backup manifest")
|
||||
apply := flag.Bool("apply", false, "apply changes atomically")
|
||||
flag.Parse()
|
||||
data, err := os.ReadFile(*input)
|
||||
must(err)
|
||||
var plan []change
|
||||
must(json.Unmarshal(data, &plan))
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
for _, c := range plan {
|
||||
must(validateChange(c, loc))
|
||||
}
|
||||
if !*apply {
|
||||
fmt.Printf("planned=%d dry_run=true\n", len(plan))
|
||||
return
|
||||
}
|
||||
var backup struct {
|
||||
File string `json:"backup_file"`
|
||||
SHA string `json:"backup_sha256"`
|
||||
PlanSHA string `json:"plan_sha256"`
|
||||
Verified bool `json:"gzip_and_json_verified"`
|
||||
}
|
||||
b, err := os.ReadFile(*manifest)
|
||||
must(err)
|
||||
must(json.Unmarshal(b, &backup))
|
||||
digest := sha256.Sum256(data)
|
||||
if !backup.Verified || hex.EncodeToString(digest[:]) != backup.PlanSHA {
|
||||
panic("reviewed plan is not the backed-up plan")
|
||||
}
|
||||
compressed, err := os.ReadFile(backup.File)
|
||||
must(err)
|
||||
digest = sha256.Sum256(compressed)
|
||||
if hex.EncodeToString(digest[:]) != backup.SHA {
|
||||
panic("backup checksum mismatch")
|
||||
}
|
||||
db, err := sql.Open("mysql", os.Getenv("MYSQL_DSN"))
|
||||
must(err)
|
||||
defer db.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
must(err)
|
||||
defer tx.Rollback()
|
||||
sort.Slice(plan, func(i, j int) bool {
|
||||
a, b := plan[i].Before, plan[j].Before
|
||||
return a.Date+a.VIN+a.Source < b.Date+b.VIN+b.Source
|
||||
})
|
||||
targets := map[string]row{}
|
||||
for _, c := range plan {
|
||||
b, a := c.Before, c.After
|
||||
res, err := tx.ExecContext(ctx, `UPDATE vehicle_daily_mileage_source SET first_total_mileage_km=?,first_event_time=?,latest_total_mileage_km=?,latest_event_time=?,daily_mileage_km=?,quality_status=?,quality_reason=?
|
||||
WHERE vin=? AND stat_date=? AND protocol='JT808' AND source_key=?
|
||||
AND first_total_mileage_km <=> ? AND first_event_time <=> ? AND latest_total_mileage_km=? AND latest_event_time=?
|
||||
AND daily_mileage_km=? AND quality_status=?`, a.First, a.FirstTime, a.Last, a.LastTime, a.Daily, a.Status, a.Reason,
|
||||
b.VIN, b.Date, b.Source, b.First, b.FirstTime, b.Last, b.LastTime, b.Daily, b.Status)
|
||||
must(err)
|
||||
n, err := res.RowsAffected()
|
||||
must(err)
|
||||
if n != 1 {
|
||||
panic(fmt.Sprintf("concurrent change: %s %s %s", b.VIN, b.Date, b.Source))
|
||||
}
|
||||
targets[b.Date+"|"+b.VIN] = b
|
||||
}
|
||||
keys := make([]string, 0, len(targets))
|
||||
for k := range targets {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
projected, preserved := 0, 0
|
||||
for _, k := range keys {
|
||||
b := targets[k]
|
||||
rows, err := tx.QueryContext(ctx, `SELECT source_ip,COALESCE(quality_reason,'') FROM vehicle_daily_mileage_source WHERE vin=? AND stat_date=? AND protocol='JT808' AND is_selected=1 FOR UPDATE`, b.VIN, b.Date)
|
||||
must(err)
|
||||
manual := false
|
||||
for rows.Next() {
|
||||
var ip, reason string
|
||||
must(rows.Scan(&ip, &reason))
|
||||
manual = manual || strings.HasPrefix(ip, "manual") || strings.HasPrefix(ip, "legacy-mysql") || strings.HasPrefix(reason, "manual")
|
||||
}
|
||||
must(rows.Err())
|
||||
must(rows.Close())
|
||||
if manual {
|
||||
preserved++
|
||||
continue
|
||||
}
|
||||
must(stats.ProjectDailyMileage(ctx, tx, b.VIN, b.Date, envelope.ProtocolJT808))
|
||||
projected++
|
||||
}
|
||||
must(tx.Commit())
|
||||
must(json.NewEncoder(os.Stdout).Encode(map[string]any{"committed": true, "updatedSources": len(plan), "reprojectedDays": projected, "preservedManualDays": preserved}))
|
||||
}
|
||||
|
||||
func validateChange(c change, loc *time.Location) error {
|
||||
b, a := c.Before, c.After
|
||||
if b.Protocol != "JT808" || b.VIN == "" || b.Source == "" {
|
||||
return fmt.Errorf("invalid GPS identity")
|
||||
}
|
||||
first, err := strconv.ParseFloat(a.First, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
last, err := strconv.ParseFloat(a.Last, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
daily, err := strconv.ParseFloat(a.Daily, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
firstTime, err := time.ParseInLocation("2006-01-02 15:04:05", a.FirstTime, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lastTime, err := time.ParseInLocation("2006-01-02 15:04:05", a.LastTime, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldEnd, err := time.ParseInLocation("2006-01-02 15:04:05", b.LastTime, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if lastTime.Before(oldEnd) || firstTime.After(lastTime) || lastTime.Format("2006-01-02") != b.Date || first <= 0 || last <= 0 {
|
||||
return fmt.Errorf("invalid event-time boundary %s %s", b.VIN, b.Date)
|
||||
}
|
||||
sample := stats.SourceMileageSample{DailyKM: last - first, FirstEventTime: firstTime, LatestEventTime: lastTime, QualityStatus: stats.QualityOK, QualityReason: a.Reason}
|
||||
stats.ApplyMileageQualityRules(&sample)
|
||||
if math.IsNaN(daily) || math.IsInf(daily, 0) || math.Abs(sample.DailyKM-daily) > 0.0001 || sample.QualityStatus != a.Status {
|
||||
return fmt.Errorf("incorrect reviewed mileage %s %s", b.VIN, b.Date)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestReviewedEventTimeRepairRequiresConsistentForwardTime(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
c := change{Before: row{VIN: "LGAX2BG42H1003112", Date: "2026-10-21", Protocol: "JT808", Source: "JT808:terminal@PLATFORM:forwarder", LastTime: "2026-10-21 08:59:34"},
|
||||
After: row{First: "44124.600", Last: "44402.100", FirstTime: "2026-10-20 23:29:53", LastTime: "2026-10-21 23:59:55", Daily: "277.500", Status: "OK", Reason: "historical_source_baseline"}}
|
||||
if err := validateChange(c, loc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mutate := range []func(*change){
|
||||
func(c *change) { c.After.LastTime = "2026-10-21 08:00:00" },
|
||||
func(c *change) { c.After.Daily = "353.800" },
|
||||
func(c *change) { c.Before.Protocol = "GB32960" },
|
||||
func(c *change) { c.After.FirstTime = "2026-10-22 00:00:00" },
|
||||
} {
|
||||
bad := c
|
||||
mutate(&bad)
|
||||
if err := validateChange(bad, loc); err == nil {
|
||||
t.Fatalf("unsafe reviewed plan accepted: %+v", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -75,6 +75,28 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if loaded, err := writer.ReloadSourceIdentities(ctx); err != nil {
|
||||
logger.Error("source identity snapshot load failed", "error", err)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
registry.SetGauge("vehicle_stat_source_identity_snapshot_entries", nil, float64(loaded))
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if loaded, err := writer.ReloadSourceIdentities(ctx); err != nil {
|
||||
logger.Warn("source identity snapshot refresh failed; retaining previous snapshot", "error", err)
|
||||
} else {
|
||||
registry.SetGauge("vehicle_stat_source_identity_snapshot_entries", nil, float64(loaded))
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
if cfg.HydrogenCapacitySyncEnabled {
|
||||
syncHydrogenCapacities(ctx, logger, registry, db, writer, cfg)
|
||||
} else if loaded, err := writer.ReloadHydrogenTankCapacities(ctx); err != nil {
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
const customerHydrogenAlgorithm = "MODE_INTERVAL_V1"
|
||||
|
||||
type customerHydrogenDay struct {
|
||||
VIN, Date, Protocol, SourceKey string
|
||||
HydrogenKM, ObservedKM, ClassifiedKM float64
|
||||
Samples, KnownSamples, Intervals, Rejected int64
|
||||
First, Last time.Time
|
||||
PreviousKM float64
|
||||
}
|
||||
|
||||
func addCustomerHydrogenSample(days map[string]*customerHydrogenDay, s stats.MetricSample) {
|
||||
if s.Protocol != envelope.ProtocolGB32960 && s.Protocol != envelope.ProtocolYutongMQTT {
|
||||
return
|
||||
}
|
||||
if math.IsNaN(s.TotalMileageKM) || math.IsInf(s.TotalMileageKM, 0) || s.TotalMileageKM < 0 {
|
||||
return
|
||||
}
|
||||
key := s.VIN + "/" + s.StatDate + "/" + string(s.Protocol) + "/" + s.SourceKey
|
||||
d := days[key]
|
||||
if d == nil {
|
||||
d = &customerHydrogenDay{VIN: s.VIN, Date: s.StatDate, Protocol: string(s.Protocol), SourceKey: s.SourceKey, First: s.EventTime, Last: s.EventTime, PreviousKM: s.TotalMileageKM, Samples: 1}
|
||||
if s.PureHydrogenModeKnown {
|
||||
d.KnownSamples = 1
|
||||
}
|
||||
days[key] = d
|
||||
return
|
||||
}
|
||||
// Do not count duplicate or out-of-order reports as extra evidence.
|
||||
if !s.EventTime.After(d.Last) {
|
||||
return
|
||||
}
|
||||
elapsed := s.EventTime.Sub(d.Last)
|
||||
delta := s.TotalMileageKM - d.PreviousKM
|
||||
d.Samples++
|
||||
if s.PureHydrogenModeKnown {
|
||||
d.KnownSamples++
|
||||
}
|
||||
if delta < -0.2 || delta > math.Max(0.5, elapsed.Hours()*130) || elapsed > 15*time.Minute {
|
||||
d.Rejected++
|
||||
} else {
|
||||
delta = math.Max(0, delta)
|
||||
d.ObservedKM += delta
|
||||
d.Intervals++
|
||||
if s.PureHydrogenModeKnown {
|
||||
d.ClassifiedKM += delta
|
||||
if s.PureHydrogenActive {
|
||||
d.HydrogenKM += delta
|
||||
}
|
||||
}
|
||||
}
|
||||
d.Last = s.EventTime
|
||||
d.PreviousKM = s.TotalMileageKM
|
||||
}
|
||||
|
||||
// This path reads raw work-mode evidence and only writes the dedicated customer
|
||||
// monitor projection. It never resets/reprojects the existing GPS/daily tables.
|
||||
func runCustomerHydrogenBackfill(ctx context.Context, mysqlDB, td *sql.DB, cfg config) error {
|
||||
return runCustomerHydrogenBackfillWithLease(ctx, mysqlDB, td, cfg, "")
|
||||
}
|
||||
|
||||
func runCustomerHydrogenBackfillWithLease(ctx context.Context, mysqlDB, td *sql.DB, cfg config, owner string) error {
|
||||
if cfg.Reset {
|
||||
return fmt.Errorf("hydrogen-only backfill does not allow reset")
|
||||
}
|
||||
if cfg.Limit > 0 && !cfg.DryRun {
|
||||
return fmt.Errorf("limited scan cannot publish a daily result")
|
||||
}
|
||||
for _, p := range cfg.Protocols {
|
||||
if p != envelope.ProtocolGB32960 && p != envelope.ProtocolYutongMQTT {
|
||||
return fmt.Errorf("hydrogen-only backfill accepts GB32960,YUTONG_MQTT only")
|
||||
}
|
||||
}
|
||||
rows, err := queryRawFrames(ctx, td, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
days := map[string]*customerHydrogenDay{}
|
||||
var scanned, skipped int64
|
||||
for rows.Next() {
|
||||
scanned++
|
||||
row, err := scanRawFrame(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := strings.TrimSpace(row.ParsedJSON)
|
||||
if isChunkManifest(text) {
|
||||
text, err = readChunkedPayload(ctx, td, cfg.TDengineDatabase, row.EventID, "parsed_fields")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read complete mode evidence: %w", err)
|
||||
}
|
||||
}
|
||||
fields := fieldsForStats(row.Protocol, row.VIN, text)
|
||||
if len(fields) == 0 && row.Protocol == envelope.ProtocolYutongMQTT {
|
||||
fields = fieldsForStats(row.Protocol, row.VIN, row.RawText)
|
||||
}
|
||||
samples, err := stats.SamplesFromEnvelope(envelope.FrameEnvelope{EventID: row.EventID, Protocol: row.Protocol, MessageID: row.MessageID, VIN: row.VIN, Phone: row.Phone, DeviceID: row.DeviceID, SourceEndpoint: row.SourceEndpoint, EventTimeMS: row.EventTimeMS, ReceivedAtMS: row.ReceivedAtMS, Fields: fields, ParsedFields: fields, ParseStatus: envelope.ParseOK}, cfg.Location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(samples) == 0 {
|
||||
skipped++
|
||||
}
|
||||
for _, s := range samples {
|
||||
if s.StatDate >= cfg.DateFrom && s.StatDate <= cfg.DateTo {
|
||||
addCustomerHydrogenSample(days, s)
|
||||
}
|
||||
}
|
||||
if cfg.ProgressEvery > 0 && scanned%cfg.ProgressEvery == 0 {
|
||||
slog.Info("customer hydrogen scan", "scanned", scanned, "source_days", len(days), "skipped", skipped)
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
var known, samples int64
|
||||
var hydrogen float64
|
||||
for _, d := range days {
|
||||
known += d.KnownSamples
|
||||
samples += d.Samples
|
||||
hydrogen += d.HydrogenKM
|
||||
}
|
||||
slog.Info("customer hydrogen evidence", "scanned", scanned, "samples", samples, "known_mode_samples", known, "source_days", len(days), "hydrogen_km_across_sources", hydrogen, "dry_run", cfg.DryRun)
|
||||
if cfg.DryRun {
|
||||
return nil
|
||||
}
|
||||
return publishCustomerHydrogenDays(ctx, mysqlDB, days, cfg.DateFrom, owner)
|
||||
}
|
||||
|
||||
func publishCustomerHydrogenDays(ctx context.Context, mysqlDB *sql.DB, days map[string]*customerHydrogenDay, date, owner string) error {
|
||||
// Publish only after a complete successful scan. Upserts are deterministic;
|
||||
// failures roll back the entire projection update.
|
||||
tx, err := mysqlDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if owner != "" {
|
||||
// Fence publication with the same row lock used to claim jobs. A stale
|
||||
// worker must not overwrite results even before its next heartbeat.
|
||||
var valid int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT 1 FROM platform_customer_hydrogen_job WHERE stat_date=? AND lease_owner=? AND status='running' AND lease_until>NOW(3) FOR UPDATE`, date, owner).Scan(&valid); err != nil {
|
||||
return fmt.Errorf("customer hydrogen publication lease lost: %w", err)
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(days))
|
||||
for k := range days {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for start := 0; start < len(keys); start += 200 {
|
||||
end := start + 200
|
||||
if end > len(keys) {
|
||||
end = len(keys)
|
||||
}
|
||||
values := []string{}
|
||||
args := []any{}
|
||||
for _, key := range keys[start:end] {
|
||||
d := days[key]
|
||||
values = append(values, "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,NOW(3))")
|
||||
args = append(args, d.VIN, d.Date, d.Protocol, d.SourceKey, d.HydrogenKM, d.ObservedKM, d.ClassifiedKM, d.Samples, d.KnownSamples, d.Intervals, d.Rejected, d.First, d.Last, customerHydrogenAlgorithm)
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO platform_customer_hydrogen_daily(vin,stat_date,protocol,source_key,hydrogen_km,observed_km,classified_km,sample_count,known_sample_count,interval_count,rejected_intervals,first_event_time,last_event_time,algorithm_version,calculated_at) VALUES `+strings.Join(values, ",")+` ON DUPLICATE KEY UPDATE hydrogen_km=VALUES(hydrogen_km),observed_km=VALUES(observed_km),classified_km=VALUES(classified_km),sample_count=VALUES(sample_count),known_sample_count=VALUES(known_sample_count),interval_count=VALUES(interval_count),rejected_intervals=VALUES(rejected_intervals),first_event_time=VALUES(first_event_time),last_event_time=VALUES(last_event_time),algorithm_version=VALUES(algorithm_version),calculated_at=VALUES(calculated_at)`, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCustomerHydrogenIntervalsClassifyElectricZeroAndUnknown(t *testing.T) {
|
||||
start := time.Date(2026, 9, 29, 8, 0, 0, 0, time.UTC)
|
||||
days := map[string]*customerHydrogenDay{}
|
||||
sample := func(min int, km float64, known, active bool) stats.MetricSample {
|
||||
return stats.MetricSample{VIN: "VIN", StatDate: "2026-09-29", Protocol: envelope.ProtocolGB32960, SourceKey: "source", EventTime: start.Add(time.Duration(min) * time.Minute), TotalMileageKM: km, PureHydrogenModeKnown: known, PureHydrogenActive: active}
|
||||
}
|
||||
for _, s := range []stats.MetricSample{sample(0, 100, true, false), sample(5, 105, true, false), sample(10, 110, true, true), sample(15, 115, false, false), sample(15, 115, true, true), sample(60, 130, true, true), sample(65, 140, true, true)} {
|
||||
addCustomerHydrogenSample(days, s)
|
||||
}
|
||||
for _, d := range days {
|
||||
if d.HydrogenKM != 15 || d.ObservedKM != 25 || d.ClassifiedKM != 20 || d.Samples != 6 || d.KnownSamples != 5 || d.Rejected != 1 {
|
||||
t.Fatalf("incorrect evidence %+v", d)
|
||||
}
|
||||
}
|
||||
zero := map[string]*customerHydrogenDay{}
|
||||
addCustomerHydrogenSample(zero, sample(0, 100, true, false))
|
||||
addCustomerHydrogenSample(zero, sample(5, 105, true, false))
|
||||
for _, d := range zero {
|
||||
if d.HydrogenKM != 0 || d.ClassifiedKM != 5 || d.KnownSamples != 2 {
|
||||
t.Fatal("observed electric-only day not distinguished from missing")
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestCustomerHydrogenSeparatesSourcesDaysAndRejectsJumps(t *testing.T) {
|
||||
days := map[string]*customerHydrogenDay{}
|
||||
s := stats.MetricSample{VIN: "VIN", StatDate: "2026-09-29", Protocol: envelope.ProtocolGB32960, SourceKey: "A", EventTime: time.Now(), TotalMileageKM: 100, PureHydrogenModeKnown: true, PureHydrogenActive: true}
|
||||
addCustomerHydrogenSample(days, s)
|
||||
s.EventTime = s.EventTime.Add(time.Minute)
|
||||
s.TotalMileageKM = 1100
|
||||
addCustomerHydrogenSample(days, s)
|
||||
s.SourceKey = "B"
|
||||
s.TotalMileageKM = 100
|
||||
addCustomerHydrogenSample(days, s)
|
||||
s.StatDate = "2026-09-30"
|
||||
addCustomerHydrogenSample(days, s)
|
||||
if len(days) != 3 {
|
||||
t.Fatal("source/day evidence mixed")
|
||||
}
|
||||
for _, d := range days {
|
||||
if d.HydrogenKM != 0 {
|
||||
t.Fatal("odometer jump counted as hydrogen")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func runCustomerHydrogenWorker(parent context.Context, mysqlDB, td *sql.DB, cfg config) error {
|
||||
ctx, stop := signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
maxJobs := envInt("CUSTOMER_HYDROGEN_MAX_JOBS", 0)
|
||||
completedJobs := 0
|
||||
for ctx.Err() == nil {
|
||||
date, owner, err := claimCustomerHydrogenDay(ctx, mysqlDB)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
slog.Error("customer hydrogen claim failed", "error", err)
|
||||
}
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(5 * time.Second):
|
||||
continue
|
||||
}
|
||||
}
|
||||
dayCtx, cancel := context.WithTimeout(ctx, 2*time.Hour)
|
||||
heartbeatDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(heartbeatDone)
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-dayCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
result, e := mysqlDB.ExecContext(dayCtx, `UPDATE platform_customer_hydrogen_job SET lease_until=DATE_ADD(NOW(3),INTERVAL 5 MINUTE) WHERE stat_date=? AND lease_owner=? AND status='running'`, date, owner)
|
||||
if e != nil {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
n, e := result.RowsAffected()
|
||||
if e != nil || n != 1 {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
task := cfg
|
||||
task.DateFrom = date
|
||||
task.DateTo = date
|
||||
task.Limit = 0
|
||||
task.DryRun = false
|
||||
task.Reset = false
|
||||
task.VINs = nil
|
||||
slog.Info("customer hydrogen job started", "date", date)
|
||||
err = runCustomerHydrogenBackfillWithLease(dayCtx, mysqlDB, td, task, owner)
|
||||
cancel()
|
||||
<-heartbeatDone
|
||||
status, message := "complete", ""
|
||||
if err != nil {
|
||||
status, message = "failed", "原始报文计算失败,请检查任务日志"
|
||||
slog.Error("customer hydrogen job failed", "date", date, "error", err)
|
||||
}
|
||||
// A cancelled/expired worker cannot acknowledge another worker's lease.
|
||||
finishCtx, finishCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_, saveErr := mysqlDB.ExecContext(finishCtx, `UPDATE platform_customer_hydrogen_job SET status=?,error_message=?,completed_at=IF(?='complete',NOW(3),completed_at),lease_until=NULL,lease_owner='' WHERE stat_date=? AND lease_owner=?`, status, message, status, date, owner)
|
||||
finishCancel()
|
||||
if saveErr != nil {
|
||||
slog.Error("customer hydrogen job completion failed", "date", date, "error", saveErr)
|
||||
}
|
||||
completedJobs++
|
||||
if maxJobs > 0 && completedJobs >= maxJobs {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func claimCustomerHydrogenDay(ctx context.Context, db *sql.DB) (string, string, error) {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var date string
|
||||
err = tx.QueryRowContext(ctx, `SELECT DATE_FORMAT(stat_date,'%Y-%m-%d') FROM platform_customer_hydrogen_job WHERE status='pending' OR (status='running' AND lease_until<NOW(3)) OR (status='failed' AND attempts<3 AND started_at<DATE_SUB(NOW(3),INTERVAL 1 MINUTE)) ORDER BY requested_at ASC,stat_date DESC LIMIT 1 FOR UPDATE`).Scan(&date)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
var bytes [16]byte
|
||||
if _, err = rand.Read(bytes[:]); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
owner := hex.EncodeToString(bytes[:])
|
||||
_, err = tx.ExecContext(ctx, `UPDATE platform_customer_hydrogen_job SET status='running',started_at=NOW(3),lease_until=DATE_ADD(NOW(3),INTERVAL 5 MINUTE),lease_owner=?,attempts=attempts+1,error_message='' WHERE stat_date=?`, owner, date)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return date, owner, nil
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestCustomerHydrogenJobClaimUsesTransactionAndOwner(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT DATE_FORMAT.*FOR UPDATE").WillReturnRows(sqlmock.NewRows([]string{"date"}).AddRow("2026-09-29"))
|
||||
mock.ExpectExec("UPDATE platform_customer_hydrogen_job").WithArgs(sqlmock.AnyArg(), "2026-09-29").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
date, owner, err := claimCustomerHydrogenDay(context.Background(), db)
|
||||
if err != nil || date != "2026-09-29" || len(owner) != 32 {
|
||||
t.Fatalf("invalid claim %s %s %v", date, owner, err)
|
||||
}
|
||||
if err = mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomerHydrogenBoundedWorkerPublishesOneJobThenExits(t *testing.T) {
|
||||
t.Setenv("CUSTOMER_HYDROGEN_MAX_JOBS", "1")
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
td, raw, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer td.Close()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT DATE_FORMAT.*FOR UPDATE").WillReturnRows(sqlmock.NewRows([]string{"date"}).AddRow("2026-09-29"))
|
||||
mock.ExpectExec("UPDATE platform_customer_hydrogen_job").WithArgs(sqlmock.AnyArg(), "2026-09-29").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
raw.ExpectQuery("SELECT").WillReturnRows(sqlmock.NewRows([]string{"event_id"}))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT 1.*FOR UPDATE").WithArgs("2026-09-29", sqlmock.AnyArg()).WillReturnRows(sqlmock.NewRows([]string{"valid"}).AddRow(1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("UPDATE platform_customer_hydrogen_job SET status").WithArgs("complete", "", "complete", "2026-09-29", sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
err = runCustomerHydrogenWorker(ctx, db, td, config{Location: time.UTC, TDengineDatabase: "vehicle_ingest", Protocols: []envelope.Protocol{envelope.ProtocolGB32960}})
|
||||
if err != nil || ctx.Err() != nil {
|
||||
t.Fatalf("bounded worker did not exit after publication: %v / %v", err, ctx.Err())
|
||||
}
|
||||
if err = mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = raw.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomerHydrogenStaleWorkerCannotPublish(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT 1.*lease_until>NOW.*FOR UPDATE").WithArgs("2026-09-29", "old-owner").WillReturnRows(sqlmock.NewRows([]string{"valid"}))
|
||||
mock.ExpectRollback()
|
||||
days := map[string]*customerHydrogenDay{"sample": {VIN: "VIN", Date: "2026-09-29", HydrogenKM: 10}}
|
||||
if err = publishCustomerHydrogenDays(context.Background(), db, days, "2026-09-29", "old-owner"); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("stale worker must fail before writing: %v", err)
|
||||
}
|
||||
if err = mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomerHydrogenPublicationLocksCurrentLease(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT 1.*lease_until>NOW.*FOR UPDATE").WithArgs("2026-09-29", "current-owner").WillReturnRows(sqlmock.NewRows([]string{"valid"}).AddRow(1))
|
||||
mock.ExpectExec("INSERT INTO platform_customer_hydrogen_daily").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
days := map[string]*customerHydrogenDay{"sample": {VIN: "VIN", Date: "2026-09-29", HydrogenKM: 10}}
|
||||
if err = publishCustomerHydrogenDays(context.Background(), db, days, "2026-09-29", "current-owner"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func TestCustomerHydrogenIdleWorkerDoesNotCreateJobs(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT DATE_FORMAT.*FOR UPDATE").WillReturnRows(sqlmock.NewRows([]string{"date"}))
|
||||
mock.ExpectRollback()
|
||||
if _, _, err = claimCustomerHydrogenDay(context.Background(), db); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
var errNoUsableSourceBoundary = errors.New("source has no usable mileage boundary")
|
||||
|
||||
// FIRST/LAST are ordered by TDengine's storage timestamp, which represents
|
||||
// receipt order. Keep the fast aggregate path, but independently verify both
|
||||
// ends against extrema of the vehicle event timestamp and fetch the matching
|
||||
// complete row whenever late delivery has moved either end.
|
||||
type sourceBoundaryEvidence struct {
|
||||
vin, phone, deviceID, endpoint string
|
||||
firstTS, lastTS time.Time
|
||||
firstParsed, firstRaw string
|
||||
lastParsed, lastRaw string
|
||||
count, minimumMS, maximumMS int64
|
||||
}
|
||||
|
||||
func correctSourceEventBoundaries(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, where string, includeFirst bool, e *sourceBoundaryEvidence) error {
|
||||
_, firstUsable := mileageFromEvidence(protocol, e.vin, e.firstParsed, e.firstRaw, e.firstTS, cfg.Location)
|
||||
_, lastUsable := mileageFromEvidence(protocol, e.vin, e.lastParsed, e.lastRaw, e.lastTS, cfg.Location)
|
||||
if e.lastTS.UnixMilli() != e.maximumMS || !lastUsable {
|
||||
stamp, parsed, raw, err := querySourceEventBoundary(ctx, db, cfg, protocol, where, *e, e.maximumMS, false)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) && !firstUsable && !lastUsable {
|
||||
return errNoUsableSourceBoundary
|
||||
}
|
||||
return err
|
||||
}
|
||||
slog.Info("corrected last source event boundary", "vin", e.vin, "endpoint", e.endpoint, "storage_last", e.lastTS, "event_last", stamp)
|
||||
e.lastTS, e.lastParsed, e.lastRaw = stamp, parsed, raw
|
||||
}
|
||||
if !includeFirst {
|
||||
e.firstTS, e.firstParsed, e.firstRaw = e.lastTS, e.lastParsed, e.lastRaw
|
||||
return nil
|
||||
}
|
||||
if e.firstTS.UnixMilli() != e.minimumMS || !firstUsable {
|
||||
stamp, parsed, raw, err := querySourceEventBoundary(ctx, db, cfg, protocol, where, *e, e.minimumMS, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Info("corrected first source event boundary", "vin", e.vin, "endpoint", e.endpoint, "storage_first", e.firstTS, "event_first", stamp)
|
||||
e.firstTS, e.firstParsed, e.firstRaw = stamp, parsed, raw
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func querySourceEventBoundary(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, where string, e sourceBoundaryEvidence, eventMS int64, first bool) (time.Time, string, string, error) {
|
||||
// An equal event timestamp is resolved consistently by newest storage ts.
|
||||
// Never obtain the timestamp and payload through independent aggregates.
|
||||
comparison, order := "<=", "DESC"
|
||||
if first {
|
||||
comparison, order = ">=", "ASC"
|
||||
}
|
||||
query := fmt.Sprintf(`SELECT event_time, parsed_json, raw_text
|
||||
FROM %s.raw_frames
|
||||
WHERE %s AND vin = '%s' AND phone = '%s' AND device_id = '%s'
|
||||
AND source_endpoint = '%s' AND CAST(event_time AS BIGINT) %s %d
|
||||
ORDER BY event_time %s, ts DESC`, ident(cfg.TDengineDatabase), where, quote(e.vin), quote(e.phone), quote(e.deviceID), quote(e.endpoint), comparison, eventMS, order)
|
||||
rows, err := db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return time.Time{}, "", "", fmt.Errorf("resolve event-time boundary %s/%s/%d: %w", e.vin, e.endpoint, eventMS, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var stamp time.Time
|
||||
var parsed, raw string
|
||||
if err := rows.Scan(&stamp, &parsed, &raw); err != nil {
|
||||
return time.Time{}, "", "", err
|
||||
}
|
||||
if _, ok := mileageFromEvidence(protocol, e.vin, parsed, raw, stamp, cfg.Location); ok {
|
||||
return stamp, parsed, raw, nil
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return time.Time{}, "", "", err
|
||||
}
|
||||
return time.Time{}, "", "", sql.ErrNoRows
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestDailySourceEndUsesVehicleEventTimeAfterLateDelivery(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
vin, phone, ip, code string
|
||||
lateHour, lateMin int
|
||||
endHour, endMin int
|
||||
lateKM, endKM float64
|
||||
}{
|
||||
{"LGAX2BG42H1003112", "40409727033", "122.152.221.156", "dongfang_beidou", 8, 59, 23, 59, 44478.4, 44402.1},
|
||||
{"LA9HE60A0PBAF4002", "13307765803", "115.231.168.135", "xinda", 20, 46, 21, 14, 6194.8, 6195.1},
|
||||
} {
|
||||
t.Run(tc.vin, func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
first := time.Date(2026, 10, 21, 0, 0, 0, 0, loc)
|
||||
late := time.Date(2026, 10, 21, tc.lateHour, tc.lateMin, 0, 0, loc)
|
||||
last := time.Date(2026, 10, 21, tc.endHour, tc.endMin, 0, 0, loc)
|
||||
parsed := func(km float64) string { return fmt.Sprintf(`{"jt808.location.total_mileage_km":%f}`, km) }
|
||||
mock.ExpectQuery("(?s)SELECT vin.*MIN\\(CAST\\(event_time AS BIGINT\\)\\).*MAX\\(CAST\\(event_time AS BIGINT\\)\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "first", "first_parsed", "first_raw", "last", "last_parsed", "last_raw", "count", "minimum", "maximum"}).
|
||||
AddRow(tc.vin, tc.phone, "", tc.ip+":1234", first, parsed(tc.endKM-100), "", late, parsed(tc.lateKM), "", 100, first.UnixMilli(), last.UnixMilli())).RowsWillBeClosed()
|
||||
mock.ExpectQuery(fmt.Sprintf("(?s)SELECT event_time, parsed_json, raw_text.*phone = '%s'.*CAST\\(event_time AS BIGINT\\) <= %d.*ORDER BY event_time DESC, ts DESC", tc.phone, last.UnixMilli())).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"event_time", "parsed_json", "raw_text"}).AddRow(last, parsed(tc.endKM), ""))
|
||||
cfg := config{TDengineDatabase: "lingniu_vehicle_ts", Location: loc}
|
||||
result, err := queryDailyLastSourceRows(context.Background(), db, cfg, envelope.ProtocolJT808, "2026-10-21")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := result[tc.vin][0]
|
||||
if !got.TS.Equal(last) || math.Abs(got.TotalKM-tc.endKM) > 0.0001 || got.RawSampleCount != 100 {
|
||||
t.Fatalf("late delivery replaced the vehicle day end: %+v", got)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDailySourceFirstAlsoUsesVehicleEventTime(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
first := time.Date(2026, 10, 21, 8, 0, 0, 0, loc)
|
||||
storageFirst := first.Add(time.Hour)
|
||||
last := first.Add(10 * time.Hour)
|
||||
mock.ExpectQuery("(?s)SELECT vin.*MIN\\(CAST").WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "first", "first_parsed", "first_raw", "last", "last_parsed", "last_raw", "count", "minimum", "maximum"}).
|
||||
AddRow("VIN", "phone", "", "1.2.3.4:123", storageFirst, `{"jt808.location.total_mileage_km":101}`, "", last, `{"jt808.location.total_mileage_km":110}`, "", 5, first.UnixMilli(), last.UnixMilli()))
|
||||
mock.ExpectQuery(fmt.Sprintf("(?s)SELECT event_time.*CAST\\(event_time AS BIGINT\\) >= %d", first.UnixMilli())).WillReturnRows(sqlmock.NewRows([]string{"event", "parsed", "raw"}).AddRow(first, `{"jt808.location.total_mileage_km":100}`, ""))
|
||||
got, err := queryDailyLastSourceRows(context.Background(), db, config{TDengineDatabase: "db", Location: loc}, envelope.ProtocolJT808, "2026-10-21")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["VIN"][0].FirstTotalKM != 100 || !got["VIN"][0].FirstTS.Equal(first) {
|
||||
t.Fatalf("wrong first event: %+v", got)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMissingVehicleEventBoundaryFailsBeforeWriting(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
stamp := time.Date(2026, 10, 20, 20, 0, 0, 0, time.UTC)
|
||||
mock.ExpectQuery("(?s)SELECT vin.*MAX\\(CAST").WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "last", "parsed", "raw", "count", "maximum"}).AddRow("VIN", "phone", "", "1.2.3.4:123", stamp, `{"jt808.location.total_mileage_km":100}`, "", 2, stamp.Add(time.Hour).UnixMilli()))
|
||||
mock.ExpectQuery("SELECT event_time").WillReturnRows(sqlmock.NewRows([]string{"event", "parsed", "raw"}))
|
||||
if _, err := queryPreviousLastSourceRows(context.Background(), db, config{TDengineDatabase: "db", Location: time.UTC}, envelope.ProtocolJT808, "2026-10-21"); err == nil {
|
||||
t.Fatal("missing event boundary silently accepted")
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroMileageTailDoesNotHideLastUsableVehicleEvent(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
first := time.Date(2026, 10, 21, 8, 0, 0, 0, loc)
|
||||
last := first.Add(time.Hour)
|
||||
zero := last.Add(time.Hour)
|
||||
mock.ExpectQuery("(?s)SELECT vin.*MIN\\(CAST").WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "first", "first_parsed", "first_raw", "last", "last_parsed", "last_raw", "count", "minimum", "maximum"}).AddRow("VIN", "phone", "", "1.2.3.4:123", first, `{"jt808.location.total_mileage_km":100}`, "", zero, `{"jt808.location.total_mileage_km":0}`, "", 3, first.UnixMilli(), zero.UnixMilli()))
|
||||
mock.ExpectQuery("(?s)SELECT event_time.*ORDER BY event_time DESC, ts DESC").WillReturnRows(sqlmock.NewRows([]string{"event", "parsed", "raw"}).AddRow(zero, `{"jt808.location.total_mileage_km":0}`, "").AddRow(last, `{"jt808.location.total_mileage_km":110}`, ""))
|
||||
got, err := queryDailyLastSourceRows(context.Background(), db, config{TDengineDatabase: "db", Location: loc}, envelope.ProtocolJT808, "2026-10-21")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["VIN"][0].TotalKM != 110 || !got["VIN"][0].TS.Equal(last) {
|
||||
t.Fatalf("non-positive frame replaced usable end: %+v", got)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
@@ -43,6 +44,7 @@ type config struct {
|
||||
ListHydrogenVINs bool
|
||||
ListActiveVINs bool
|
||||
Location *time.Location
|
||||
SourceCatalog map[backfillSourceID]stats.SourceIdentity
|
||||
}
|
||||
|
||||
type rawFrameRow struct {
|
||||
@@ -150,9 +152,27 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
schemaWriter := stats.NewWriter(mysqlDB, cfg.Location)
|
||||
if err := schemaWriter.EnsureSchema(ctx); err != nil {
|
||||
fail("ensure schema", err)
|
||||
if envBool("CUSTOMER_HYDROGEN_WORKER", false) {
|
||||
if err := runCustomerHydrogenWorker(ctx, mysqlDB, td, cfg); err != nil {
|
||||
fail("customer hydrogen worker", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if envBool("BACKFILL_HYDROGEN_ONLY", false) {
|
||||
if err := runCustomerHydrogenBackfill(ctx, mysqlDB, td, cfg); err != nil {
|
||||
fail("customer hydrogen backfill", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
cfg.SourceCatalog, err = loadBackfillSourceCatalog(ctx, mysqlDB)
|
||||
if err != nil {
|
||||
fail("load backfill source catalog", err)
|
||||
}
|
||||
if !cfg.DryRun {
|
||||
schemaWriter := stats.NewWriter(mysqlDB, cfg.Location)
|
||||
if err := schemaWriter.EnsureSchema(ctx); err != nil {
|
||||
fail("ensure schema", err)
|
||||
}
|
||||
}
|
||||
if cfg.Reset && !cfg.DryRun {
|
||||
deleted, err := resetStats(ctx, mysqlDB, cfg)
|
||||
@@ -178,6 +198,9 @@ func main() {
|
||||
if err != nil {
|
||||
fail("cleanup sub-threshold GPS mileage candidates", err)
|
||||
}
|
||||
if cfg.Debug {
|
||||
logBackfillCandidates(aggregates)
|
||||
}
|
||||
var written int64
|
||||
var normalized int
|
||||
if !cfg.DryRun {
|
||||
@@ -250,6 +273,7 @@ func main() {
|
||||
ParsedFields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
env.SourceCode, env.PlatformName, env.SourceKind = backfillSourceMetadata(cfg, row.Protocol, row.SourceEndpoint)
|
||||
samples, err := stats.SamplesFromEnvelope(env, cfg.Location)
|
||||
if err != nil {
|
||||
slog.Warn("skip invalid sample", "event_id", row.EventID, "error", err)
|
||||
@@ -347,6 +371,23 @@ func activeModeSampleCount(sample stats.MetricSample) int64 {
|
||||
}
|
||||
|
||||
func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg, batchSize int) (int64, error) {
|
||||
gps, other := map[string]*metricAgg{}, map[string]*metricAgg{}
|
||||
for key, agg := range aggregates {
|
||||
if agg.Protocol == envelope.ProtocolJT808 {
|
||||
gps[key] = agg
|
||||
} else {
|
||||
other[key] = agg
|
||||
}
|
||||
}
|
||||
written, err := writeProtectedGPSAggregates(ctx, db, gps)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
n, err := writeLegacyAggregates(ctx, db, other, batchSize)
|
||||
return written + n, err
|
||||
}
|
||||
|
||||
func writeLegacyAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg, batchSize int) (int64, error) {
|
||||
if len(aggregates) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -429,11 +470,25 @@ func normalizeHistoricalPlatformSources(ctx context.Context, db *sql.DB, cfg con
|
||||
continue
|
||||
}
|
||||
for _, date := range dates {
|
||||
count, err := stats.NormalizePlatformSourceMileageForDate(ctx, db, date, protocol)
|
||||
if len(cfg.VINs) > 0 {
|
||||
for _, vin := range cfg.VINs {
|
||||
if err := normalizeProtectedGPSDay(ctx, db, vin, date); err != nil {
|
||||
return normalized, err
|
||||
}
|
||||
normalized++
|
||||
}
|
||||
continue
|
||||
}
|
||||
vins, err := stats.PlatformSourceMileageLegacyVINs(ctx, db, date, protocol)
|
||||
if err != nil {
|
||||
return normalized, err
|
||||
}
|
||||
normalized += count
|
||||
for _, vin := range vins {
|
||||
if err := normalizeProtectedGPSDay(ctx, db, vin, date); err != nil {
|
||||
return normalized, err
|
||||
}
|
||||
normalized++
|
||||
}
|
||||
}
|
||||
}
|
||||
return normalized, nil
|
||||
@@ -592,7 +647,8 @@ FROM vehicle_daily_mileage_source
|
||||
WHERE stat_date >= ? AND stat_date <= ?
|
||||
AND quality_reason = ?
|
||||
AND daily_mileage_km >= 0 AND daily_mileage_km < ?
|
||||
AND protocol IN (` + strings.Join(placeholders, ",") + `)
|
||||
AND protocol IN (` + strings.Join(placeholders, ",") + `)`
|
||||
query = appendBackfillVINScope(query, cfg, "vin") + `
|
||||
GROUP BY vin, stat_date, protocol
|
||||
ORDER BY stat_date, protocol, vin`
|
||||
rows, err := db.QueryContext(ctx, query, args...)
|
||||
@@ -697,6 +753,9 @@ func addProtocolGPSCoordinateFallbackAggregates(ctx context.Context, mysqlDB *sq
|
||||
"latitude BETWEEN -90 AND 90",
|
||||
"NOT (longitude = 0 AND latitude = 0)",
|
||||
}
|
||||
if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
if protocol != envelope.ProtocolJT808 {
|
||||
where = append(where, "speed_kmh > 0")
|
||||
}
|
||||
@@ -1060,13 +1119,17 @@ func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, proto
|
||||
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
|
||||
FIRST(event_time), FIRST(parsed_json), FIRST(raw_text),
|
||||
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
|
||||
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*),
|
||||
MIN(CAST(event_time AS BIGINT)), MAX(CAST(event_time AS BIGINT))
|
||||
FROM %s.raw_frames
|
||||
WHERE %s
|
||||
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
return querySourceRows(ctx, db, cfg, protocol, sqlText, true)
|
||||
return querySourceRows(ctx, db, cfg, protocol, sqlText, true, strings.Join(where, " AND "))
|
||||
}
|
||||
|
||||
func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
|
||||
@@ -1085,12 +1148,15 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
|
||||
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
|
||||
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
|
||||
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*), MAX(CAST(event_time AS BIGINT))
|
||||
FROM %s.raw_frames
|
||||
WHERE %s
|
||||
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
|
||||
return querySourceRows(ctx, db, cfg, protocol, sqlText, false, strings.Join(where, " AND "))
|
||||
}
|
||||
|
||||
func backfillBeforePredicates(cfg config, eventDateExclusive string) []string {
|
||||
@@ -1141,6 +1207,7 @@ WHERE l.protocol = ?
|
||||
if cfg.StationaryCarry {
|
||||
args = append(args, date)
|
||||
}
|
||||
sqlText = appendBackfillVINScope(sqlText, cfg, "l.vin")
|
||||
rows, err := db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1167,7 +1234,7 @@ WHERE l.protocol = ?
|
||||
if protocol == envelope.ProtocolYutongMQTT {
|
||||
deviceID = vin
|
||||
}
|
||||
sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
|
||||
sourceCode, platformName, sourceKind := backfillSourceMetadata(cfg, protocol, sourceEndpoint)
|
||||
sourceKey := normalizedSourceKeyForSource(string(protocol), "", deviceID, sourceEndpoint, sourceKind, sourceCode)
|
||||
if sourceKey == "" {
|
||||
continue
|
||||
@@ -1194,37 +1261,46 @@ WHERE l.protocol = ?
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool) (map[string][]dailySourceLast, error) {
|
||||
func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool, boundaryWhere string) (map[string][]dailySourceLast, error) {
|
||||
rows, err := db.QueryContext(ctx, sqlText)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
latestBySource := map[string]dailySourceLast{}
|
||||
var evidence []sourceBoundaryEvidence
|
||||
for rows.Next() {
|
||||
var vin string
|
||||
var phone string
|
||||
var deviceID string
|
||||
var sourceEndpoint string
|
||||
var firstTS time.Time
|
||||
var firstParsedJSON string
|
||||
var firstRawText string
|
||||
var ts time.Time
|
||||
var parsedJSON string
|
||||
var rawText string
|
||||
var rawSampleCount int64
|
||||
var e sourceBoundaryEvidence
|
||||
if includeFirst {
|
||||
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &firstRawText, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
|
||||
if err := rows.Scan(&e.vin, &e.phone, &e.deviceID, &e.endpoint, &e.firstTS, &e.firstParsed, &e.firstRaw, &e.lastTS, &e.lastParsed, &e.lastRaw, &e.count, &e.minimumMS, &e.maximumMS); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
|
||||
if err := rows.Scan(&e.vin, &e.phone, &e.deviceID, &e.endpoint, &e.lastTS, &e.lastParsed, &e.lastRaw, &e.count, &e.maximumMS); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
firstTS = ts
|
||||
firstParsedJSON = parsedJSON
|
||||
firstRawText = rawText
|
||||
e.minimumMS = e.maximumMS
|
||||
}
|
||||
evidence = append(evidence, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Release the aggregate cursor before resolving exceptional event-time
|
||||
// boundaries; this also works with a single TDengine connection.
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
latestBySource := map[string]dailySourceLast{}
|
||||
for _, e := range evidence {
|
||||
if err := correctSourceEventBoundaries(ctx, db, cfg, protocol, boundaryWhere, includeFirst, &e); err != nil {
|
||||
if errors.Is(err, errNoUsableSourceBoundary) {
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
vin, phone, deviceID, sourceEndpoint := e.vin, e.phone, e.deviceID, e.endpoint
|
||||
firstTS, firstParsedJSON, firstRawText := e.firstTS, e.firstParsed, e.firstRaw
|
||||
ts, parsedJSON, rawText, rawSampleCount := e.lastTS, e.lastParsed, e.lastRaw, e.count
|
||||
latestTotalKM, ok := mileageFromEvidence(protocol, vin, parsedJSON, rawText, ts, cfg.Location)
|
||||
if !ok {
|
||||
continue
|
||||
@@ -1235,7 +1311,7 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
|
||||
firstTotalKM = parsedFirst
|
||||
}
|
||||
}
|
||||
sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint)
|
||||
sourceCode, platformName, sourceKind := backfillSourceMetadata(cfg, protocol, sourceEndpoint)
|
||||
sourceKey := normalizedSourceKeyForSource(string(protocol), phone, deviceID, sourceEndpoint, sourceKind, sourceCode)
|
||||
if sourceKey == "" {
|
||||
continue
|
||||
@@ -1256,9 +1332,10 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
|
||||
RawSampleCount: rawSampleCount,
|
||||
}
|
||||
key := row.VIN + "|" + row.SourceKey
|
||||
if existing, ok := latestBySource[key]; !ok || row.TS.After(existing.TS) {
|
||||
latestBySource[key] = row
|
||||
if existing, ok := latestBySource[key]; ok {
|
||||
row = mergeDailySourceRows(existing, row)
|
||||
}
|
||||
latestBySource[key] = row
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
@@ -1731,6 +1808,9 @@ func resetStats(ctx context.Context, db *sql.DB, cfg config) (int64, error) {
|
||||
}
|
||||
clauses = append(clauses, "protocol IN ("+strings.Join(placeholders, ",")+")")
|
||||
}
|
||||
if predicate := backfillVINPredicate(cfg, "vin"); predicate != "" {
|
||||
clauses = append(clauses, predicate)
|
||||
}
|
||||
var deleted int64
|
||||
for _, table := range []string{"vehicle_daily_mileage_source", "vehicle_daily_mileage"} {
|
||||
result, err := db.ExecContext(ctx, "DELETE FROM "+table+" WHERE "+strings.Join(clauses, " AND "), args...)
|
||||
|
||||
@@ -302,23 +302,21 @@ func TestBuildLastDiffAggregatesRetainsBoundaryAfterEmptyDays(t *testing.T) {
|
||||
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
|
||||
currentRows := func() *sqlmock.Rows {
|
||||
return sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
})
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MIN_EVENT_MS", "MAX_EVENT_MS"})
|
||||
}
|
||||
previousRows := func() *sqlmock.Rows {
|
||||
return sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
})
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MAX_EVENT_MS"})
|
||||
}
|
||||
|
||||
mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(previousRows())
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", int64(4)))
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", int64(4), dayOneTS.UnixMilli(), dayOneTS.UnixMilli()))
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows())
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", int64(6)))
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", int64(6), dayThreeTS.UnixMilli(), dayThreeTS.UnixMilli()))
|
||||
|
||||
aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
@@ -632,20 +630,7 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC6R1004108",
|
||||
"",
|
||||
"LMRKH9AC6R1004108",
|
||||
"mqtt://yutong/ytforward/shln/3",
|
||||
firstTS,
|
||||
`{"yutong_mqtt.data.latitude":"30.0"}`,
|
||||
`{"data":{"TOTAL_MILEAGE":65422000}}`,
|
||||
lastTS,
|
||||
`{"yutong_mqtt.data.latitude":"30.1"}`,
|
||||
`{"data":{"TOTAL_MILEAGE":65423000}}`,
|
||||
int64(42),
|
||||
))
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MIN_EVENT_MS", "MAX_EVENT_MS"}).AddRow("LMRKH9AC6R1004108", "", "LMRKH9AC6R1004108", "mqtt://yutong/ytforward/shln/3", firstTS, `{"yutong_mqtt.data.latitude":"30.0"}`, `{"data":{"TOTAL_MILEAGE":65422000}}`, lastTS, `{"yutong_mqtt.data.latitude":"30.1"}`, `{"data":{"TOTAL_MILEAGE":65423000}}`, int64(42), firstTS.UnixMilli(), lastTS.UnixMilli()))
|
||||
|
||||
rows, err := queryDailyLastSourceRows(context.Background(), db, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
@@ -677,17 +662,7 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC6R1004108",
|
||||
"",
|
||||
"LMRKH9AC6R1004108",
|
||||
"mqtt://yutong/ytforward/shln/3",
|
||||
lastTS,
|
||||
`{"yutong_mqtt.data.latitude":"30.0"}`,
|
||||
`{"data":{"TOTAL_MILEAGE":65377000}}`,
|
||||
int64(31),
|
||||
))
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MAX_EVENT_MS"}).AddRow("LMRKH9AC6R1004108", "", "LMRKH9AC6R1004108", "mqtt://yutong/ytforward/shln/3", lastTS, `{"yutong_mqtt.data.latitude":"30.0"}`, `{"data":{"TOTAL_MILEAGE":65377000}}`, int64(31), lastTS.UnixMilli()))
|
||||
|
||||
rows, err := queryPreviousLastSourceRows(context.Background(), db, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
@@ -735,17 +710,7 @@ func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone
|
||||
utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC7R1004098",
|
||||
"",
|
||||
"LMRKH9AC7R1004098",
|
||||
"mqtt://yutong/ytforward/shln/4",
|
||||
utcInstant,
|
||||
`{"yutong_mqtt.data.total_mileage":"41249000"}`,
|
||||
"",
|
||||
int64(1),
|
||||
))
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)", "MAX_EVENT_MS"}).AddRow("LMRKH9AC7R1004098", "", "LMRKH9AC7R1004098", "mqtt://yutong/ytforward/shln/4", utcInstant, `{"yutong_mqtt.data.total_mileage":"41249000"}`, "", int64(1), utcInstant.UnixMilli()))
|
||||
|
||||
rows, err := queryPreviousLastSourceRows(context.Background(), db, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
@@ -866,106 +831,6 @@ func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.MatchExpectationsInOrder(true)
|
||||
|
||||
projNow := time.Date(2026, 7, 8, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
aggregates := map[string]*metricAgg{
|
||||
"LA9GG64L7PBAF4001|2026-07-08|JT808|JT808:13307765812@115.231.168.135": {
|
||||
VIN: "LA9GG64L7PBAF4001",
|
||||
Date: "2026-07-08",
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
FirstKM: 4100,
|
||||
LatestKM: 4101,
|
||||
Count: 1,
|
||||
SourceKey: "JT808:13307765812@115.231.168.135",
|
||||
Phone: "13307765812",
|
||||
SourceEndpoint: "115.231.168.135:20215",
|
||||
FirstEventTime: projNow,
|
||||
LatestEventTime: projNow,
|
||||
QualityStatus: stats.QualityInvalidDelta,
|
||||
QualityReason: "outside_daily_range",
|
||||
},
|
||||
}
|
||||
|
||||
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage_source WHERE vin = \? AND stat_date = \? AND protocol = \?`).
|
||||
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage WHERE vin = \? AND stat_date = \? AND protocol = \?`).
|
||||
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`INSERT INTO vehicle_data_source`).
|
||||
WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`).
|
||||
WithArgs(
|
||||
"LA9GG64L7PBAF4001",
|
||||
"2026-07-08",
|
||||
"JT808",
|
||||
"JT808:13307765812@115.231.168.135",
|
||||
"115.231.168.135",
|
||||
"115.231.168.135:20215",
|
||||
"13307765812",
|
||||
"",
|
||||
"",
|
||||
float64(4100),
|
||||
float64(4101),
|
||||
float64(1),
|
||||
float64(0),
|
||||
int64(1),
|
||||
int64(0),
|
||||
projNow,
|
||||
projNow,
|
||||
false,
|
||||
false,
|
||||
stats.QualityInvalidDelta,
|
||||
"outside_daily_range",
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`).
|
||||
WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(2500)).
|
||||
WillReturnResult(sqlmock.NewResult(1, 0))
|
||||
mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`).
|
||||
WithArgs(
|
||||
"LA9GG64L7PBAF4001",
|
||||
"2026-07-08",
|
||||
"JT808",
|
||||
int64(2500),
|
||||
"LA9GG64L7PBAF4001",
|
||||
"2026-07-08",
|
||||
"JT808",
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec(`DELETE FROM vehicle_daily_mileage`).
|
||||
WithArgs(
|
||||
"LA9GG64L7PBAF4001",
|
||||
"2026-07-08",
|
||||
"JT808",
|
||||
"LA9GG64L7PBAF4001",
|
||||
"2026-07-08",
|
||||
"JT808",
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
written, err := writeAggregates(context.Background(), db, aggregates, 500)
|
||||
if err != nil {
|
||||
t.Fatalf("writeAggregates() error = %v", err)
|
||||
}
|
||||
if written != 1 {
|
||||
t.Fatalf("written = %d, want 1", written)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteAggregatesSkipsBlankSourceBeforeClearingTarget(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
// Rebuild each GPS vehicle-day atomically. Retained RAW is not guaranteed to
|
||||
// contain every source or a later sample already accepted by the live writer.
|
||||
// Preserve those durable candidates instead of deleting the complete day.
|
||||
func writeProtectedGPSAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*metricAgg) (int64, error) {
|
||||
groups := map[string][]*metricAgg{}
|
||||
for _, agg := range aggregates {
|
||||
if strings.TrimSpace(stats.NormalizeSourceIP(agg.SourceEndpoint)) == "" {
|
||||
continue
|
||||
}
|
||||
key := agg.VIN + "|" + agg.Date
|
||||
groups[key] = append(groups[key], agg)
|
||||
}
|
||||
keys := make([]string, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var written int64
|
||||
for _, key := range keys {
|
||||
group := groups[key]
|
||||
sort.Slice(group, func(i, j int) bool { return group[i].SourceKey < group[j].SourceKey })
|
||||
n, err := writeProtectedGPSDay(ctx, db, group)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
written += n
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func writeProtectedGPSDay(ctx context.Context, db *sql.DB, group []*metricAgg) (int64, error) {
|
||||
first := group[0]
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
existing, selectedManual, err := lockGPSCandidates(ctx, tx, first.VIN, first.Date)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var written int64
|
||||
for _, agg := range group {
|
||||
candidate := backfillCandidate(agg)
|
||||
if old, ok := existing[candidate.SourceKey]; ok {
|
||||
if isManualGPSCandidate(old) {
|
||||
continue
|
||||
}
|
||||
candidate = mergeProtectedGPSCandidate(old, candidate)
|
||||
}
|
||||
identity := stats.SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: candidate.SourceIP,
|
||||
SourceEndpoint: candidate.SourceEndpoint, SourceCode: agg.SourceCode, PlatformName: agg.PlatformName, SourceKind: agg.SourceKind}
|
||||
if candidate.QualityReason != stats.QualityReasonGPSCoordinate {
|
||||
if err := stats.UpsertDataSource(ctx, tx, identity, candidate.LatestEventTime); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := replaceBackfillCandidate(ctx, tx, candidate); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written++
|
||||
}
|
||||
// An explicitly selected imported report is a business override. Keep that
|
||||
// final result while still correcting the underlying automatic candidates.
|
||||
if !selectedManual {
|
||||
if err := stats.ProjectDailyMileage(ctx, tx, first.VIN, first.Date, envelope.ProtocolJT808); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func normalizeProtectedGPSDay(ctx context.Context, db *sql.DB, vin, date string) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
_, manual, err := lockGPSCandidates(ctx, tx, vin, date)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if manual {
|
||||
return tx.Commit()
|
||||
}
|
||||
if err := stats.NormalizePlatformSourceMileage(ctx, tx, vin, date, envelope.ProtocolJT808); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := stats.ProjectDailyMileage(ctx, tx, vin, date, envelope.ProtocolJT808); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func isManualGPSCandidate(c stats.SourceMileageSample) bool {
|
||||
return strings.HasPrefix(c.QualityReason, "manual") || strings.HasPrefix(c.SourceIP, "manual") || strings.HasPrefix(c.SourceIP, "legacy-mysql")
|
||||
}
|
||||
|
||||
func backfillCandidate(agg *metricAgg) stats.SourceMileageSample {
|
||||
c := stats.SourceMileageSample{
|
||||
VIN: agg.VIN, StatDate: agg.Date, Protocol: agg.Protocol, SourceKey: agg.SourceKey,
|
||||
SourceIP: stats.NormalizeSourceIP(agg.SourceEndpoint), SourceEndpoint: agg.SourceEndpoint,
|
||||
Phone: agg.Phone, DeviceID: agg.DeviceID, PlatformName: agg.PlatformName,
|
||||
FirstTotalKM: agg.FirstKM, LatestTotalKM: agg.LatestKM,
|
||||
DailyKM: stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM),
|
||||
PureHydrogenMileageKM: agg.PureHydrogenMileageKM, SampleCount: agg.Count,
|
||||
PureHydrogenSampleCount: agg.PureHydrogenSampleCount,
|
||||
FirstEventTime: agg.FirstEventTime, LatestEventTime: agg.LatestEventTime,
|
||||
LatestPureHydrogenActive: agg.LatestPureHydrogenActive, LatestPureHydrogenModeKnown: agg.LatestPureHydrogenModeKnown,
|
||||
QualityStatus: agg.QualityStatus, QualityReason: agg.QualityReason,
|
||||
}
|
||||
stats.ApplyMileageQualityRules(&c)
|
||||
return c
|
||||
}
|
||||
|
||||
func mergeProtectedGPSCandidate(old, next stats.SourceMileageSample) stats.SourceMileageSample {
|
||||
if next.QualityReason == stats.QualityReasonGPSCoordinate || old.QualityReason == stats.QualityReasonGPSCoordinate {
|
||||
if old.LatestEventTime.After(next.LatestEventTime) {
|
||||
return old
|
||||
}
|
||||
return next
|
||||
}
|
||||
if old.LatestEventTime.After(next.LatestEventTime) {
|
||||
slog.Info("preserved newer durable GPS end", "vin", next.VIN, "date", next.StatDate,
|
||||
"source_key", next.SourceKey, "raw_end", next.LatestEventTime, "durable_end", old.LatestEventTime)
|
||||
next.LatestEventTime, next.LatestTotalKM = old.LatestEventTime, old.LatestTotalKM
|
||||
next.SourceEndpoint = old.SourceEndpoint
|
||||
next.LatestPureHydrogenActive, next.LatestPureHydrogenModeKnown = old.LatestPureHydrogenActive, old.LatestPureHydrogenModeKnown
|
||||
}
|
||||
// Prefer the newest prior-day boundary; otherwise retain the earliest
|
||||
// current-day sample. Always move the timestamp and odometer together.
|
||||
if preferDurableGPSFirst(old.FirstEventTime, next.FirstEventTime, next.StatDate) {
|
||||
next.FirstEventTime, next.FirstTotalKM = old.FirstEventTime, old.FirstTotalKM
|
||||
}
|
||||
if old.SampleCount > next.SampleCount {
|
||||
next.SampleCount = old.SampleCount
|
||||
}
|
||||
if old.PureHydrogenSampleCount > next.PureHydrogenSampleCount {
|
||||
next.PureHydrogenSampleCount = old.PureHydrogenSampleCount
|
||||
}
|
||||
if old.PureHydrogenMileageKM > next.PureHydrogenMileageKM {
|
||||
next.PureHydrogenMileageKM = old.PureHydrogenMileageKM
|
||||
}
|
||||
next.DailyKM = stats.DailyMileageFromDayBoundary(next.FirstTotalKM, next.LatestTotalKM)
|
||||
next.QualityStatus = stats.QualityOK
|
||||
if next.FirstEventTime.Format("2006-01-02") < next.StatDate {
|
||||
next.QualityReason = stats.QualityReasonHistorical
|
||||
} else {
|
||||
next.QualityReason = stats.QualityReasonCurrentDayFirst
|
||||
}
|
||||
stats.ApplyMileageQualityRules(&next)
|
||||
return next
|
||||
}
|
||||
|
||||
func preferDurableGPSFirst(old, next time.Time, date string) bool {
|
||||
if old.IsZero() {
|
||||
return false
|
||||
}
|
||||
if next.IsZero() {
|
||||
return true
|
||||
}
|
||||
oldBefore, nextBefore := old.Format("2006-01-02") < date, next.Format("2006-01-02") < date
|
||||
if oldBefore != nextBefore {
|
||||
return oldBefore
|
||||
}
|
||||
if oldBefore {
|
||||
return old.After(next)
|
||||
}
|
||||
return old.Before(next)
|
||||
}
|
||||
|
||||
func lockGPSCandidates(ctx context.Context, tx *sql.Tx, vin, date string) (map[string]stats.SourceMileageSample, bool, error) {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT source_key,source_ip,COALESCE(source_endpoint,''),COALESCE(phone,''),COALESCE(device_id,''),COALESCE(platform_name,''),
|
||||
first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,pure_hydrogen_mileage_km,sample_count,pure_hydrogen_sample_count,
|
||||
first_event_time,latest_event_time,latest_pure_hydrogen_active,latest_pure_hydrogen_mode_known,quality_status,COALESCE(quality_reason,''),is_selected
|
||||
FROM vehicle_daily_mileage_source WHERE vin=? AND stat_date=? AND protocol='JT808' FOR UPDATE`, vin, date)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]stats.SourceMileageSample{}
|
||||
manual := false
|
||||
for rows.Next() {
|
||||
c := stats.SourceMileageSample{VIN: vin, StatDate: date, Protocol: envelope.ProtocolJT808}
|
||||
var first, last sql.NullTime
|
||||
var firstKM, lastKM sql.NullFloat64
|
||||
var selected bool
|
||||
if err := rows.Scan(&c.SourceKey, &c.SourceIP, &c.SourceEndpoint, &c.Phone, &c.DeviceID, &c.PlatformName,
|
||||
&firstKM, &lastKM, &c.DailyKM, &c.PureHydrogenMileageKM, &c.SampleCount, &c.PureHydrogenSampleCount,
|
||||
&first, &last, &c.LatestPureHydrogenActive, &c.LatestPureHydrogenModeKnown, &c.QualityStatus, &c.QualityReason, &selected); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
c.FirstEventTime, c.LatestEventTime = first.Time, last.Time
|
||||
c.FirstTotalKM, c.LatestTotalKM = firstKM.Float64, lastKM.Float64
|
||||
out[c.SourceKey] = c
|
||||
if selected && isManualGPSCandidate(c) {
|
||||
manual = true
|
||||
}
|
||||
}
|
||||
return out, manual, rows.Err()
|
||||
}
|
||||
|
||||
func replaceBackfillCandidate(ctx context.Context, tx *sql.Tx, c stats.SourceMileageSample) error {
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO vehicle_daily_mileage_source
|
||||
(vin,stat_date,protocol,source_key,source_ip,source_endpoint,phone,device_id,platform_name,
|
||||
first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,pure_hydrogen_mileage_km,
|
||||
sample_count,pure_hydrogen_sample_count,first_event_time,latest_event_time,
|
||||
latest_pure_hydrogen_active,latest_pure_hydrogen_mode_known,quality_status,quality_reason)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON DUPLICATE KEY UPDATE source_ip=VALUES(source_ip),source_endpoint=VALUES(source_endpoint),phone=VALUES(phone),device_id=VALUES(device_id),
|
||||
platform_name=VALUES(platform_name),first_total_mileage_km=VALUES(first_total_mileage_km),latest_total_mileage_km=VALUES(latest_total_mileage_km),
|
||||
daily_mileage_km=VALUES(daily_mileage_km),pure_hydrogen_mileage_km=VALUES(pure_hydrogen_mileage_km),
|
||||
sample_count=VALUES(sample_count),pure_hydrogen_sample_count=VALUES(pure_hydrogen_sample_count),
|
||||
first_event_time=VALUES(first_event_time),latest_event_time=VALUES(latest_event_time),
|
||||
latest_pure_hydrogen_active=VALUES(latest_pure_hydrogen_active),latest_pure_hydrogen_mode_known=VALUES(latest_pure_hydrogen_mode_known),
|
||||
quality_status=VALUES(quality_status),quality_reason=VALUES(quality_reason),updated_at=CURRENT_TIMESTAMP`,
|
||||
c.VIN, c.StatDate, string(c.Protocol), c.SourceKey, c.SourceIP, c.SourceEndpoint, c.Phone, c.DeviceID, c.PlatformName,
|
||||
c.FirstTotalKM, c.LatestTotalKM, c.DailyKM, c.PureHydrogenMileageKM, c.SampleCount, c.PureHydrogenSampleCount,
|
||||
c.FirstEventTime.Truncate(time.Second), c.LatestEventTime.Truncate(time.Second),
|
||||
c.LatestPureHydrogenActive, c.LatestPureHydrogenModeKnown, c.QualityStatus, c.QualityReason)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mysql "github.com/go-sql-driver/mysql"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
// A lost session must never reconnect without its temporary shadow tables.
|
||||
type oneSessionConnector struct {
|
||||
driver.Connector
|
||||
used bool
|
||||
}
|
||||
|
||||
func (c *oneSessionConnector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
if c.used {
|
||||
return nil, errors.New("temporary-table integration session cannot reconnect")
|
||||
}
|
||||
c.used = true
|
||||
return c.Connector.Connect(ctx)
|
||||
}
|
||||
|
||||
func TestProtectedGPSWriteMySQLTemporaryTables(t *testing.T) {
|
||||
dsn := os.Getenv("BACKFILL_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("set BACKFILL_TEST_MYSQL_DSN for isolated temporary-table integration")
|
||||
}
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal("invalid integration DSN")
|
||||
}
|
||||
cfg.ParseTime = true
|
||||
cfg.Loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
connector, err := mysql.NewConnector(cfg)
|
||||
if err != nil {
|
||||
t.Fatal("integration connector unavailable")
|
||||
}
|
||||
db := sql.OpenDB(&oneSessionConnector{Connector: connector})
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
defer db.Close()
|
||||
ctx := context.Background()
|
||||
for _, table := range []string{"vehicle_daily_mileage_source", "vehicle_daily_mileage", "vehicle_data_source"} {
|
||||
if _, err := db.ExecContext(ctx, "CREATE TEMPORARY TABLE gps_fixture_"+table+" LIKE "+table); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "CREATE TEMPORARY TABLE "+table+" LIKE gps_fixture_"+table); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loc := cfg.Loc
|
||||
prev := time.Date(2026, 10, 20, 23, 59, 0, 0, loc)
|
||||
end := time.Date(2026, 10, 21, 23, 59, 0, 0, loc)
|
||||
const vin = "LTESTGPSPROTECT001"
|
||||
key := "JT808:fixture@PLATFORM:g7s"
|
||||
if _, err := db.Exec(`INSERT INTO vehicle_daily_mileage_source
|
||||
(vin,stat_date,protocol,source_key,source_ip,source_endpoint,phone,platform_name,first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,sample_count,first_event_time,latest_event_time,quality_status,quality_reason)
|
||||
VALUES (?,'2026-10-21','JT808',?,'1.2.3.4','1.2.3.4:1','fixture','G7s',100,120,20,200,?,?,'OK','historical_source_baseline'),
|
||||
(?,'2026-10-21','JT808','JT808:other@PLATFORM:other','2.3.4.5','2.3.4.5:2','other','Other',100,101,1,3,?,?,'OK','historical_source_baseline')`, vin, key, prev, end, vin, prev, end.Add(-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Keep the existing manual final selected. This exercises the protected
|
||||
// write path without MySQL's temporary-table self-join restriction in the
|
||||
// independent projection query (covered separately by replay verification).
|
||||
if _, err := db.Exec(`INSERT INTO vehicle_daily_mileage_source (vin,stat_date,protocol,source_key,source_ip,daily_mileage_km,sample_count,quality_status,quality_reason,is_selected)
|
||||
VALUES (?,'2026-10-21','JT808','manual-report','manual-import',88,1,'OK','manual_gps_report',1)`, vin); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO vehicle_daily_mileage (vin,stat_date,protocol,source_id,daily_mileage_km) VALUES (?,'2026-10-21','JT808',0,88)`, vin); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
agg := &metricAgg{VIN: vin, Date: "2026-10-21", Protocol: envelope.ProtocolJT808, SourceKey: key, Phone: "fixture", SourceEndpoint: "1.2.3.4:3", SourceCode: "g7s", SourceKind: "PLATFORM", PlatformName: "G7s", FirstKM: 90, LatestKM: 115, Count: 100, FirstEventTime: prev.Add(-24 * time.Hour), LatestEventTime: end.Add(-time.Hour), QualityStatus: "OK", QualityReason: "historical_source_baseline"}
|
||||
read := func() []any {
|
||||
var first, last, daily float64
|
||||
var count int
|
||||
var firstTS, lastTS time.Time
|
||||
if err := db.QueryRow(`SELECT first_total_mileage_km,latest_total_mileage_km,daily_mileage_km,sample_count,first_event_time,latest_event_time FROM vehicle_daily_mileage_source WHERE vin=? AND source_key=?`, vin, key).Scan(&first, &last, &daily, &count, &firstTS, &lastTS); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return []any{first, last, daily, count, firstTS, lastTS}
|
||||
}
|
||||
var before []any
|
||||
for run := 0; run < 2; run++ {
|
||||
if _, err := writeAggregates(ctx, db, map[string]*metricAgg{"fixture": agg}, 500); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := read()
|
||||
if got[0] != float64(100) || got[1] != float64(120) || got[2] != float64(20) || got[3] != 200 {
|
||||
t.Fatalf("durable data overwritten: %+v", got)
|
||||
}
|
||||
if run == 1 && !reflect.DeepEqual(before, got) {
|
||||
t.Fatalf("repeated backfill changed business values: before=%+v after=%+v", before, got)
|
||||
}
|
||||
before = got
|
||||
}
|
||||
var sources int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM vehicle_daily_mileage_source WHERE vin=?", vin).Scan(&sources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sources != 3 {
|
||||
t.Fatalf("missing durable alternate source: %d", sources)
|
||||
}
|
||||
// Newer event with a lower counter wins by time, not MAX(odometer).
|
||||
agg.FirstKM, agg.FirstEventTime = 100, prev
|
||||
agg.LatestKM, agg.LatestEventTime = 119, end.Add(time.Second)
|
||||
if _, err := writeAggregates(ctx, db, map[string]*metricAgg{"fixture": agg}, 500); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := read(); got[1] != float64(119) || got[2] != float64(19) {
|
||||
t.Fatalf("newer event rejected: %+v", got)
|
||||
}
|
||||
agg.LatestKM, agg.LatestEventTime = 121, end.Add(2*time.Second)
|
||||
if _, err := writeAggregates(ctx, db, map[string]*metricAgg{"fixture": agg}, 500); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var final float64
|
||||
if err := db.QueryRow("SELECT daily_mileage_km FROM vehicle_daily_mileage WHERE vin=?", vin).Scan(&final); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if final != 88 {
|
||||
t.Fatalf("manual final overwritten: %v", final)
|
||||
}
|
||||
t.Log("temporary-table scenarios passed: incomplete RAW, alternate source retention, repeated backfill, event ordering, manual final; production rows modified=0")
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func TestDurableGPSBoundariesSurviveIncompleteRawAndRepeatedBackfill(t *testing.T) {
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
previous := time.Date(2026, 10, 20, 23, 59, 0, 0, loc)
|
||||
last := time.Date(2026, 10, 21, 23, 59, 0, 0, loc)
|
||||
old := stats.SourceMileageSample{VIN: "VIN", StatDate: "2026-10-21", Protocol: envelope.ProtocolJT808, SourceKey: "JT808:phone@PLATFORM:g7s", SourceEndpoint: "1.2.3.4:1", FirstEventTime: previous, FirstTotalKM: 100, LatestEventTime: last, LatestTotalKM: 120, DailyKM: 20, SampleCount: 200, QualityStatus: stats.QualityOK, QualityReason: stats.QualityReasonHistorical}
|
||||
partial := old
|
||||
partial.FirstEventTime, partial.FirstTotalKM = previous.Add(-24*time.Hour), 90
|
||||
partial.LatestEventTime, partial.LatestTotalKM = last.Add(-time.Hour), 115
|
||||
partial.SampleCount = 100
|
||||
for i := 0; i < 3; i++ {
|
||||
got := mergeProtectedGPSCandidate(old, partial)
|
||||
if !reflect.DeepEqual(got, old) {
|
||||
t.Fatalf("run %d replaced durable boundaries or double-counted: got=%+v old=%+v", i, got, old)
|
||||
}
|
||||
old = got
|
||||
}
|
||||
newer := partial
|
||||
newer.FirstEventTime, newer.FirstTotalKM = previous, 100
|
||||
newer.LatestEventTime, newer.LatestTotalKM = last.Add(time.Minute), 119
|
||||
got := mergeProtectedGPSCandidate(old, newer)
|
||||
if got.LatestTotalKM != 119 || got.DailyKM != 19 {
|
||||
t.Fatalf("event ordering incorrectly replaced with maximum odometer: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillCurrentDayFirstBoundaryRetainsEarliestEvent(t *testing.T) {
|
||||
day := time.Date(2026, 10, 21, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
if !preferDurableGPSFirst(day, day.Add(time.Hour), "2026-10-21") {
|
||||
t.Fatal("earliest current-day event lost")
|
||||
}
|
||||
if preferDurableGPSFirst(day.Add(-48*time.Hour), day.Add(-24*time.Hour), "2026-10-21") {
|
||||
t.Fatal("older previous-day baseline won")
|
||||
}
|
||||
}
|
||||
|
||||
func gpsLockedRows() *sqlmock.Rows {
|
||||
return sqlmock.NewRows([]string{"source_key", "source_ip", "endpoint", "phone", "device", "platform", "first_km", "last_km", "daily_km", "hydrogen_km", "count", "hydrogen_count", "first_time", "last_time", "active", "known", "status", "reason", "selected"})
|
||||
}
|
||||
|
||||
func TestGPSBackfillKeepsManualFinalAndRollsBackPartialWrites(t *testing.T) {
|
||||
for _, fail := range []bool{false, true} {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stamp := time.Date(2026, 10, 21, 23, 59, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600))
|
||||
agg := &metricAgg{VIN: "VIN", Date: "2026-10-21", Protocol: envelope.ProtocolJT808, SourceKey: "JT808:phone@PLATFORM:g7s", SourceEndpoint: "1.2.3.4:123", FirstKM: 100, LatestKM: 120, FirstEventTime: stamp.Add(-24 * time.Hour), LatestEventTime: stamp, Count: 5, QualityStatus: stats.QualityOK, QualityReason: stats.QualityReasonHistorical}
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("(?s)SELECT source_key.*FOR UPDATE").WithArgs("VIN", "2026-10-21").WillReturnRows(gpsLockedRows().AddRow("manual-report", "manual-import", "", "", "", "G7s", nil, nil, 88, 0, 1, 0, nil, nil, false, false, "OK", "manual_gps_report", true))
|
||||
mock.ExpectExec("INSERT INTO vehicle_data_source").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
if fail {
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage_source").WillReturnError(errors.New("write failed"))
|
||||
mock.ExpectRollback()
|
||||
} else {
|
||||
mock.ExpectExec("(?s)INSERT INTO vehicle_daily_mileage_source.*sample_count=VALUES\\(sample_count\\)").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
// No clearing other candidates, source-count increment or final
|
||||
// projection is permitted over a selected manual report.
|
||||
mock.ExpectCommit()
|
||||
}
|
||||
n, err := writeAggregates(context.Background(), db, map[string]*metricAgg{"target": agg}, 500)
|
||||
if fail && (err == nil || n != 0) {
|
||||
t.Fatalf("failed target committed: n=%d err=%v", n, err)
|
||||
}
|
||||
if !fail && (err != nil || n != 1) {
|
||||
t.Fatalf("manual override write: n=%d err=%v", n, err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPSNormalizationDoesNotRewriteSelectedManualEvidence(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("(?s)SELECT source_key.*FOR UPDATE").WithArgs("VIN", "2026-07-01").WillReturnRows(gpsLockedRows().AddRow("manual-report", "manual-import", "", "", "", "G7s", nil, nil, 88, 0, 1, 0, nil, nil, false, false, "OK", "manual_gps_report", true))
|
||||
// Do not normalize/delete the selected manual source, lose its selection
|
||||
// flag, or reproject over the manual final result.
|
||||
mock.ExpectCommit()
|
||||
if err := normalizeProtectedGPSDay(context.Background(), db, "VIN", "2026-07-01"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
type backfillSourceID struct {
|
||||
protocol envelope.Protocol
|
||||
ip string
|
||||
}
|
||||
|
||||
// Resolve forwarding endpoints before grouping RAW or looking up durable daily
|
||||
// boundaries. Normalizing only after writing cannot repair an IP-key lookup
|
||||
// that has already missed the newer @PLATFORM boundary.
|
||||
func loadBackfillSourceCatalog(ctx context.Context, db *sql.DB) (map[backfillSourceID]stats.SourceIdentity, error) {
|
||||
rows, err := db.QueryContext(ctx, `SELECT protocol, source_ip,
|
||||
COALESCE(source_code, ''), COALESCE(platform_name, ''), source_kind
|
||||
FROM vehicle_data_source WHERE protocol = 'JT808'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
catalog := map[backfillSourceID]stats.SourceIdentity{}
|
||||
for rows.Next() {
|
||||
var protocol string
|
||||
var source stats.SourceIdentity
|
||||
if err := rows.Scan(&protocol, &source.SourceIP, &source.SourceCode, &source.PlatformName, &source.SourceKind); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
source.Protocol = envelope.Protocol(strings.TrimSpace(protocol))
|
||||
source.SourceIP = stats.NormalizeSourceIP(source.SourceIP)
|
||||
source.SourceCode = strings.TrimSpace(source.SourceCode)
|
||||
source.PlatformName = strings.TrimSpace(source.PlatformName)
|
||||
source.SourceKind = strings.ToUpper(strings.TrimSpace(source.SourceKind))
|
||||
if source.SourceIP != "" {
|
||||
catalog[backfillSourceID{source.Protocol, source.SourceIP}] = source
|
||||
}
|
||||
}
|
||||
return catalog, rows.Err()
|
||||
}
|
||||
|
||||
func backfillSourceMetadata(cfg config, protocol envelope.Protocol, endpoint string) (string, string, string) {
|
||||
if source, ok := cfg.SourceCatalog[backfillSourceID{protocol, stats.NormalizeSourceIP(endpoint)}]; ok {
|
||||
return source.SourceCode, source.PlatformName, source.SourceKind
|
||||
}
|
||||
return knownPlatformSourceMetadata(protocol, endpoint)
|
||||
}
|
||||
|
||||
func mergeDailySourceRows(existing, incoming dailySourceLast) dailySourceLast {
|
||||
latest := existing
|
||||
if incoming.TS.After(existing.TS) {
|
||||
latest = incoming
|
||||
}
|
||||
if incoming.FirstTS.Before(existing.FirstTS) {
|
||||
latest.FirstTS, latest.FirstTotalKM = incoming.FirstTS, incoming.FirstTotalKM
|
||||
} else {
|
||||
latest.FirstTS, latest.FirstTotalKM = existing.FirstTS, existing.FirstTotalKM
|
||||
}
|
||||
latest.RawSampleCount = existing.RawSampleCount + incoming.RawSampleCount
|
||||
return latest
|
||||
}
|
||||
|
||||
func backfillVINPredicate(cfg config, column string) string {
|
||||
if len(cfg.VINs) == 0 {
|
||||
return ""
|
||||
}
|
||||
values := make([]string, 0, len(cfg.VINs))
|
||||
for _, vin := range cfg.VINs {
|
||||
values = append(values, "'"+quote(vin)+"'")
|
||||
}
|
||||
return column + " IN (" + strings.Join(values, ",") + ")"
|
||||
}
|
||||
|
||||
func appendBackfillVINScope(query string, cfg config, column string) string {
|
||||
if predicate := backfillVINPredicate(cfg, column); predicate != "" {
|
||||
query += "\n AND " + predicate
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func logBackfillCandidates(aggregates map[string]*metricAgg) {
|
||||
keys := make([]string, 0, len(aggregates))
|
||||
for key := range aggregates {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
a := aggregates[key]
|
||||
slog.Info("backfill candidate", "vin", a.VIN, "date", a.Date, "protocol", a.Protocol,
|
||||
"source_key", a.SourceKey, "platform", a.PlatformName,
|
||||
"baseline_time", a.FirstEventTime.Format("2006-01-02 15:04:05"), "baseline_km", a.FirstKM,
|
||||
"end_time", a.LatestEventTime.Format("2006-01-02 15:04:05"), "end_km", a.LatestKM,
|
||||
"daily_km", fmt.Sprintf("%.3f", stats.DailyMileageFromDayBoundary(a.FirstKM, a.LatestKM)),
|
||||
"samples", a.Count, "quality", a.QualityStatus)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
||||
)
|
||||
|
||||
func TestBackfillPlatformBoundarySurvivesRawGapAndRepeatedRuns(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
vin, terminal, ip, code string
|
||||
old, previous, end float64
|
||||
}{
|
||||
{"LMRKH9AC9R1004099", "64341233471", "115.159.85.149", "g7s", 62749.7, 64432.7, 65136},
|
||||
{"LMRKH9AC7R1004120", "64341233712", "115.159.85.149", "g7s", 51126.2, 52704.4, 53614.1},
|
||||
{"LMRKH9AC3R1004129", "64341233795", "122.152.221.156", "dongfang_beidou", 57481.7, 59072.7, 59434.6},
|
||||
} {
|
||||
t.Run(tc.vin, func(t *testing.T) {
|
||||
mysqlDB, mysql, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer mysqlDB.Close()
|
||||
tdDB, td, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tdDB.Close()
|
||||
mysql.ExpectQuery("SELECT protocol, source_ip").WillReturnRows(sqlmock.NewRows([]string{"protocol", "ip", "code", "name", "kind"}).
|
||||
AddRow("JT808", tc.ip, tc.code, tc.code, "PLATFORM"))
|
||||
catalog, err := loadBackfillSourceCatalog(context.Background(), mysqlDB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
oldTS := time.Date(2026, 10, 16, 15, 26, 0, 0, loc)
|
||||
previousTS := time.Date(2026, 10, 18, 23, 59, 0, 0, loc)
|
||||
firstTS := time.Date(2026, 10, 19, 11, 0, 0, 0, loc)
|
||||
endTS := time.Date(2026, 10, 19, 23, 59, 0, 0, loc)
|
||||
parsed := func(km float64) string { return fmt.Sprintf(`{"jt808.location.total_mileage_km":%f}`, km) }
|
||||
key := "JT808:" + tc.terminal + "@PLATFORM:" + tc.code
|
||||
cfg := config{DateFrom: "2026-10-19", DateTo: "2026-10-19", Protocols: []envelope.Protocol{envelope.ProtocolJT808},
|
||||
VINs: []string{tc.vin}, TDengineDatabase: "lingniu_vehicle_ts", Location: loc, SourceCatalog: catalog}
|
||||
// Run twice against the same durable October 18 boundary. A replay must
|
||||
// neither use October 16 RAW nor subtract today's already-written result.
|
||||
for run := 0; run < 2; run++ {
|
||||
td.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-10-19 00:00:00'.*vin IN \\('" + tc.vin + "'\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "last", "parsed", "raw", "count", "MAX_EVENT_MS"}).
|
||||
AddRow(tc.vin, tc.terminal, "", tc.ip+":1000", oldTS, parsed(tc.old), "", 10, oldTS.UnixMilli()))
|
||||
td.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-10-19 00:00:00'.*vin IN \\('" + tc.vin + "'\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "phone", "device", "endpoint", "first", "first_parsed", "first_raw", "last", "last_parsed", "last_raw", "count", "MIN_EVENT_MS", "MAX_EVENT_MS"}).
|
||||
AddRow(tc.vin, tc.terminal, "", tc.ip+":2000", firstTS, parsed(tc.previous+1), "", firstTS.Add(time.Hour), parsed(tc.previous+10), "", 100, firstTS.UnixMilli(), firstTS.Add(time.Hour).UnixMilli()).
|
||||
AddRow(tc.vin, tc.terminal, "", tc.ip+":3000", firstTS.Add(2*time.Hour), parsed(tc.previous+11), "", endTS, parsed(tc.end), "", 200, firstTS.Add(2*time.Hour).UnixMilli(), endTS.UnixMilli()))
|
||||
mysql.ExpectQuery("SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source").
|
||||
WithArgs(tc.vin, "2026-10-19", "JT808", key).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total", "time"}).AddRow(tc.previous, previousTS))
|
||||
aggregates, err := buildLastDiffAggregates(context.Background(), mysqlDB, tdDB, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := aggregates[tc.vin+"|2026-10-19|JT808|"+key]
|
||||
if len(aggregates) != 1 || a == nil || a.FirstKM != tc.previous || !a.FirstEventTime.Equal(previousTS) || a.Count != 300 {
|
||||
t.Fatalf("run %d: wrong boundary or connection grouping: %+v", run, a)
|
||||
}
|
||||
if math.Abs((a.LatestKM-a.FirstKM)-(tc.end-tc.previous)) > 0.0001 || a.SourceCode != tc.code {
|
||||
t.Fatalf("run %d: recounted historical mileage: %+v", run, a)
|
||||
}
|
||||
}
|
||||
if err := mysql.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := td.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillSourceCatalogPreservesIndependentAndUnknownCounters(t *testing.T) {
|
||||
cfg := config{SourceCatalog: map[backfillSourceID]stats.SourceIdentity{
|
||||
{envelope.ProtocolJT808, "1.2.3.4"}: {SourceCode: "registered", SourceKind: "PLATFORM"},
|
||||
{envelope.ProtocolJT808, "2.3.4.5"}: {SourceKind: "DIRECT"},
|
||||
}}
|
||||
key := func(protocol envelope.Protocol, terminal, endpoint string) string {
|
||||
code, _, kind := backfillSourceMetadata(cfg, protocol, endpoint)
|
||||
return normalizedSourceKeyForSource(string(protocol), terminal, "", endpoint, kind, code)
|
||||
}
|
||||
for _, tc := range []struct{ terminal, endpoint, want string }{
|
||||
{"terminal-a", "1.2.3.4:123", "JT808:terminal-a@PLATFORM:registered"},
|
||||
{"terminal-b", "1.2.3.4:456", "JT808:terminal-b@PLATFORM:registered"},
|
||||
{"terminal-a", "2.3.4.5:123", "JT808:terminal-a@DIRECT"},
|
||||
{"terminal-a", "3.4.5.6:123", "JT808:terminal-a@3.4.5.6"},
|
||||
} {
|
||||
if got := key(envelope.ProtocolJT808, tc.terminal, tc.endpoint); got != tc.want {
|
||||
t.Fatalf("key=%s want=%s", got, tc.want)
|
||||
}
|
||||
}
|
||||
if got := key(envelope.ProtocolGB32960, "terminal-a", "1.2.3.4:123"); strings.Contains(got, "PLATFORM") {
|
||||
t.Fatalf("JT808 metadata leaked into GB32960: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillSourceCatalogFailureIsNotIgnored(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
mock.ExpectQuery("SELECT protocol, source_ip").WillReturnError(errors.New("database unavailable"))
|
||||
if _, err := loadBackfillSourceCatalog(context.Background(), db); err == nil {
|
||||
t.Fatal("missing source catalog must fail before recomputation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillVINScopeIncludesCleanupAndRealtimeFallback(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
cfg := config{VINs: []string{"ONLY-THIS-VIN"}, DateFrom: "2026-10-19", DateTo: "2026-10-19", Protocols: []envelope.Protocol{envelope.ProtocolJT808}}
|
||||
mock.ExpectQuery("(?s)FROM vehicle_daily_mileage_source.*AND vin IN \\('ONLY-THIS-VIN'\\).*GROUP BY").
|
||||
WithArgs(cfg.DateFrom, cfg.DateTo, stats.QualityReasonGPSCoordinate, stats.GPSMinimumDailyDistanceKM, "JT808").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "date", "protocol", "count"}))
|
||||
if _, _, err := cleanupSubthresholdGPSMileageCandidates(context.Background(), db, cfg, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mock.ExpectQuery("(?s)FROM vehicle_realtime_location.*AND l.vin IN \\('ONLY-THIS-VIN'\\)").
|
||||
WithArgs("YUTONG_MQTT", cfg.DateFrom, cfg.DateFrom).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "time", "total"}))
|
||||
if _, err := queryRealtimeLocationLastRows(context.Background(), db, cfg, envelope.ProtocolYutongMQTT, cfg.DateFrom); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillScopedNormalizationDoesNotScanOtherVehicles(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
vin, date := "ONLY-THIS-VIN", "2026-10-19"
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("(?s)SELECT source_key.*FOR UPDATE").WithArgs(vin, date).WillReturnRows(gpsLockedRows())
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage_source").WithArgs(vin, date, "JT808").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("DELETE s").WithArgs(vin, date, "JT808").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage").WithArgs(vin, date, "JT808", int64(2500)).WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("UPDATE vehicle_daily_mileage_source s").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("DELETE FROM vehicle_daily_mileage").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectCommit()
|
||||
_, err = normalizeHistoricalPlatformSources(context.Background(), db, config{VINs: []string{vin}, DateFrom: date, DateTo: date, Protocols: []envelope.Protocol{envelope.ProtocolJT808}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeDailySourceRowsIsIndependentOfConnectionOrder(t *testing.T) {
|
||||
first := dailySourceLast{FirstTS: time.Unix(100, 0), TS: time.Unix(200, 0), FirstTotalKM: 10, TotalKM: 20, RawSampleCount: 4}
|
||||
last := dailySourceLast{FirstTS: time.Unix(300, 0), TS: time.Unix(400, 0), FirstTotalKM: 30, TotalKM: 40, RawSampleCount: 6}
|
||||
for _, merged := range []dailySourceLast{mergeDailySourceRows(first, last), mergeDailySourceRows(last, first)} {
|
||||
if merged.FirstTotalKM != 10 || merged.TotalKM != 40 || merged.RawSampleCount != 10 {
|
||||
t.Fatalf("lost connection boundary: %+v", merged)
|
||||
}
|
||||
}
|
||||
// Scope quoting is shared by SQL paths, including escaped input.
|
||||
if got := backfillVINPredicate(config{VINs: []string{"A'B"}}, "vin"); got != "vin IN ('A''B')" {
|
||||
t.Fatalf("unexpected predicate: %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
)
|
||||
|
||||
func TestFeichiCustom80UsesConnectionLoginAndSurvivesCanonicalRaw(t *testing.T) {
|
||||
payload, _ := hex.DecodeString("3B1428271014282710024E01560000")
|
||||
body := []byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0, 0x80, 0, 15}
|
||||
body = append(body, payload...)
|
||||
body = append(body, 0x80, 0, 3, 0x05, 0x80, 0x02)
|
||||
body = append(body, 0x05, 0, 0x07, 0x36, 0x50, 0x40, 0x01, 0xd2, 0x4f, 0)
|
||||
frame := buildGBFrame(0x02, 0xfe, "LA9GG68L7PBAF4803", body)
|
||||
for _, tc := range []struct {
|
||||
account string
|
||||
password string
|
||||
wantCustom bool
|
||||
}{
|
||||
{"Feichi", "test-secret", true},
|
||||
{"Feichi", "wrong-secret", false},
|
||||
{"Hyundai", "test-secret", false},
|
||||
{"YueJin", "test-secret", false},
|
||||
} {
|
||||
t.Run(tc.account+"/"+tc.password, func(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960, Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames, Parse: gb32960.ParseFrame,
|
||||
ParseForPlatform: gb32960.ParseFrameForPlatform,
|
||||
Authenticate: authentication.NewGB32960PlatformAuthenticator(authentication.ModeEnforce, map[string][]string{
|
||||
"Feichi": {"test-secret"}, "Hyundai": {"test-secret"}, "YueJin": {"test-secret"},
|
||||
}),
|
||||
}, sink)
|
||||
state := &connectionState{}
|
||||
login := []byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0, 0, 1}
|
||||
login = append(login, fixedASCIIBytes(tc.account, 12)...)
|
||||
login = append(login, fixedASCIIBytes(tc.password, 20)...)
|
||||
login = append(login, 1)
|
||||
server.handleFrame(context.Background(), nil, buildGBFrame(0x05, 0xfe, "", login), "183.239.48.25:2147", state)
|
||||
server.handleFrame(context.Background(), nil, frame, "183.239.48.25:2147", state)
|
||||
if len(sink.raw) != 2 {
|
||||
t.Fatalf("raw frames=%d", len(sink.raw))
|
||||
}
|
||||
raw := sink.raw[1]
|
||||
if raw.RawHex != strings.ToUpper(hex.EncodeToString(frame)) || len(raw.Parsed) != 0 {
|
||||
t.Fatal("canonical whole RAW frame was changed or contains duplicate tree")
|
||||
}
|
||||
prefix := "gb32960.feichi_custom_80.blocks."
|
||||
_, custom := raw.ParsedFields[prefix+"block_1.payload_hex"]
|
||||
if custom != tc.wantCustom {
|
||||
t.Fatalf("account=%s wantCustom=%v fields=%#v", tc.account, tc.wantCustom, raw.ParsedFields)
|
||||
}
|
||||
if !tc.wantCustom {
|
||||
if raw.ParsedFields["gb32960.gd_fc_demo_extension.declared_length"] != "15" {
|
||||
t.Fatal("legacy decoder was not retained")
|
||||
}
|
||||
return
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
prefix + "block_1.declared_length": "15",
|
||||
prefix + "block_1.actual_length": "15",
|
||||
prefix + "block_1.payload_hex": "3B1428271014282710024E01560000",
|
||||
prefix + "block_1.raw_hex": "80000F3B1428271014282710024E01560000",
|
||||
prefix + "block_2.payload_hex": "058002",
|
||||
prefix + "block_2.raw_hex": "800003058002",
|
||||
"gb32960.position.longitude": "121",
|
||||
"gb32960.position.latitude": "30.56",
|
||||
} {
|
||||
if got := fmt.Sprint(raw.ParsedFields[key]); got != want {
|
||||
t.Fatalf("%s=%s want=%s", key, got, want)
|
||||
}
|
||||
}
|
||||
if raw.PlatformName != "Feichi" || raw.ParseStatus != envelope.ParseOK {
|
||||
t.Fatalf("platform=%q status=%s", raw.PlatformName, raw.ParseStatus)
|
||||
}
|
||||
// A fresh TCP connection does not inherit the previous account.
|
||||
server.handleFrame(context.Background(), nil, frame, "183.239.48.25:9999", &connectionState{})
|
||||
if _, exists := sink.raw[2].ParsedFields[prefix+"block_1.payload_hex"]; exists {
|
||||
t.Fatal("Feichi account leaked into a fresh connection")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeichiCustom80TruncationIsPreservedThroughRawProjection(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960, Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames, Parse: gb32960.ParseFrame,
|
||||
ParseForPlatform: gb32960.ParseFrameForPlatform,
|
||||
}, sink)
|
||||
frame := buildGBFrame(0x02, 0xfe, "LA9GG68L7PBAF4803", []byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0, 0x80, 0, 15, 0x3b, 0x14})
|
||||
server.handleFrame(context.Background(), nil, frame, "183.239.48.25:2147", &connectionState{platformName: "Feichi"})
|
||||
raw := sink.raw[0]
|
||||
for key, want := range map[string]string{
|
||||
"gb32960.feichi_custom_80.blocks.block_1.declared_length": "15",
|
||||
"gb32960.feichi_custom_80.blocks.block_1.actual_length": "2",
|
||||
"gb32960.feichi_custom_80.blocks.block_1.payload_hex": "3B14",
|
||||
"gb32960.feichi_custom_80.blocks.block_1.raw_hex": "80000F3B14",
|
||||
"gb32960.feichi_custom_80.blocks.block_1.parse_error": "truncated_payload",
|
||||
} {
|
||||
if got := fmt.Sprint(raw.ParsedFields[key]); got != want {
|
||||
t.Fatalf("%s=%s want=%s", key, got, want)
|
||||
}
|
||||
}
|
||||
if raw.ParseStatus != envelope.ParsePartial || raw.RawHex != strings.ToUpper(hex.EncodeToString(frame)) {
|
||||
t.Fatalf("status=%s raw=%s", raw.ParseStatus, raw.RawHex)
|
||||
}
|
||||
}
|
||||
@@ -25,15 +25,18 @@ type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
|
||||
|
||||
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
|
||||
|
||||
type PlatformFrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint, platformAccount string) (envelope.FrameEnvelope, error)
|
||||
|
||||
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
|
||||
|
||||
type TCPProtocol struct {
|
||||
Protocol envelope.Protocol
|
||||
Addr string
|
||||
Extract FrameExtractor
|
||||
Parse FrameParser
|
||||
Authenticate authentication.Authenticator
|
||||
Respond FrameResponder
|
||||
Protocol envelope.Protocol
|
||||
Addr string
|
||||
Extract FrameExtractor
|
||||
Parse FrameParser
|
||||
ParseForPlatform PlatformFrameParser
|
||||
Authenticate authentication.Authenticator
|
||||
Respond FrameResponder
|
||||
}
|
||||
|
||||
type connectionState struct {
|
||||
@@ -283,7 +286,17 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
|
||||
defer cancelFrame()
|
||||
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := s.protocol.Parse(raw, receivedAtMS, source)
|
||||
var env envelope.FrameEnvelope
|
||||
var err error
|
||||
if s.protocol.ParseForPlatform != nil {
|
||||
platformAccount := ""
|
||||
if state != nil {
|
||||
platformAccount = state.platformName
|
||||
}
|
||||
env, err = s.protocol.ParseForPlatform(raw, receivedAtMS, source, platformAccount)
|
||||
} else {
|
||||
env, err = s.protocol.Parse(raw, receivedAtMS, source)
|
||||
}
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: s.protocol.Protocol,
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const feichiCustom80Payload = "3B1428271014282710024E01560000"
|
||||
|
||||
func TestFeichiCustom80UsesDeclaredLengthAndContinuesWithStandardData(t *testing.T) {
|
||||
for _, command := range []byte{0x02, 0x03} {
|
||||
for _, length := range []int{0, 1, 9, 15, 32, 513} {
|
||||
t.Run(fmt.Sprintf("command_%02X_length_%d", command, length), func(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte{0x80, 0x05, 0x02, 0x23, 0x23}, (length+4)/5)[:length]
|
||||
if length == 15 {
|
||||
payload, _ = hex.DecodeString(feichiCustom80Payload)
|
||||
}
|
||||
body := []byte{0x1a, 0x0a, 0x17, 0x0a, 0x00, 0x00, 0x80, byte(length >> 8), byte(length)}
|
||||
body = append(body, payload...)
|
||||
body = append(body, feichiTestPositionUnit()...)
|
||||
body = append(body, 0x34, 0, 0, 0, 0, 0, 0, 0x2b)
|
||||
frame := buildFrame(command, 0xfe, "LA9GG68L7PBAF4803", body)
|
||||
env, err := ParseFrameForPlatform(frame, 1234, "183.239.48.25:2147", "Feichi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env.ParseStatus != envelope.ParseOK || env.RawHex != strings.ToUpper(hex.EncodeToString(frame)) {
|
||||
t.Fatalf("status=%s raw=%s", env.ParseStatus, env.RawHex)
|
||||
}
|
||||
units := env.Parsed["data_units"].([]map[string]any)
|
||||
if len(units) != 3 || units[0]["name"] != "feichi_custom_80" || units[1]["name"] != "position" || units[2]["name"] != "gd_fc_vehicle_info" {
|
||||
t.Fatalf("wrong unit boundaries: %#v", units)
|
||||
}
|
||||
block := feichiBlock(t, units[0], 1)
|
||||
if block["declared_length"] != length || block["actual_length"] != length || block["payload_hex"] != strings.ToUpper(hex.EncodeToString(payload)) {
|
||||
t.Fatalf("block=%#v", block)
|
||||
}
|
||||
if want := fmt.Sprintf("80%04X%X", length, payload); block["raw_hex"] != want {
|
||||
t.Fatalf("raw=%v want=%s", block["raw_hex"], want)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.56)
|
||||
assertFloatField(t, env, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
|
||||
if _, exists := env.Fields["gd_fc_demo_stack_temp_c"]; exists {
|
||||
t.Fatal("Feichi opaque bytes must not become Guangdong measurement fields")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeichiCustom80TruncationKeepsEveryReceivedByte(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
data []byte
|
||||
error string
|
||||
payload string
|
||||
}{
|
||||
{"missing_length", nil, "truncated_length", ""},
|
||||
{"partial_length", []byte{0}, "truncated_length", ""},
|
||||
{"short_payload", []byte{0, 15, 0x3b, 0x14, 0x28}, "truncated_payload", "3B1428"},
|
||||
{"oversized_length", []byte{0xff, 0xff, 0x80}, "truncated_payload", "80"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := append([]byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0}, feichiTestPositionUnit()...)
|
||||
body = append(body, 0x80)
|
||||
body = append(body, tc.data...)
|
||||
frame := buildFrame(0x02, 0xfe, "LA9GG68L7PBAF4803", body)
|
||||
env, err := ParseFrameForPlatform(frame, 1234, "source", "Feichi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env.ParseStatus != envelope.ParsePartial || !strings.Contains(env.ParseError, tc.error) {
|
||||
t.Fatalf("status=%s error=%q", env.ParseStatus, env.ParseError)
|
||||
}
|
||||
if env.RawHex != strings.ToUpper(hex.EncodeToString(frame)) {
|
||||
t.Fatal("whole RAW frame was changed")
|
||||
}
|
||||
units := env.Parsed["data_units"].([]map[string]any)
|
||||
if len(units) != 2 || units[0]["name"] != "position" {
|
||||
t.Fatalf("units=%#v", units)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121)
|
||||
block := feichiBlock(t, units[1], 1)
|
||||
if block["raw_hex"] != fmt.Sprintf("80%X", tc.data) || block["payload_hex"] != tc.payload || block["parse_error"] != tc.error {
|
||||
t.Fatalf("received bytes not preserved: %#v", block)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformParserPreservesExisting32960Accounts(t *testing.T) {
|
||||
frame, err := hex.DecodeString(realFuelCellFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, err := ParseFrame(frame, 1234, "same-source")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, account := range []string{"", "Hyundai", "YueJin", "GuangJiaoTou", "feichi", "Feichi-other"} {
|
||||
t.Run(account, func(t *testing.T) {
|
||||
got, err := ParseFrameForPlatform(frame, 1234, "same-source", account)
|
||||
if err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("legacy account changed: account=%q err=%v", account, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
assertIntField(t, want, "gd_fc_demo_stack_temp_c", 65)
|
||||
assertFloatField(t, want, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
|
||||
}
|
||||
|
||||
func TestFeichiFramesWithoutCustom80KeepExistingResults(t *testing.T) {
|
||||
for _, command := range []byte{0x02, 0x03} {
|
||||
frame := buildFrame(command, 0xfe, "LA9GG68L7PBAF4803", append([]byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0}, feichiTestPositionUnit()...))
|
||||
want, err := ParseFrame(frame, 1234, "same-source")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := ParseFrameForPlatform(frame, 1234, "same-source", "Feichi")
|
||||
if err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("standard Feichi frame changed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func feichiBlock(t *testing.T, unit map[string]any, index int) map[string]any {
|
||||
t.Helper()
|
||||
return unit["value"].(map[string]any)["blocks"].(map[string]any)[fmt.Sprintf("block_%d", index)].(map[string]any)
|
||||
}
|
||||
|
||||
func feichiTestPositionUnit() []byte {
|
||||
return []byte{0x05, 0x00, 0x07, 0x36, 0x50, 0x40, 0x01, 0xd2, 0x4f, 0x00}
|
||||
}
|
||||
@@ -51,6 +51,17 @@ func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error)
|
||||
}
|
||||
|
||||
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||
return parseFrame(raw, receivedAtMS, sourceEndpoint, false)
|
||||
}
|
||||
|
||||
// ParseFrameForPlatform selects Feichi's opaque, length-prefixed 0x80 format
|
||||
// using the account established by this TCP connection's platform login.
|
||||
// All other accounts retain the existing Guangdong extension decoder.
|
||||
func ParseFrameForPlatform(raw []byte, receivedAtMS int64, sourceEndpoint, platformAccount string) (envelope.FrameEnvelope, error) {
|
||||
return parseFrame(raw, receivedAtMS, sourceEndpoint, strings.TrimSpace(platformAccount) == "Feichi")
|
||||
}
|
||||
|
||||
func parseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string, feichiCustom80 bool) (envelope.FrameEnvelope, error) {
|
||||
if len(raw) < minFrameLen {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
@@ -86,7 +97,7 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
|
||||
eventTimeMS := receivedAtMS
|
||||
|
||||
if command == 0x02 || command == 0x03 {
|
||||
eventTime, units := parseDataBody(version, body, fields)
|
||||
eventTime, units := parseDataBodyWithCustom80(version, body, fields, feichiCustom80)
|
||||
if !eventTime.IsZero() {
|
||||
eventTimeMS = eventTime.UnixMilli()
|
||||
fields["device_time"] = eventTime.Format(time.RFC3339)
|
||||
@@ -117,17 +128,33 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
if feichiCustom80 {
|
||||
if units, ok := parsed["data_units"].([]map[string]any); ok {
|
||||
for _, unit := range units {
|
||||
if unit["name"] == "feichi_custom_80" && unit["error"] != nil {
|
||||
env.ParseStatus = envelope.ParsePartial
|
||||
env.ParseError = "Feichi 0x80: " + fmt.Sprint(unit["error"])
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
|
||||
return parseDataBodyWithCustom80(version, body, fields, false)
|
||||
}
|
||||
|
||||
func parseDataBodyWithCustom80(version string, body []byte, fields map[string]any, feichiCustom80 bool) (time.Time, []map[string]any) {
|
||||
if len(body) < 6 {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
eventTime := parseGBTime(body[:6])
|
||||
cursor := 6
|
||||
var units []map[string]any
|
||||
custom80Blocks := 0
|
||||
for cursor < len(body) {
|
||||
unitType := body[cursor]
|
||||
cursor++
|
||||
@@ -293,6 +320,16 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
|
||||
fields["gd_fc_vehicle_hydrogen_mass_kg"] = unit["hydrogen_mass_kg"]
|
||||
cursor += 7
|
||||
case 0x80:
|
||||
if feichiCustom80 {
|
||||
custom80Blocks++
|
||||
unit, size := parseFeichiCustom80Data(body[cursor:], custom80Blocks)
|
||||
units = append(units, unit)
|
||||
cursor += size
|
||||
if unit["error"] != nil {
|
||||
return eventTime, units
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(body[cursor:]) < 11 {
|
||||
units = append(units, map[string]any{"type": "0x80", "error": "truncated"})
|
||||
return eventTime, units
|
||||
@@ -323,6 +360,43 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
|
||||
return eventTime, units
|
||||
}
|
||||
|
||||
func parseFeichiCustom80Data(data []byte, blockNumber int) (map[string]any, int) {
|
||||
size := len(data)
|
||||
lengthSize := min(2, len(data))
|
||||
payload := []byte(nil)
|
||||
block := map[string]any{
|
||||
"length_hex": strings.ToUpper(hex.EncodeToString(data[:lengthSize])),
|
||||
}
|
||||
parseError := ""
|
||||
if len(data) < 2 {
|
||||
parseError = "truncated_length"
|
||||
} else {
|
||||
declared := int(binary.BigEndian.Uint16(data[:2]))
|
||||
block["declared_length"] = declared
|
||||
size = min(2+declared, len(data))
|
||||
payload = data[2:size]
|
||||
if size < 2+declared {
|
||||
parseError = "truncated_payload"
|
||||
}
|
||||
}
|
||||
block["actual_length"] = len(payload)
|
||||
block["payload_hex"] = strings.ToUpper(hex.EncodeToString(payload))
|
||||
block["raw_hex"] = "80" + strings.ToUpper(hex.EncodeToString(data[:size]))
|
||||
unit := map[string]any{
|
||||
"type": "0x80",
|
||||
"name": "feichi_custom_80",
|
||||
"value": map[string]any{
|
||||
// Distinct keys preserve repeated 0x80 blocks in flattened RAW fields.
|
||||
"blocks": map[string]any{fmt.Sprintf("block_%d", blockNumber): block},
|
||||
},
|
||||
}
|
||||
if parseError != "" {
|
||||
unit["error"] = parseError
|
||||
block["parse_error"] = parseError
|
||||
}
|
||||
return unit, size
|
||||
}
|
||||
|
||||
func frameLength(frameStart []byte, bodyLen int) int {
|
||||
standardLen := headerLen + bodyLen + 1
|
||||
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
|
||||
|
||||
@@ -49,6 +49,7 @@ type Writer struct {
|
||||
lastProjection map[string]projectionCacheEntry
|
||||
pendingProjection map[string]pendingProjectionEntry
|
||||
baselineCache map[string]sourceBaselineCacheEntry
|
||||
sourceIdentities map[string]SourceIdentity
|
||||
mileageKeysByPrefix map[string]map[string]struct{}
|
||||
projectionKeysByPrefix map[string]map[string]struct{}
|
||||
baselineKeysByPrefix map[string]map[string]struct{}
|
||||
@@ -355,6 +356,7 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
|
||||
result.SamplesSkippedMissingFields = 1
|
||||
return result, nil
|
||||
}
|
||||
env = w.withSourceIdentity(env)
|
||||
seenAt := w.sourceSeenAt(env)
|
||||
w.maybeCleanupCaches(seenAt)
|
||||
identity, hasSource := NewSourceIdentityFromEnvelope(env)
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
// ReloadSourceIdentities aligns live JT808 candidates with the durable platform
|
||||
// keys used by backfill and daily statistics. A snapshot avoids a database
|
||||
// lookup per frame; a failed refresh leaves the last successful snapshot intact.
|
||||
func (w *Writer) ReloadSourceIdentities(ctx context.Context) (int, error) {
|
||||
rows, err := w.query.QueryContext(ctx, `SELECT source_ip,COALESCE(source_code,''),COALESCE(platform_name,''),source_kind
|
||||
FROM vehicle_data_source WHERE protocol='JT808'`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
next := map[string]SourceIdentity{}
|
||||
for rows.Next() {
|
||||
var source SourceIdentity
|
||||
if err := rows.Scan(&source.SourceIP, &source.SourceCode, &source.PlatformName, &source.SourceKind); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
source.Protocol = envelope.ProtocolJT808
|
||||
source.SourceIP = NormalizeSourceIP(source.SourceIP)
|
||||
source.SourceCode = strings.TrimSpace(source.SourceCode)
|
||||
source.SourceKind = strings.ToUpper(strings.TrimSpace(source.SourceKind))
|
||||
if source.SourceIP != "" && ((source.SourceKind == "PLATFORM" && source.SourceCode != "") || source.SourceKind == "DIRECT") {
|
||||
next[source.SourceIP] = source
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.sourceIdentities = next
|
||||
w.mu.Unlock()
|
||||
return len(next), nil
|
||||
}
|
||||
|
||||
func (w *Writer) withSourceIdentity(env envelope.FrameEnvelope) envelope.FrameEnvelope {
|
||||
if env.Protocol != envelope.ProtocolJT808 {
|
||||
return env
|
||||
}
|
||||
ip := NormalizeSourceIP(env.SourceEndpoint)
|
||||
w.mu.Lock()
|
||||
source, found := w.sourceIdentities[ip]
|
||||
w.mu.Unlock()
|
||||
if found {
|
||||
env.SourceCode = source.SourceCode
|
||||
env.SourceKind = source.SourceKind
|
||||
if strings.TrimSpace(source.PlatformName) != "" {
|
||||
env.PlatformName = source.PlatformName
|
||||
}
|
||||
}
|
||||
return env
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestWriterResolvesForwardingIPBeforeDurableBaselineLookup(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
writer := NewWriter(db, loc)
|
||||
mock.ExpectQuery("SELECT source_ip,COALESCE").WillReturnRows(sqlmock.NewRows([]string{"ip", "code", "name", "kind"}).AddRow("115.159.85.149", "g7s", "G7s", "PLATFORM"))
|
||||
if _, err := writer.ReloadSourceIdentities(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vin, phone, key := "LMRKH9AC7R1004120", "64341233712", "JT808:64341233712@PLATFORM:g7s"
|
||||
previous := time.Date(2026, 10, 18, 23, 59, 49, 0, loc)
|
||||
current := time.Date(2026, 10, 19, 12, 0, 0, 0, loc)
|
||||
mock.ExpectExec("INSERT INTO vehicle_data_source").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectQuery("SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source").WithArgs(vin, "2026-10-19", "JT808", key).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total", "time"}).AddRow(52704.4, previous))
|
||||
mock.ExpectBegin()
|
||||
args := []driver.Value{vin, "2026-10-19", "JT808", key}
|
||||
for len(args) < 21 {
|
||||
args = append(args, sqlmock.AnyArg())
|
||||
}
|
||||
args[9] = 52704.4
|
||||
args[10] = 52714.4
|
||||
args[11] = 10.0
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage_source").WithArgs(args...).WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage_source").WithArgs(vin, "2026-10-19", "JT808").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("DELETE s").WithArgs(vin, "2026-10-19", "JT808").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("UPDATE vehicle_daily_mileage_source s").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("DELETE FROM vehicle_daily_mileage").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectCommit()
|
||||
fields := map[string]any{"jt808.location.total_mileage_km": 52714.4}
|
||||
_, err = writer.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: vin, Phone: phone, SourceEndpoint: "115.159.85.149:12345", Fields: fields, ParsedFields: fields, EventTimeMS: current.UnixMilli(), ReceivedAtMS: current.UnixMilli(), ParseStatus: envelope.ParseOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceIdentityRefreshRetainsSnapshotOnFailureAndDoesNotMixProtocols(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
writer := NewWriter(db, time.UTC)
|
||||
mock.ExpectQuery("SELECT source_ip,COALESCE").WillReturnRows(sqlmock.NewRows([]string{"ip", "code", "name", "kind"}).AddRow("1.2.3.4", "g7s", "G7s", "PLATFORM").AddRow("2.3.4.5", "", "", "DIRECT").AddRow("3.4.5.6", "", "", "UNKNOWN"))
|
||||
if n, err := writer.ReloadSourceIdentities(context.Background()); err != nil || n != 2 {
|
||||
t.Fatalf("loaded=%d err=%v", n, err)
|
||||
}
|
||||
mock.ExpectQuery("SELECT source_ip,COALESCE").WillReturnError(errors.New("temporary database failure"))
|
||||
if _, err := writer.ReloadSourceIdentities(context.Background()); err == nil {
|
||||
t.Fatal("expected refresh failure")
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
protocol envelope.Protocol
|
||||
endpoint, kind, code string
|
||||
}{
|
||||
{envelope.ProtocolJT808, "1.2.3.4:10", "PLATFORM", "g7s"},
|
||||
{envelope.ProtocolJT808, "2.3.4.5:20", "DIRECT", ""},
|
||||
{envelope.ProtocolJT808, "3.4.5.6:30", "", ""},
|
||||
{envelope.ProtocolGB32960, "1.2.3.4:40", "", ""},
|
||||
} {
|
||||
got := writer.withSourceIdentity(envelope.FrameEnvelope{Protocol: tc.protocol, SourceEndpoint: tc.endpoint})
|
||||
if got.SourceCode != tc.code || got.SourceKind != tc.kind {
|
||||
t.Fatalf("unexpected source metadata: %+v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -282,25 +282,10 @@ func NormalizePlatformSourceMileageForDate(ctx context.Context, db interface {
|
||||
if strings.TrimSpace(statDate) == "" {
|
||||
return 0, nil
|
||||
}
|
||||
rows, err := db.QueryContext(ctx, selectPlatformSourceMileageLegacyVINSQL, statDate, string(protocol))
|
||||
vins, err := PlatformSourceMileageLegacyVINs(ctx, db, statDate, protocol)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var vins []string
|
||||
for rows.Next() {
|
||||
var vin string
|
||||
if err := rows.Scan(&vin); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if strings.TrimSpace(vin) != "" {
|
||||
vins = append(vins, vin)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
normalized := 0
|
||||
for _, vin := range vins {
|
||||
@@ -315,6 +300,27 @@ func NormalizePlatformSourceMileageForDate(ctx context.Context, db interface {
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
// PlatformSourceMileageLegacyVINs lets batch callers apply their own atomic
|
||||
// projection policy while sharing the exact registry/terminal alias rules.
|
||||
func PlatformSourceMileageLegacyVINs(ctx context.Context, db Queryer, statDate string, protocol envelope.Protocol) ([]string, error) {
|
||||
rows, err := db.QueryContext(ctx, selectPlatformSourceMileageLegacyVINSQL, statDate, string(protocol))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var vins []string
|
||||
for rows.Next() {
|
||||
var vin string
|
||||
if err := rows.Scan(&vin); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(vin) != "" {
|
||||
vins = append(vins, vin)
|
||||
}
|
||||
}
|
||||
return vins, rows.Err()
|
||||
}
|
||||
|
||||
func ShouldNormalizePlatformSourceMileage(sample SourceMileageSample) bool {
|
||||
return strings.Contains(sample.SourceKey, platformSourceKeyPrefix)
|
||||
}
|
||||
@@ -441,7 +447,20 @@ const upsertSourceMergedQualityWindowDaysSQL = `GREATEST(1, DATEDIFF(DATE(` + up
|
||||
|
||||
const upsertSourceMergedMaxMileageSQL = `(` + upsertSourceMergedQualityWindowDaysSQL + ` * ` + maxSelectedDailyMileageKMSQL + `)`
|
||||
|
||||
const normalizePlatformFirstTotalSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.first_total_mileage_km ORDER BY s.first_event_time ASC, s.updated_at ASC SEPARATOR ','), ',', 1) AS DECIMAL(18,3))`
|
||||
// Prefer the newest boundary before this natural day. If no historical
|
||||
// boundary exists, retain the earliest sample inside the day. Select the
|
||||
// odometer and its timestamp from the same row so stale forwarding aliases
|
||||
// cannot restore an older baseline after a repair or a cache refresh.
|
||||
const normalizePlatformFirstOrderSQL = `CASE
|
||||
WHEN s.first_event_time < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME) THEN 0
|
||||
WHEN s.first_event_time IS NOT NULL THEN 1
|
||||
ELSE 2 END,
|
||||
CASE WHEN s.first_event_time < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME) THEN s.first_event_time ELSE NULL END DESC,
|
||||
s.first_event_time ASC, s.updated_at DESC`
|
||||
|
||||
const normalizePlatformFirstTotalSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.first_total_mileage_km ORDER BY ` + normalizePlatformFirstOrderSQL + ` SEPARATOR ','), ',', 1) AS DECIMAL(18,3))`
|
||||
|
||||
const normalizePlatformFirstEventSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.first_event_time ORDER BY ` + normalizePlatformFirstOrderSQL + ` SEPARATOR ','), ',', 1) AS DATETIME)`
|
||||
|
||||
const normalizePlatformLatestTotalSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_total_mileage_km ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS DECIMAL(18,3))`
|
||||
|
||||
@@ -454,7 +473,7 @@ const normalizePlatformDailySQL = `CASE
|
||||
ELSE ` + normalizePlatformDeltaSQL + `
|
||||
END`
|
||||
|
||||
const normalizePlatformQualityWindowDaysSQL = `GREATEST(1, DATEDIFF(DATE(MAX(s.latest_event_time)), DATE(MIN(s.first_event_time))))`
|
||||
const normalizePlatformQualityWindowDaysSQL = `GREATEST(1, DATEDIFF(DATE(MAX(s.latest_event_time)), DATE(` + normalizePlatformFirstEventSQL + `)))`
|
||||
|
||||
const normalizePlatformMaxMileageSQL = `(` + normalizePlatformQualityWindowDaysSQL + ` * ` + maxSelectedDailyMileageKMSQL + `)`
|
||||
|
||||
@@ -537,7 +556,7 @@ SELECT
|
||||
SUM(s.pure_hydrogen_mileage_km) AS pure_hydrogen_mileage_km,
|
||||
SUM(s.sample_count) AS sample_count,
|
||||
SUM(s.pure_hydrogen_sample_count) AS pure_hydrogen_sample_count,
|
||||
MIN(s.first_event_time) AS first_event_time,
|
||||
` + normalizePlatformFirstEventSQL + ` AS first_event_time,
|
||||
MAX(s.latest_event_time) AS latest_event_time,
|
||||
CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_pure_hydrogen_active ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS UNSIGNED) AS latest_pure_hydrogen_active,
|
||||
CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_pure_hydrogen_mode_known ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS UNSIGNED) AS latest_pure_hydrogen_mode_known,
|
||||
@@ -548,7 +567,7 @@ SELECT
|
||||
CASE
|
||||
WHEN ` + normalizePlatformOutsideDailyRangeSQL + ` THEN 'outside_daily_range'
|
||||
WHEN ` + normalizePlatformDailySQL + ` = 0 AND ` + normalizePlatformDeltaSQL + ` < 0 THEN 'negative_jitter_clamped'
|
||||
WHEN MIN(s.first_event_time) < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)
|
||||
WHEN ` + normalizePlatformFirstEventSQL + ` < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)
|
||||
THEN '` + QualityReasonHistorical + `'
|
||||
ELSE '` + QualityReasonCurrentDayFirst + `'
|
||||
END AS quality_reason
|
||||
@@ -611,10 +630,7 @@ ON DUPLICATE KEY UPDATE
|
||||
latest_total_mileage_km = ` + upsertSourceMergedLatestTotalSQL + `,
|
||||
daily_mileage_km = ` + upsertSourceMergedDailySQL + `,
|
||||
sample_count = sample_count + VALUES(sample_count),
|
||||
first_event_time = CASE
|
||||
WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time THEN VALUES(first_event_time)
|
||||
ELSE first_event_time
|
||||
END,
|
||||
first_event_time = ` + upsertSourceMergedFirstEventSQL + `,
|
||||
latest_event_time = CASE
|
||||
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(latest_event_time)
|
||||
ELSE latest_event_time
|
||||
|
||||
@@ -344,7 +344,7 @@ func TestNormalizePlatformSourceMileageMergesLegacyIPKeys(t *testing.T) {
|
||||
"ON DUPLICATE KEY UPDATE",
|
||||
"sample_count = sample_count + VALUES(sample_count)",
|
||||
"s.quality_status IN ('" + QualityOK + "', '" + QualityNoPreviousBaseline + "')",
|
||||
"WHEN MIN(s.first_event_time) < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)",
|
||||
normalizePlatformFirstEventSQL + " < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)",
|
||||
"THEN '" + QualityInvalidDelta + "'",
|
||||
"ELSE '" + QualityOK + "'",
|
||||
"THEN 'negative_jitter_clamped'",
|
||||
@@ -392,6 +392,25 @@ func TestNormalizePlatformSourceMileageExcludesIncompleteLegacyTotals(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePlatformSourceMileagePreservesTheNewerBoundaryPair(t *testing.T) {
|
||||
// A stale IP-key candidate must not change the canonical baseline's time
|
||||
// back to September while retaining October's first_total_mileage_km.
|
||||
for _, expression := range []string{normalizePlatformFirstTotalSQL, normalizePlatformFirstEventSQL} {
|
||||
if !strings.Contains(expression, "ORDER BY "+normalizePlatformFirstOrderSQL) {
|
||||
t.Fatal("baseline total and timestamp must select the same legacy row")
|
||||
}
|
||||
}
|
||||
if !strings.Contains(normalizePlatformFirstOrderSQL, "THEN s.first_event_time ELSE NULL END DESC") {
|
||||
t.Fatal("prior-day boundaries must prefer the newest timestamp")
|
||||
}
|
||||
if !strings.Contains(normalizePlatformSourceMileageInsertSQL, "first_event_time = "+upsertSourceMergedFirstEventSQL) {
|
||||
t.Fatal("legacy upsert must use the same boundary preference as its odometer")
|
||||
}
|
||||
if strings.Contains(normalizePlatformSourceMileageInsertSQL, "MIN(s.first_event_time)") {
|
||||
t.Fatal("an old forwarding alias must not restore an older boundary")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePlatformSourceMileageForDateNormalizesAndProjectsLegacyVINs(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user