feat: sync vehicle monitoring and accumulated platform updates
Add monitoring metrics, numeric sorting, report-time display, vehicle number labels, and map layout fixes. Include accumulated mileage, hydrogen, OneOS authentication, open platform, ingestion, deployment, and UI updates. Exclude temporary files, generated reports, and credentials. Validate frontend 632 tests and production build, API packages, vehicle-map tests, and release installer tests. Gateway suite has one pre-existing date-sensitive identity throttling test failure reproduced on the parent commit.
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
)
|
||||
|
||||
func TestFeichiCustom80UsesConnectionLoginAndSurvivesCanonicalRaw(t *testing.T) {
|
||||
payload, _ := hex.DecodeString("3B1428271014282710024E01560000")
|
||||
body := []byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0, 0x80, 0, 15}
|
||||
body = append(body, payload...)
|
||||
body = append(body, 0x80, 0, 3, 0x05, 0x80, 0x02)
|
||||
body = append(body, 0x05, 0, 0x07, 0x36, 0x50, 0x40, 0x01, 0xd2, 0x4f, 0)
|
||||
frame := buildGBFrame(0x02, 0xfe, "LA9GG68L7PBAF4803", body)
|
||||
for _, tc := range []struct {
|
||||
account string
|
||||
password string
|
||||
wantCustom bool
|
||||
}{
|
||||
{"Feichi", "test-secret", true},
|
||||
{"Feichi", "wrong-secret", false},
|
||||
{"Hyundai", "test-secret", false},
|
||||
{"YueJin", "test-secret", false},
|
||||
} {
|
||||
t.Run(tc.account+"/"+tc.password, func(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960, Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames, Parse: gb32960.ParseFrame,
|
||||
ParseForPlatform: gb32960.ParseFrameForPlatform,
|
||||
Authenticate: authentication.NewGB32960PlatformAuthenticator(authentication.ModeEnforce, map[string][]string{
|
||||
"Feichi": {"test-secret"}, "Hyundai": {"test-secret"}, "YueJin": {"test-secret"},
|
||||
}),
|
||||
}, sink)
|
||||
state := &connectionState{}
|
||||
login := []byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0, 0, 1}
|
||||
login = append(login, fixedASCIIBytes(tc.account, 12)...)
|
||||
login = append(login, fixedASCIIBytes(tc.password, 20)...)
|
||||
login = append(login, 1)
|
||||
server.handleFrame(context.Background(), nil, buildGBFrame(0x05, 0xfe, "", login), "183.239.48.25:2147", state)
|
||||
server.handleFrame(context.Background(), nil, frame, "183.239.48.25:2147", state)
|
||||
if len(sink.raw) != 2 {
|
||||
t.Fatalf("raw frames=%d", len(sink.raw))
|
||||
}
|
||||
raw := sink.raw[1]
|
||||
if raw.RawHex != strings.ToUpper(hex.EncodeToString(frame)) || len(raw.Parsed) != 0 {
|
||||
t.Fatal("canonical whole RAW frame was changed or contains duplicate tree")
|
||||
}
|
||||
prefix := "gb32960.feichi_custom_80.blocks."
|
||||
_, custom := raw.ParsedFields[prefix+"block_1.payload_hex"]
|
||||
if custom != tc.wantCustom {
|
||||
t.Fatalf("account=%s wantCustom=%v fields=%#v", tc.account, tc.wantCustom, raw.ParsedFields)
|
||||
}
|
||||
if !tc.wantCustom {
|
||||
if raw.ParsedFields["gb32960.gd_fc_demo_extension.declared_length"] != "15" {
|
||||
t.Fatal("legacy decoder was not retained")
|
||||
}
|
||||
return
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
prefix + "block_1.declared_length": "15",
|
||||
prefix + "block_1.actual_length": "15",
|
||||
prefix + "block_1.payload_hex": "3B1428271014282710024E01560000",
|
||||
prefix + "block_1.raw_hex": "80000F3B1428271014282710024E01560000",
|
||||
prefix + "block_2.payload_hex": "058002",
|
||||
prefix + "block_2.raw_hex": "800003058002",
|
||||
"gb32960.position.longitude": "121",
|
||||
"gb32960.position.latitude": "30.56",
|
||||
} {
|
||||
if got := fmt.Sprint(raw.ParsedFields[key]); got != want {
|
||||
t.Fatalf("%s=%s want=%s", key, got, want)
|
||||
}
|
||||
}
|
||||
if raw.PlatformName != "Feichi" || raw.ParseStatus != envelope.ParseOK {
|
||||
t.Fatalf("platform=%q status=%s", raw.PlatformName, raw.ParseStatus)
|
||||
}
|
||||
// A fresh TCP connection does not inherit the previous account.
|
||||
server.handleFrame(context.Background(), nil, frame, "183.239.48.25:9999", &connectionState{})
|
||||
if _, exists := sink.raw[2].ParsedFields[prefix+"block_1.payload_hex"]; exists {
|
||||
t.Fatal("Feichi account leaked into a fresh connection")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeichiCustom80TruncationIsPreservedThroughRawProjection(t *testing.T) {
|
||||
sink := &recordingSink{}
|
||||
server := newTestServer(t, TCPProtocol{
|
||||
Protocol: envelope.ProtocolGB32960, Addr: ":0",
|
||||
Extract: gb32960.ExtractFrames, Parse: gb32960.ParseFrame,
|
||||
ParseForPlatform: gb32960.ParseFrameForPlatform,
|
||||
}, sink)
|
||||
frame := buildGBFrame(0x02, 0xfe, "LA9GG68L7PBAF4803", []byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0, 0x80, 0, 15, 0x3b, 0x14})
|
||||
server.handleFrame(context.Background(), nil, frame, "183.239.48.25:2147", &connectionState{platformName: "Feichi"})
|
||||
raw := sink.raw[0]
|
||||
for key, want := range map[string]string{
|
||||
"gb32960.feichi_custom_80.blocks.block_1.declared_length": "15",
|
||||
"gb32960.feichi_custom_80.blocks.block_1.actual_length": "2",
|
||||
"gb32960.feichi_custom_80.blocks.block_1.payload_hex": "3B14",
|
||||
"gb32960.feichi_custom_80.blocks.block_1.raw_hex": "80000F3B14",
|
||||
"gb32960.feichi_custom_80.blocks.block_1.parse_error": "truncated_payload",
|
||||
} {
|
||||
if got := fmt.Sprint(raw.ParsedFields[key]); got != want {
|
||||
t.Fatalf("%s=%s want=%s", key, got, want)
|
||||
}
|
||||
}
|
||||
if raw.ParseStatus != envelope.ParsePartial || raw.RawHex != strings.ToUpper(hex.EncodeToString(frame)) {
|
||||
t.Fatalf("status=%s raw=%s", raw.ParseStatus, raw.RawHex)
|
||||
}
|
||||
}
|
||||
@@ -25,15 +25,18 @@ type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
|
||||
|
||||
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
|
||||
|
||||
type PlatformFrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint, platformAccount string) (envelope.FrameEnvelope, error)
|
||||
|
||||
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
|
||||
|
||||
type TCPProtocol struct {
|
||||
Protocol envelope.Protocol
|
||||
Addr string
|
||||
Extract FrameExtractor
|
||||
Parse FrameParser
|
||||
Authenticate authentication.Authenticator
|
||||
Respond FrameResponder
|
||||
Protocol envelope.Protocol
|
||||
Addr string
|
||||
Extract FrameExtractor
|
||||
Parse FrameParser
|
||||
ParseForPlatform PlatformFrameParser
|
||||
Authenticate authentication.Authenticator
|
||||
Respond FrameResponder
|
||||
}
|
||||
|
||||
type connectionState struct {
|
||||
@@ -283,7 +286,17 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
|
||||
defer cancelFrame()
|
||||
|
||||
receivedAtMS := time.Now().UnixMilli()
|
||||
env, err := s.protocol.Parse(raw, receivedAtMS, source)
|
||||
var env envelope.FrameEnvelope
|
||||
var err error
|
||||
if s.protocol.ParseForPlatform != nil {
|
||||
platformAccount := ""
|
||||
if state != nil {
|
||||
platformAccount = state.platformName
|
||||
}
|
||||
env, err = s.protocol.ParseForPlatform(raw, receivedAtMS, source, platformAccount)
|
||||
} else {
|
||||
env, err = s.protocol.Parse(raw, receivedAtMS, source)
|
||||
}
|
||||
if err != nil {
|
||||
env = envelope.FrameEnvelope{
|
||||
Protocol: s.protocol.Protocol,
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package gb32960
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const feichiCustom80Payload = "3B1428271014282710024E01560000"
|
||||
|
||||
func TestFeichiCustom80UsesDeclaredLengthAndContinuesWithStandardData(t *testing.T) {
|
||||
for _, command := range []byte{0x02, 0x03} {
|
||||
for _, length := range []int{0, 1, 9, 15, 32, 513} {
|
||||
t.Run(fmt.Sprintf("command_%02X_length_%d", command, length), func(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte{0x80, 0x05, 0x02, 0x23, 0x23}, (length+4)/5)[:length]
|
||||
if length == 15 {
|
||||
payload, _ = hex.DecodeString(feichiCustom80Payload)
|
||||
}
|
||||
body := []byte{0x1a, 0x0a, 0x17, 0x0a, 0x00, 0x00, 0x80, byte(length >> 8), byte(length)}
|
||||
body = append(body, payload...)
|
||||
body = append(body, feichiTestPositionUnit()...)
|
||||
body = append(body, 0x34, 0, 0, 0, 0, 0, 0, 0x2b)
|
||||
frame := buildFrame(command, 0xfe, "LA9GG68L7PBAF4803", body)
|
||||
env, err := ParseFrameForPlatform(frame, 1234, "183.239.48.25:2147", "Feichi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env.ParseStatus != envelope.ParseOK || env.RawHex != strings.ToUpper(hex.EncodeToString(frame)) {
|
||||
t.Fatalf("status=%s raw=%s", env.ParseStatus, env.RawHex)
|
||||
}
|
||||
units := env.Parsed["data_units"].([]map[string]any)
|
||||
if len(units) != 3 || units[0]["name"] != "feichi_custom_80" || units[1]["name"] != "position" || units[2]["name"] != "gd_fc_vehicle_info" {
|
||||
t.Fatalf("wrong unit boundaries: %#v", units)
|
||||
}
|
||||
block := feichiBlock(t, units[0], 1)
|
||||
if block["declared_length"] != length || block["actual_length"] != length || block["payload_hex"] != strings.ToUpper(hex.EncodeToString(payload)) {
|
||||
t.Fatalf("block=%#v", block)
|
||||
}
|
||||
if want := fmt.Sprintf("80%04X%X", length, payload); block["raw_hex"] != want {
|
||||
t.Fatalf("raw=%v want=%s", block["raw_hex"], want)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121)
|
||||
assertFloatField(t, env, envelope.FieldLatitude, 30.56)
|
||||
assertFloatField(t, env, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
|
||||
if _, exists := env.Fields["gd_fc_demo_stack_temp_c"]; exists {
|
||||
t.Fatal("Feichi opaque bytes must not become Guangdong measurement fields")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeichiCustom80TruncationKeepsEveryReceivedByte(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
data []byte
|
||||
error string
|
||||
payload string
|
||||
}{
|
||||
{"missing_length", nil, "truncated_length", ""},
|
||||
{"partial_length", []byte{0}, "truncated_length", ""},
|
||||
{"short_payload", []byte{0, 15, 0x3b, 0x14, 0x28}, "truncated_payload", "3B1428"},
|
||||
{"oversized_length", []byte{0xff, 0xff, 0x80}, "truncated_payload", "80"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := append([]byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0}, feichiTestPositionUnit()...)
|
||||
body = append(body, 0x80)
|
||||
body = append(body, tc.data...)
|
||||
frame := buildFrame(0x02, 0xfe, "LA9GG68L7PBAF4803", body)
|
||||
env, err := ParseFrameForPlatform(frame, 1234, "source", "Feichi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env.ParseStatus != envelope.ParsePartial || !strings.Contains(env.ParseError, tc.error) {
|
||||
t.Fatalf("status=%s error=%q", env.ParseStatus, env.ParseError)
|
||||
}
|
||||
if env.RawHex != strings.ToUpper(hex.EncodeToString(frame)) {
|
||||
t.Fatal("whole RAW frame was changed")
|
||||
}
|
||||
units := env.Parsed["data_units"].([]map[string]any)
|
||||
if len(units) != 2 || units[0]["name"] != "position" {
|
||||
t.Fatalf("units=%#v", units)
|
||||
}
|
||||
assertFloatField(t, env, envelope.FieldLongitude, 121)
|
||||
block := feichiBlock(t, units[1], 1)
|
||||
if block["raw_hex"] != fmt.Sprintf("80%X", tc.data) || block["payload_hex"] != tc.payload || block["parse_error"] != tc.error {
|
||||
t.Fatalf("received bytes not preserved: %#v", block)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformParserPreservesExisting32960Accounts(t *testing.T) {
|
||||
frame, err := hex.DecodeString(realFuelCellFrame)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, err := ParseFrame(frame, 1234, "same-source")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, account := range []string{"", "Hyundai", "YueJin", "GuangJiaoTou", "feichi", "Feichi-other"} {
|
||||
t.Run(account, func(t *testing.T) {
|
||||
got, err := ParseFrameForPlatform(frame, 1234, "same-source", account)
|
||||
if err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("legacy account changed: account=%q err=%v", account, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
assertIntField(t, want, "gd_fc_demo_stack_temp_c", 65)
|
||||
assertFloatField(t, want, "gd_fc_vehicle_hydrogen_mass_kg", 4.3)
|
||||
}
|
||||
|
||||
func TestFeichiFramesWithoutCustom80KeepExistingResults(t *testing.T) {
|
||||
for _, command := range []byte{0x02, 0x03} {
|
||||
frame := buildFrame(command, 0xfe, "LA9GG68L7PBAF4803", append([]byte{0x1a, 0x0a, 0x17, 0x0a, 0, 0}, feichiTestPositionUnit()...))
|
||||
want, err := ParseFrame(frame, 1234, "same-source")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := ParseFrameForPlatform(frame, 1234, "same-source", "Feichi")
|
||||
if err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("standard Feichi frame changed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func feichiBlock(t *testing.T, unit map[string]any, index int) map[string]any {
|
||||
t.Helper()
|
||||
return unit["value"].(map[string]any)["blocks"].(map[string]any)[fmt.Sprintf("block_%d", index)].(map[string]any)
|
||||
}
|
||||
|
||||
func feichiTestPositionUnit() []byte {
|
||||
return []byte{0x05, 0x00, 0x07, 0x36, 0x50, 0x40, 0x01, 0xd2, 0x4f, 0x00}
|
||||
}
|
||||
@@ -51,6 +51,17 @@ func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error)
|
||||
}
|
||||
|
||||
func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||
return parseFrame(raw, receivedAtMS, sourceEndpoint, false)
|
||||
}
|
||||
|
||||
// ParseFrameForPlatform selects Feichi's opaque, length-prefixed 0x80 format
|
||||
// using the account established by this TCP connection's platform login.
|
||||
// All other accounts retain the existing Guangdong extension decoder.
|
||||
func ParseFrameForPlatform(raw []byte, receivedAtMS int64, sourceEndpoint, platformAccount string) (envelope.FrameEnvelope, error) {
|
||||
return parseFrame(raw, receivedAtMS, sourceEndpoint, strings.TrimSpace(platformAccount) == "Feichi")
|
||||
}
|
||||
|
||||
func parseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string, feichiCustom80 bool) (envelope.FrameEnvelope, error) {
|
||||
if len(raw) < minFrameLen {
|
||||
return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
|
||||
}
|
||||
@@ -86,7 +97,7 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
|
||||
eventTimeMS := receivedAtMS
|
||||
|
||||
if command == 0x02 || command == 0x03 {
|
||||
eventTime, units := parseDataBody(version, body, fields)
|
||||
eventTime, units := parseDataBodyWithCustom80(version, body, fields, feichiCustom80)
|
||||
if !eventTime.IsZero() {
|
||||
eventTimeMS = eventTime.UnixMilli()
|
||||
fields["device_time"] = eventTime.Format(time.RFC3339)
|
||||
@@ -117,17 +128,33 @@ func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope
|
||||
Fields: fields,
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
if feichiCustom80 {
|
||||
if units, ok := parsed["data_units"].([]map[string]any); ok {
|
||||
for _, unit := range units {
|
||||
if unit["name"] == "feichi_custom_80" && unit["error"] != nil {
|
||||
env.ParseStatus = envelope.ParsePartial
|
||||
env.ParseError = "Feichi 0x80: " + fmt.Sprint(unit["error"])
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
env.EventID = env.StableEventID()
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
|
||||
return parseDataBodyWithCustom80(version, body, fields, false)
|
||||
}
|
||||
|
||||
func parseDataBodyWithCustom80(version string, body []byte, fields map[string]any, feichiCustom80 bool) (time.Time, []map[string]any) {
|
||||
if len(body) < 6 {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
eventTime := parseGBTime(body[:6])
|
||||
cursor := 6
|
||||
var units []map[string]any
|
||||
custom80Blocks := 0
|
||||
for cursor < len(body) {
|
||||
unitType := body[cursor]
|
||||
cursor++
|
||||
@@ -293,6 +320,16 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
|
||||
fields["gd_fc_vehicle_hydrogen_mass_kg"] = unit["hydrogen_mass_kg"]
|
||||
cursor += 7
|
||||
case 0x80:
|
||||
if feichiCustom80 {
|
||||
custom80Blocks++
|
||||
unit, size := parseFeichiCustom80Data(body[cursor:], custom80Blocks)
|
||||
units = append(units, unit)
|
||||
cursor += size
|
||||
if unit["error"] != nil {
|
||||
return eventTime, units
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(body[cursor:]) < 11 {
|
||||
units = append(units, map[string]any{"type": "0x80", "error": "truncated"})
|
||||
return eventTime, units
|
||||
@@ -323,6 +360,43 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
|
||||
return eventTime, units
|
||||
}
|
||||
|
||||
func parseFeichiCustom80Data(data []byte, blockNumber int) (map[string]any, int) {
|
||||
size := len(data)
|
||||
lengthSize := min(2, len(data))
|
||||
payload := []byte(nil)
|
||||
block := map[string]any{
|
||||
"length_hex": strings.ToUpper(hex.EncodeToString(data[:lengthSize])),
|
||||
}
|
||||
parseError := ""
|
||||
if len(data) < 2 {
|
||||
parseError = "truncated_length"
|
||||
} else {
|
||||
declared := int(binary.BigEndian.Uint16(data[:2]))
|
||||
block["declared_length"] = declared
|
||||
size = min(2+declared, len(data))
|
||||
payload = data[2:size]
|
||||
if size < 2+declared {
|
||||
parseError = "truncated_payload"
|
||||
}
|
||||
}
|
||||
block["actual_length"] = len(payload)
|
||||
block["payload_hex"] = strings.ToUpper(hex.EncodeToString(payload))
|
||||
block["raw_hex"] = "80" + strings.ToUpper(hex.EncodeToString(data[:size]))
|
||||
unit := map[string]any{
|
||||
"type": "0x80",
|
||||
"name": "feichi_custom_80",
|
||||
"value": map[string]any{
|
||||
// Distinct keys preserve repeated 0x80 blocks in flattened RAW fields.
|
||||
"blocks": map[string]any{fmt.Sprintf("block_%d", blockNumber): block},
|
||||
},
|
||||
}
|
||||
if parseError != "" {
|
||||
unit["error"] = parseError
|
||||
block["parse_error"] = parseError
|
||||
}
|
||||
return unit, size
|
||||
}
|
||||
|
||||
func frameLength(frameStart []byte, bodyLen int) int {
|
||||
standardLen := headerLen + bodyLen + 1
|
||||
if len(frameStart) < headerLen || frameStart[2] != 0x05 || bodyLen != 41 {
|
||||
|
||||
@@ -49,6 +49,7 @@ type Writer struct {
|
||||
lastProjection map[string]projectionCacheEntry
|
||||
pendingProjection map[string]pendingProjectionEntry
|
||||
baselineCache map[string]sourceBaselineCacheEntry
|
||||
sourceIdentities map[string]SourceIdentity
|
||||
mileageKeysByPrefix map[string]map[string]struct{}
|
||||
projectionKeysByPrefix map[string]map[string]struct{}
|
||||
baselineKeysByPrefix map[string]map[string]struct{}
|
||||
@@ -355,6 +356,7 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
|
||||
result.SamplesSkippedMissingFields = 1
|
||||
return result, nil
|
||||
}
|
||||
env = w.withSourceIdentity(env)
|
||||
seenAt := w.sourceSeenAt(env)
|
||||
w.maybeCleanupCaches(seenAt)
|
||||
identity, hasSource := NewSourceIdentityFromEnvelope(env)
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
// ReloadSourceIdentities aligns live JT808 candidates with the durable platform
|
||||
// keys used by backfill and daily statistics. A snapshot avoids a database
|
||||
// lookup per frame; a failed refresh leaves the last successful snapshot intact.
|
||||
func (w *Writer) ReloadSourceIdentities(ctx context.Context) (int, error) {
|
||||
rows, err := w.query.QueryContext(ctx, `SELECT source_ip,COALESCE(source_code,''),COALESCE(platform_name,''),source_kind
|
||||
FROM vehicle_data_source WHERE protocol='JT808'`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
next := map[string]SourceIdentity{}
|
||||
for rows.Next() {
|
||||
var source SourceIdentity
|
||||
if err := rows.Scan(&source.SourceIP, &source.SourceCode, &source.PlatformName, &source.SourceKind); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
source.Protocol = envelope.ProtocolJT808
|
||||
source.SourceIP = NormalizeSourceIP(source.SourceIP)
|
||||
source.SourceCode = strings.TrimSpace(source.SourceCode)
|
||||
source.SourceKind = strings.ToUpper(strings.TrimSpace(source.SourceKind))
|
||||
if source.SourceIP != "" && ((source.SourceKind == "PLATFORM" && source.SourceCode != "") || source.SourceKind == "DIRECT") {
|
||||
next[source.SourceIP] = source
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.sourceIdentities = next
|
||||
w.mu.Unlock()
|
||||
return len(next), nil
|
||||
}
|
||||
|
||||
func (w *Writer) withSourceIdentity(env envelope.FrameEnvelope) envelope.FrameEnvelope {
|
||||
if env.Protocol != envelope.ProtocolJT808 {
|
||||
return env
|
||||
}
|
||||
ip := NormalizeSourceIP(env.SourceEndpoint)
|
||||
w.mu.Lock()
|
||||
source, found := w.sourceIdentities[ip]
|
||||
w.mu.Unlock()
|
||||
if found {
|
||||
env.SourceCode = source.SourceCode
|
||||
env.SourceKind = source.SourceKind
|
||||
if strings.TrimSpace(source.PlatformName) != "" {
|
||||
env.PlatformName = source.PlatformName
|
||||
}
|
||||
}
|
||||
return env
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestWriterResolvesForwardingIPBeforeDurableBaselineLookup(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
writer := NewWriter(db, loc)
|
||||
mock.ExpectQuery("SELECT source_ip,COALESCE").WillReturnRows(sqlmock.NewRows([]string{"ip", "code", "name", "kind"}).AddRow("115.159.85.149", "g7s", "G7s", "PLATFORM"))
|
||||
if _, err := writer.ReloadSourceIdentities(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vin, phone, key := "LMRKH9AC7R1004120", "64341233712", "JT808:64341233712@PLATFORM:g7s"
|
||||
previous := time.Date(2026, 10, 18, 23, 59, 49, 0, loc)
|
||||
current := time.Date(2026, 10, 19, 12, 0, 0, 0, loc)
|
||||
mock.ExpectExec("INSERT INTO vehicle_data_source").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectQuery("SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source").WithArgs(vin, "2026-10-19", "JT808", key).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"total", "time"}).AddRow(52704.4, previous))
|
||||
mock.ExpectBegin()
|
||||
args := []driver.Value{vin, "2026-10-19", "JT808", key}
|
||||
for len(args) < 21 {
|
||||
args = append(args, sqlmock.AnyArg())
|
||||
}
|
||||
args[9] = 52704.4
|
||||
args[10] = 52714.4
|
||||
args[11] = 10.0
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage_source").WithArgs(args...).WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage_source").WithArgs(vin, "2026-10-19", "JT808").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("DELETE s").WithArgs(vin, "2026-10-19", "JT808").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("INSERT INTO vehicle_daily_mileage").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("UPDATE vehicle_daily_mileage_source s").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("DELETE FROM vehicle_daily_mileage").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectCommit()
|
||||
fields := map[string]any{"jt808.location.total_mileage_km": 52714.4}
|
||||
_, err = writer.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: vin, Phone: phone, SourceEndpoint: "115.159.85.149:12345", Fields: fields, ParsedFields: fields, EventTimeMS: current.UnixMilli(), ReceivedAtMS: current.UnixMilli(), ParseStatus: envelope.ParseOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceIdentityRefreshRetainsSnapshotOnFailureAndDoesNotMixProtocols(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
writer := NewWriter(db, time.UTC)
|
||||
mock.ExpectQuery("SELECT source_ip,COALESCE").WillReturnRows(sqlmock.NewRows([]string{"ip", "code", "name", "kind"}).AddRow("1.2.3.4", "g7s", "G7s", "PLATFORM").AddRow("2.3.4.5", "", "", "DIRECT").AddRow("3.4.5.6", "", "", "UNKNOWN"))
|
||||
if n, err := writer.ReloadSourceIdentities(context.Background()); err != nil || n != 2 {
|
||||
t.Fatalf("loaded=%d err=%v", n, err)
|
||||
}
|
||||
mock.ExpectQuery("SELECT source_ip,COALESCE").WillReturnError(errors.New("temporary database failure"))
|
||||
if _, err := writer.ReloadSourceIdentities(context.Background()); err == nil {
|
||||
t.Fatal("expected refresh failure")
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
protocol envelope.Protocol
|
||||
endpoint, kind, code string
|
||||
}{
|
||||
{envelope.ProtocolJT808, "1.2.3.4:10", "PLATFORM", "g7s"},
|
||||
{envelope.ProtocolJT808, "2.3.4.5:20", "DIRECT", ""},
|
||||
{envelope.ProtocolJT808, "3.4.5.6:30", "", ""},
|
||||
{envelope.ProtocolGB32960, "1.2.3.4:40", "", ""},
|
||||
} {
|
||||
got := writer.withSourceIdentity(envelope.FrameEnvelope{Protocol: tc.protocol, SourceEndpoint: tc.endpoint})
|
||||
if got.SourceCode != tc.code || got.SourceKind != tc.kind {
|
||||
t.Fatalf("unexpected source metadata: %+v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -282,25 +282,10 @@ func NormalizePlatformSourceMileageForDate(ctx context.Context, db interface {
|
||||
if strings.TrimSpace(statDate) == "" {
|
||||
return 0, nil
|
||||
}
|
||||
rows, err := db.QueryContext(ctx, selectPlatformSourceMileageLegacyVINSQL, statDate, string(protocol))
|
||||
vins, err := PlatformSourceMileageLegacyVINs(ctx, db, statDate, protocol)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var vins []string
|
||||
for rows.Next() {
|
||||
var vin string
|
||||
if err := rows.Scan(&vin); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if strings.TrimSpace(vin) != "" {
|
||||
vins = append(vins, vin)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
normalized := 0
|
||||
for _, vin := range vins {
|
||||
@@ -315,6 +300,27 @@ func NormalizePlatformSourceMileageForDate(ctx context.Context, db interface {
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
// PlatformSourceMileageLegacyVINs lets batch callers apply their own atomic
|
||||
// projection policy while sharing the exact registry/terminal alias rules.
|
||||
func PlatformSourceMileageLegacyVINs(ctx context.Context, db Queryer, statDate string, protocol envelope.Protocol) ([]string, error) {
|
||||
rows, err := db.QueryContext(ctx, selectPlatformSourceMileageLegacyVINSQL, statDate, string(protocol))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var vins []string
|
||||
for rows.Next() {
|
||||
var vin string
|
||||
if err := rows.Scan(&vin); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(vin) != "" {
|
||||
vins = append(vins, vin)
|
||||
}
|
||||
}
|
||||
return vins, rows.Err()
|
||||
}
|
||||
|
||||
func ShouldNormalizePlatformSourceMileage(sample SourceMileageSample) bool {
|
||||
return strings.Contains(sample.SourceKey, platformSourceKeyPrefix)
|
||||
}
|
||||
@@ -441,7 +447,20 @@ const upsertSourceMergedQualityWindowDaysSQL = `GREATEST(1, DATEDIFF(DATE(` + up
|
||||
|
||||
const upsertSourceMergedMaxMileageSQL = `(` + upsertSourceMergedQualityWindowDaysSQL + ` * ` + maxSelectedDailyMileageKMSQL + `)`
|
||||
|
||||
const normalizePlatformFirstTotalSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.first_total_mileage_km ORDER BY s.first_event_time ASC, s.updated_at ASC SEPARATOR ','), ',', 1) AS DECIMAL(18,3))`
|
||||
// Prefer the newest boundary before this natural day. If no historical
|
||||
// boundary exists, retain the earliest sample inside the day. Select the
|
||||
// odometer and its timestamp from the same row so stale forwarding aliases
|
||||
// cannot restore an older baseline after a repair or a cache refresh.
|
||||
const normalizePlatformFirstOrderSQL = `CASE
|
||||
WHEN s.first_event_time < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME) THEN 0
|
||||
WHEN s.first_event_time IS NOT NULL THEN 1
|
||||
ELSE 2 END,
|
||||
CASE WHEN s.first_event_time < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME) THEN s.first_event_time ELSE NULL END DESC,
|
||||
s.first_event_time ASC, s.updated_at DESC`
|
||||
|
||||
const normalizePlatformFirstTotalSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.first_total_mileage_km ORDER BY ` + normalizePlatformFirstOrderSQL + ` SEPARATOR ','), ',', 1) AS DECIMAL(18,3))`
|
||||
|
||||
const normalizePlatformFirstEventSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.first_event_time ORDER BY ` + normalizePlatformFirstOrderSQL + ` SEPARATOR ','), ',', 1) AS DATETIME)`
|
||||
|
||||
const normalizePlatformLatestTotalSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_total_mileage_km ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS DECIMAL(18,3))`
|
||||
|
||||
@@ -454,7 +473,7 @@ const normalizePlatformDailySQL = `CASE
|
||||
ELSE ` + normalizePlatformDeltaSQL + `
|
||||
END`
|
||||
|
||||
const normalizePlatformQualityWindowDaysSQL = `GREATEST(1, DATEDIFF(DATE(MAX(s.latest_event_time)), DATE(MIN(s.first_event_time))))`
|
||||
const normalizePlatformQualityWindowDaysSQL = `GREATEST(1, DATEDIFF(DATE(MAX(s.latest_event_time)), DATE(` + normalizePlatformFirstEventSQL + `)))`
|
||||
|
||||
const normalizePlatformMaxMileageSQL = `(` + normalizePlatformQualityWindowDaysSQL + ` * ` + maxSelectedDailyMileageKMSQL + `)`
|
||||
|
||||
@@ -537,7 +556,7 @@ SELECT
|
||||
SUM(s.pure_hydrogen_mileage_km) AS pure_hydrogen_mileage_km,
|
||||
SUM(s.sample_count) AS sample_count,
|
||||
SUM(s.pure_hydrogen_sample_count) AS pure_hydrogen_sample_count,
|
||||
MIN(s.first_event_time) AS first_event_time,
|
||||
` + normalizePlatformFirstEventSQL + ` AS first_event_time,
|
||||
MAX(s.latest_event_time) AS latest_event_time,
|
||||
CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_pure_hydrogen_active ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS UNSIGNED) AS latest_pure_hydrogen_active,
|
||||
CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_pure_hydrogen_mode_known ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS UNSIGNED) AS latest_pure_hydrogen_mode_known,
|
||||
@@ -548,7 +567,7 @@ SELECT
|
||||
CASE
|
||||
WHEN ` + normalizePlatformOutsideDailyRangeSQL + ` THEN 'outside_daily_range'
|
||||
WHEN ` + normalizePlatformDailySQL + ` = 0 AND ` + normalizePlatformDeltaSQL + ` < 0 THEN 'negative_jitter_clamped'
|
||||
WHEN MIN(s.first_event_time) < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)
|
||||
WHEN ` + normalizePlatformFirstEventSQL + ` < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)
|
||||
THEN '` + QualityReasonHistorical + `'
|
||||
ELSE '` + QualityReasonCurrentDayFirst + `'
|
||||
END AS quality_reason
|
||||
@@ -611,10 +630,7 @@ ON DUPLICATE KEY UPDATE
|
||||
latest_total_mileage_km = ` + upsertSourceMergedLatestTotalSQL + `,
|
||||
daily_mileage_km = ` + upsertSourceMergedDailySQL + `,
|
||||
sample_count = sample_count + VALUES(sample_count),
|
||||
first_event_time = CASE
|
||||
WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time THEN VALUES(first_event_time)
|
||||
ELSE first_event_time
|
||||
END,
|
||||
first_event_time = ` + upsertSourceMergedFirstEventSQL + `,
|
||||
latest_event_time = CASE
|
||||
WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(latest_event_time)
|
||||
ELSE latest_event_time
|
||||
|
||||
@@ -344,7 +344,7 @@ func TestNormalizePlatformSourceMileageMergesLegacyIPKeys(t *testing.T) {
|
||||
"ON DUPLICATE KEY UPDATE",
|
||||
"sample_count = sample_count + VALUES(sample_count)",
|
||||
"s.quality_status IN ('" + QualityOK + "', '" + QualityNoPreviousBaseline + "')",
|
||||
"WHEN MIN(s.first_event_time) < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)",
|
||||
normalizePlatformFirstEventSQL + " < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)",
|
||||
"THEN '" + QualityInvalidDelta + "'",
|
||||
"ELSE '" + QualityOK + "'",
|
||||
"THEN 'negative_jitter_clamped'",
|
||||
@@ -392,6 +392,25 @@ func TestNormalizePlatformSourceMileageExcludesIncompleteLegacyTotals(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePlatformSourceMileagePreservesTheNewerBoundaryPair(t *testing.T) {
|
||||
// A stale IP-key candidate must not change the canonical baseline's time
|
||||
// back to September while retaining October's first_total_mileage_km.
|
||||
for _, expression := range []string{normalizePlatformFirstTotalSQL, normalizePlatformFirstEventSQL} {
|
||||
if !strings.Contains(expression, "ORDER BY "+normalizePlatformFirstOrderSQL) {
|
||||
t.Fatal("baseline total and timestamp must select the same legacy row")
|
||||
}
|
||||
}
|
||||
if !strings.Contains(normalizePlatformFirstOrderSQL, "THEN s.first_event_time ELSE NULL END DESC") {
|
||||
t.Fatal("prior-day boundaries must prefer the newest timestamp")
|
||||
}
|
||||
if !strings.Contains(normalizePlatformSourceMileageInsertSQL, "first_event_time = "+upsertSourceMergedFirstEventSQL) {
|
||||
t.Fatal("legacy upsert must use the same boundary preference as its odometer")
|
||||
}
|
||||
if strings.Contains(normalizePlatformSourceMileageInsertSQL, "MIN(s.first_event_time)") {
|
||||
t.Fatal("an old forwarding alias must not restore an older boundary")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePlatformSourceMileageForDateNormalizesAndProjectsLegacyVINs(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user