shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
go/vehicle-gateway/internal/envelope/envelope.go
View File

@@ -24,6 +24,14 @@ const (
ParseBadFrame ParseStatus = "BAD_FRAME"
)
type EventKind string
const (
EventKindRaw EventKind = "RAW"
EventKindFields EventKind = "FIELDS"
EventKindUnified EventKind = "UNIFIED"
)
const (
FieldSpeedKMH = "speed_kmh"
FieldTotalMileageKM = "total_mileage_km"
@@ -33,27 +41,36 @@ const (
)
type FrameEnvelope struct {
EventID string `json:"event_id"`
TraceID string `json:"trace_id"`
Protocol Protocol `json:"protocol"`
MessageID string `json:"message_id"`
Sequence uint16 `json:"sequence"`
VIN string `json:"vin,omitempty"`
VehicleKeyHint string `json:"vehicle_key,omitempty"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
Plate string `json:"plate,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
Parsed map[string]any `json:"parsed,omitempty"`
ParsedFields map[string]any `json:"parsed_fields,omitempty"`
ParsedFieldTypes map[string]string `json:"parsed_field_types,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
ParseStatus ParseStatus `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
EventID string `json:"event_id"`
TraceID string `json:"trace_id"`
EventKind EventKind `json:"event_kind,omitempty"`
SourceEventID string `json:"source_event_id,omitempty"`
FieldMapping string `json:"field_mapping,omitempty"`
Protocol Protocol `json:"protocol"`
MessageID string `json:"message_id"`
Sequence uint16 `json:"sequence"`
VIN string `json:"vin,omitempty"`
VehicleKeyHint string `json:"vehicle_key,omitempty"`
Phone string `json:"phone,omitempty"`
DeviceID string `json:"device_id,omitempty"`
Plate string `json:"plate,omitempty"`
SourceEndpoint string `json:"source_endpoint,omitempty"`
SourceCode string `json:"source_code,omitempty"`
PlatformName string `json:"platform_name,omitempty"`
SourceKind string `json:"source_kind,omitempty"`
AuthenticationMode string `json:"authentication_mode,omitempty"`
AuthenticationStatus string `json:"authentication_status,omitempty"`
AuthenticationEnforced bool `json:"authentication_enforced,omitempty"`
EventTimeMS int64 `json:"event_time_ms"`
ReceivedAtMS int64 `json:"received_at_ms"`
RawHex string `json:"raw_hex,omitempty"`
RawText string `json:"raw_text,omitempty"`
Parsed map[string]any `json:"parsed,omitempty"`
ParsedFields map[string]any `json:"parsed_fields,omitempty"`
ParsedFieldTypes map[string]string `json:"parsed_field_types,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
ParseStatus ParseStatus `json:"parse_status"`
ParseError string `json:"parse_error,omitempty"`
}
func (e FrameEnvelope) VehicleKey() string {
@@ -93,5 +110,8 @@ func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
if e.ParseStatus == "" {
e.ParseStatus = ParseOK
}
if e.EventKind == "" {
e.EventKind = EventKindRaw
}
return json.Marshal(e)
}

157
go/vehicle-gateway/internal/envelope/envelope_test.go
View File

@@ -3,6 +3,7 @@ package envelope
import (
"encoding/json"
"testing"
"time"
)
func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
@@ -52,4 +53,160 @@ func TestFrameEnvelopeMarshalDefaults(t *testing.T) {
if decoded.ParseStatus != ParseOK {
t.Fatalf("parse status = %q", decoded.ParseStatus)
}
if decoded.EventKind != EventKindRaw {
t.Fatalf("event kind = %q, want %q", decoded.EventKind, EventKindRaw)
}
}
func TestNormalizedEventTimeMSFallsBackToReceivedWhenEventIsFarFuture(t *testing.T) {
received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC).UnixMilli()
event := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC).UnixMilli()
got, ok := NormalizedEventTimeMS(FrameEnvelope{EventTimeMS: event, ReceivedAtMS: received})
if !ok {
t.Fatal("NormalizedEventTimeMS() ok = false")
}
if got != received {
t.Fatalf("normalized event time = %d, want received %d", got, received)
}
_, reason, ok := NormalizedEventTimeMSWithReason(FrameEnvelope{EventTimeMS: event, ReceivedAtMS: received})
if !ok || reason != EventTimeReasonReceivedFutureEvent {
t.Fatalf("reason = %q ok=%v, want future fallback", reason, ok)
}
}
func TestNormalizedEventTimeMSKeepsSmallFutureSkew(t *testing.T) {
received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC).UnixMilli()
event := time.Date(2026, 7, 12, 9, 35, 0, 0, time.UTC).UnixMilli()
got, ok := NormalizedEventTimeMS(FrameEnvelope{EventTimeMS: event, ReceivedAtMS: received})
if !ok {
t.Fatal("NormalizedEventTimeMS() ok = false")
}
if got != event {
t.Fatalf("normalized event time = %d, want event %d", got, event)
}
}
func TestIsRealtimeTelemetryFrameRejectsKnownNonRealtimeMessageIDs(t *testing.T) {
fields := map[string]any{
FieldLatitude: 30.590151,
FieldLongitude: 121.069881,
FieldTotalMileageKM: 10241.2,
}
tests := []FrameEnvelope{
{Protocol: ProtocolJT808, MessageID: "0x0100", Fields: fields, ParseStatus: ParseOK},
{Protocol: ProtocolGB32960, MessageID: "0x01", Fields: fields, Parsed: map[string]any{"data_units": []any{}}, ParseStatus: ParseOK},
}
for _, env := range tests {
if IsRealtimeTelemetryFrame(env) {
t.Fatalf("IsRealtimeTelemetryFrame(%s/%s) = true, want false", env.Protocol, env.MessageID)
}
}
}
func TestIsRealtimeTelemetryFrameAcceptsKnownRealtimeMessageIDs(t *testing.T) {
tests := []FrameEnvelope{
{Protocol: ProtocolJT808, MessageID: "0x0200", ParsedFields: map[string]any{"jt808.location.speed_kmh": 1}, ParseStatus: ParseOK},
{Protocol: ProtocolGB32960, MessageID: "0x02", ParsedFields: map[string]any{"gb32960.vehicle.speed_kmh": 1}, ParseStatus: ParseOK},
{Protocol: ProtocolGB32960, MessageID: "0x03", ParsedFields: map[string]any{"gb32960.vehicle.speed_kmh": 1}, ParseStatus: ParseOK},
}
for _, env := range tests {
if !IsRealtimeTelemetryFrame(env) {
t.Fatalf("IsRealtimeTelemetryFrame(%s/%s) = false, want true", env.Protocol, env.MessageID)
}
}
}
func TestIsRealtimeTelemetryFrameUsesCanonicalMQTTParsedFields(t *testing.T) {
realtime := FrameEnvelope{
Protocol: ProtocolYutongMQTT,
MessageID: "MQTT",
ParseStatus: ParseOK,
ParsedFields: map[string]any{
"yutong_mqtt.data.latitude": 30.590151,
},
}
if !IsRealtimeTelemetryFrame(realtime) {
t.Fatal("canonical MQTT data field should be classified as realtime")
}
metadataOnly := realtime
metadataOnly.ParsedFields = map[string]any{"yutong_mqtt.metadata.topic": "/ytforward/shln/3"}
if IsRealtimeTelemetryFrame(metadataOnly) {
t.Fatal("MQTT metadata-only field must not be classified as realtime")
}
}
func TestRequiresVehicleIdentity(t *testing.T) {
tests := []struct {
name string
env FrameEnvelope
want bool
}{
{
name: "gb32960 realtime upload",
env: FrameEnvelope{Protocol: ProtocolGB32960, MessageID: "0x02", Parsed: map[string]any{"data_units": []any{map[string]any{"name": "vehicle"}}}, ParseStatus: ParseOK},
want: true,
},
{
name: "gb32960 platform login",
env: FrameEnvelope{Protocol: ProtocolGB32960, MessageID: "0x05", ParseStatus: ParseOK},
want: false,
},
{
name: "gb32960 command response",
env: FrameEnvelope{Protocol: ProtocolGB32960, MessageID: "0x07", ParseStatus: ParseOK},
want: false,
},
{
name: "jt808 registration",
env: FrameEnvelope{Protocol: ProtocolJT808, MessageID: "0x0100", ParseStatus: ParseOK},
want: true,
},
{
name: "jt808 location",
env: FrameEnvelope{Protocol: ProtocolJT808, MessageID: "0x0200", Parsed: map[string]any{"location": map[string]any{"speed_kmh": 1}}, ParseStatus: ParseOK},
want: true,
},
{
name: "yutong empty data",
env: FrameEnvelope{Protocol: ProtocolYutongMQTT, MessageID: "MQTT", Parsed: map[string]any{"data": map[string]any{}}, ParseStatus: ParseOK},
want: false,
},
{
name: "yutong telemetry data",
env: FrameEnvelope{Protocol: ProtocolYutongMQTT, MessageID: "MQTT", Parsed: map[string]any{"data": map[string]any{"TOTAL_MILEAGE": 123}}, ParseStatus: ParseOK},
want: true,
},
{
name: "bad frame",
env: FrameEnvelope{Protocol: ProtocolJT808, MessageID: "0x0200", ParseStatus: ParseBadFrame},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := RequiresVehicleIdentity(tt.env); got != tt.want {
t.Fatalf("RequiresVehicleIdentity() = %v, want %v", got, tt.want)
}
})
}
}
func TestNormalizeSourceEndpointKey(t *testing.T) {
tests := map[string]string{
"115.231.168.135:20215": "115.231.168.135",
"115.231.168.135": "115.231.168.135",
" 115.159.85.149:28316 ": "115.159.85.149",
"mqtt://yutong/ytforward/shln/3": "mqtt",
"MQTT://YUTONG/topic": "mqtt",
"": "",
" ": "",
}
for input, want := range tests {
if got := NormalizeSourceEndpointKey(input); got != want {
t.Fatalf("NormalizeSourceEndpointKey(%q) = %q, want %q", input, got, want)
}
}
}

78
go/vehicle-gateway/internal/envelope/realtime.go
View File

@@ -14,33 +14,93 @@ func IsRealtimeTelemetryFrame(env FrameEnvelope) bool {
command = strings.TrimSpace(stringAny(header["command"]))
}
}
if strings.EqualFold(command, "0x02") || strings.EqualFold(command, "0x03") {
return true
if command != "" && !strings.EqualFold(command, "0x02") && !strings.EqualFold(command, "0x03") {
return false
}
_, hasDataUnits := env.Parsed["data_units"]
return hasDataUnits || hasRealtimeField(env)
return hasDataUnits || hasGB32960ParsedTelemetryField(env) || hasRealtimeField(env)
case ProtocolJT808:
if strings.EqualFold(strings.TrimSpace(env.MessageID), "0x0200") {
return true
messageID := strings.TrimSpace(env.MessageID)
if messageID != "" && !strings.EqualFold(messageID, "0x0200") {
return false
}
if _, ok := env.Parsed["location"]; ok {
return true
}
return hasRealtimeField(env)
return hasParsedFieldPrefix(env, "jt808.location.") || hasRealtimeField(env)
case ProtocolYutongMQTT:
data, ok := env.Parsed["data"].(map[string]any)
return ok && len(data) > 0
if ok && len(data) > 0 {
return true
}
return hasParsedFieldPrefix(env, "yutong_mqtt.data.") ||
hasParsedFieldPrefix(env, "yutong_mqtt.root.data.")
default:
return false
}
}
func hasGB32960ParsedTelemetryField(env FrameEnvelope) bool {
for field := range env.ParsedFields {
if !strings.HasPrefix(field, "gb32960.") {
continue
}
if strings.HasPrefix(field, "gb32960.header.") ||
strings.HasPrefix(field, "gb32960.platform.") ||
strings.HasPrefix(field, "gb32960.identity.") ||
strings.HasPrefix(field, "gb32960.device_time.") {
continue
}
return true
}
return false
}
// RequiresVehicleIdentity keeps platform/control frames out of vehicle identity
// quality metrics while preserving JT808's phone-centric registration/auth flow.
func RequiresVehicleIdentity(env FrameEnvelope) bool {
if env.ParseStatus == ParseBadFrame {
return false
}
switch env.Protocol {
case ProtocolGB32960:
return IsRealtimeTelemetryFrame(env)
case ProtocolJT808:
return true
case ProtocolYutongMQTT:
return IsRealtimeTelemetryFrame(env)
default:
return false
}
}
func hasRealtimeField(env FrameEnvelope) bool {
return env.Fields[FieldLatitude] != nil ||
if env.Fields[FieldLatitude] != nil ||
env.Fields[FieldLongitude] != nil ||
env.Fields[FieldTotalMileageKM] != nil ||
env.Fields[FieldSpeedKMH] != nil ||
env.Fields[FieldSOCPercent] != nil
env.Fields[FieldSOCPercent] != nil {
return true
}
for field := range env.ParsedFields {
if strings.HasSuffix(field, ".latitude") ||
strings.HasSuffix(field, ".longitude") ||
strings.HasSuffix(field, ".total_mileage_km") ||
strings.HasSuffix(field, ".speed_kmh") ||
strings.HasSuffix(field, ".soc_percent") {
return true
}
}
return false
}
func hasParsedFieldPrefix(env FrameEnvelope, prefix string) bool {
for field := range env.ParsedFields {
if strings.HasPrefix(field, prefix) {
return true
}
}
return false
}
func stringAny(value any) string {

19
go/vehicle-gateway/internal/envelope/source.go Normal file
View File

@@ -0,0 +1,19 @@
package envelope
import "strings"
// NormalizeSourceEndpointKey returns the stable, low-cardinality source key used
// by identity lookup and statistics source tracking.
func NormalizeSourceEndpointKey(endpoint string) string {
endpoint = strings.TrimSpace(endpoint)
if endpoint == "" {
return ""
}
if strings.HasPrefix(strings.ToLower(endpoint), "mqtt://") {
return "mqtt"
}
if host, _, ok := strings.Cut(endpoint, ":"); ok {
return strings.TrimSpace(host)
}
return endpoint
}

31
go/vehicle-gateway/internal/envelope/time.go Normal file
View File

@@ -0,0 +1,31 @@
package envelope
import "time"
const (
MaxFutureEventSkew = 10 * time.Minute
MinPlausibleReceivedTimeMS = 1577836800000 // 2020-01-01T00:00:00Z
)
const (
EventTimeReasonEventTime = "event_time"
EventTimeReasonReceivedMissingEvent = "received_time_missing_event"
EventTimeReasonReceivedFutureEvent = "received_time_future_event"
)
func NormalizedEventTimeMS(env FrameEnvelope) (int64, bool) {
eventMS, _, ok := NormalizedEventTimeMSWithReason(env)
return eventMS, ok
}
func NormalizedEventTimeMSWithReason(env FrameEnvelope) (int64, string, bool) {
eventMS := env.EventTimeMS
receivedMS := env.ReceivedAtMS
if eventMS <= 0 {
return receivedMS, EventTimeReasonReceivedMissingEvent, receivedMS > 0
}
if receivedMS >= MinPlausibleReceivedTimeMS && eventMS > receivedMS+MaxFutureEventSkew.Milliseconds() {
return receivedMS, EventTimeReasonReceivedFutureEvent, true
}
return eventMS, EventTimeReasonEventTime, true
}