shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

258
go/vehicle-gateway/internal/realtime/kv_test.go
View File

@@ -1,6 +1,8 @@
package realtime
import (
"encoding/json"
"strings"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
@@ -40,7 +42,7 @@ func TestRealtimeKVFieldsFromGB32960ParsedDomains(t *testing.T) {
}
func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) {
fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "VIN001",
MessageID: "0x02",
@@ -61,10 +63,17 @@ func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) {
}},
},
},
})
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit mapped fields")
}
if len(fieldsEnv.ParsedFields) != 0 || len(fieldsEnv.ParsedFieldTypes) != 0 || len(fieldsEnv.Parsed) != 0 {
t.Fatalf("fields envelope should keep only slim fields payload, parsed=%#v parsed_fields=%#v parsed_field_types=%#v", fieldsEnv.Parsed, fieldsEnv.ParsedFields, fieldsEnv.ParsedFieldTypes)
}
for _, bareKey := range []string{"charge_status", "soc_percent", "total_mileage_km"} {
if _, exists := fieldsEnv.Fields[bareKey]; exists {
t.Fatalf("fields envelope should not expose non-protocol bare key %q: %#v", bareKey, fieldsEnv.Fields)
@@ -81,6 +90,240 @@ func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) {
}
}
func TestEnsureParsedFieldsKeepsUnknownVINProtocolFieldsAndExcludesAnnotations(t *testing.T) {
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "13307795425",
Parsed: map[string]any{
"registration": map[string]any{
"manufacturer": "YUTNG",
},
"identity": map[string]any{
"resolved": false,
"reason": "no_binding",
},
},
ParsedFields: map[string]any{
"jt808.location.speed_kmh": "12.3",
},
}
if !EnsureParsedFields(&env) {
t.Fatal("unknown VIN raw frame should still retain parsed fields")
}
if got := env.ParsedFields["jt808.location.speed_kmh"]; got != "12.3" {
t.Fatalf("precomputed field = %#v", got)
}
if got := env.ParsedFields["jt808.registration.manufacturer"]; got != "YUTNG" {
t.Fatalf("merged registration field = %#v", got)
}
if _, exists := env.ParsedFields["jt808.identity.resolved"]; exists {
t.Fatalf("derived identity annotations must not enter protocol fields: %#v", env.ParsedFields)
}
for _, field := range []string{"jt808.location.speed_kmh", "jt808.registration.manufacturer"} {
if env.ParsedFieldTypes[field] == "" {
t.Fatalf("field type missing for %s: %#v", field, env.ParsedFieldTypes)
}
}
}
func TestBuildFieldsEnvelopeCopiesSourceMetadata(t *testing.T) {
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
VIN: "LMRKH9AC3R1004101",
MessageID: "MQTT",
SourceEndpoint: "mqtt://yutong/ytforward/shln/1",
SourceCode: "yutong",
PlatformName: "宇通",
SourceKind: "PLATFORM",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
EventID: "event-yutong",
Fields: map[string]any{
envelope.FieldTotalMileageKM: 56905,
},
Parsed: map[string]any{
"data": map[string]any{"TOTAL_MILEAGE": 56905000},
},
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit mapped fields")
}
if fieldsEnv.SourceCode != "yutong" || fieldsEnv.PlatformName != "宇通" || fieldsEnv.SourceKind != "PLATFORM" {
t.Fatalf("source metadata = code:%q platform:%q kind:%q", fieldsEnv.SourceCode, fieldsEnv.PlatformName, fieldsEnv.SourceKind)
}
}
func TestBuildFieldsEnvelopeDropsNonPositiveTotalMileage(t *testing.T) {
fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
MessageID: "0x0200",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
EventID: "event-1",
ParsedFields: map[string]any{
"jt808.location.latitude": "31.259555",
"jt808.location.longitude": "119.892413",
"jt808.location.total_mileage_km": "0",
},
})
if !ok {
t.Fatal("BuildFieldsEnvelope() should still emit valid fields")
}
if _, exists := fieldsEnv.Fields["jt808.location.total_mileage_km"]; exists {
t.Fatalf("fields envelope should drop non-positive mileage: %#v", fieldsEnv.Fields)
}
if fieldsEnv.Fields["jt808.location.latitude"] != "31.259555" {
t.Fatalf("fields envelope should keep valid location fields: %#v", fieldsEnv.Fields)
}
}
func TestBuildFieldsEnvelopeDropsNonPositiveRawTotalMileage(t *testing.T) {
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
VIN: "LMRKH9AC3R1004101",
MessageID: "MQTT",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
EventID: "event-yutong-zero-mileage",
Parsed: map[string]any{
"data": map[string]any{
"LATITUDE": 30.590921,
"LONGITUDE": 121.075044,
"TOTAL_MILEAGE": 0,
},
},
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should still emit non-mileage fields")
}
if _, exists := fieldsEnv.Fields["yutong_mqtt.data.total_mileage"]; exists {
t.Fatalf("fields envelope should drop non-positive raw mileage: %#v", fieldsEnv.Fields)
}
if fieldsEnv.Fields["yutong_mqtt.data.latitude"] != "30.590921" {
t.Fatalf("fields envelope should keep valid location fields: %#v", fieldsEnv.Fields)
}
}
func TestBuildFieldsEnvelopeJSONOmitsDuplicateParsedPayload(t *testing.T) {
fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
MessageID: "0x0200",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
EventID: "event-1",
ParsedFields: map[string]any{
"jt808.location.total_mileage_km": "10241.2",
"jt808.location.speed_kmh": "23",
},
ParsedFieldTypes: map[string]string{
"jt808.location.total_mileage_km": "number",
"jt808.location.speed_kmh": "number",
},
})
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit mapped fields")
}
data, err := json.Marshal(fieldsEnv)
if err != nil {
t.Fatalf("Marshal(fieldsEnv) error = %v", err)
}
text := string(data)
for _, duplicateKey := range []string{`"parsed_fields"`, `"parsed_field_types"`, `"parsed"`} {
if strings.Contains(text, duplicateKey) {
t.Fatalf("fields JSON should omit duplicate %s payload: %s", duplicateKey, text)
}
}
if !strings.Contains(text, `"fields"`) || !strings.Contains(text, `"source_event_id"`) || !strings.Contains(text, `"field_mapping"`) {
t.Fatalf("fields JSON missing slim payload metadata: %s", text)
}
}
func TestBuildFieldsEnvelopeOmitsGB32960VendorFragmentFields(t *testing.T) {
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "VIN001",
MessageID: "0x02",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Parsed: map[string]any{
"data_units": []any{
map[string]any{
"type": "0x30",
"name": "gd_fc_stack",
"value": map[string]any{
"stack_count": 1,
"summaries": []any{
map[string]any{
"cell_count": 432,
"stack_water_outlet_temp_c": 63,
"frame_cell_start": 401,
"frame_cell_count": 32,
"frame_max_cell_voltage_v": 1,
"frame_min_cell_voltage_v": 1,
"hydrogen_inlet_pressure_kpa": 130,
"air_inlet_pressure_kpa": 150,
"stack_water_outlet_temp_extra": "kept",
},
},
},
},
},
},
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit mapped fields")
}
for _, fragmentField := range []string{
"gb32960.gd_fc_stack.frame_cell_start",
"gb32960.gd_fc_stack.frame_cell_count",
"gb32960.gd_fc_stack.frame_max_cell_voltage_v",
"gb32960.gd_fc_stack.frame_min_cell_voltage_v",
} {
if _, exists := fieldsEnv.Fields[fragmentField]; exists {
t.Fatalf("fields envelope should omit fragment-only field %q: %#v", fragmentField, fieldsEnv.Fields)
}
}
for _, keptField := range []string{
"gb32960.gd_fc_stack.stack_count",
"gb32960.gd_fc_stack.stack_water_outlet_temp_c",
"gb32960.gd_fc_stack.hydrogen_inlet_pressure_kpa",
} {
if _, exists := fieldsEnv.Fields[keptField]; !exists {
t.Fatalf("fields envelope should keep current-state field %q: %#v", keptField, fieldsEnv.Fields)
}
}
}
func TestBuildFieldsEnvelopeRequiresIngressParsedFields(t *testing.T) {
_, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
MessageID: "0x0200",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Parsed: map[string]any{
"location": map[string]any{"speed_kmh": 99},
},
})
if ok {
t.Fatal("BuildFieldsEnvelope() must not re-flatten parsed payload downstream")
}
}
func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) {
jtRows := realtimeKVFields(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
@@ -97,6 +340,10 @@ func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) {
"latitude": 30.2,
"total_mileage_km": 10241.2,
"additional": []any{map[string]any{"id": "0x01", "value_hex": "00077235"}},
"io_status": map[string]any{
"value": uint16(2),
"bits": map[string]bool{"deep_sleep": false, "sleep": true},
},
},
})
jtValues := kvMap(jtRows)
@@ -105,9 +352,14 @@ func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) {
}
if jtValues["jt808.location/total_mileage_km"] != "10241.2" ||
jtValues["jt808.location/longitude"] != "121.1" ||
jtValues["jt808.location/additional.additional_1.id"] != "0x01" {
jtValues["jt808.location/additional.additional_1.id"] != "0x01" ||
jtValues["jt808.location/io_status.bits.deep_sleep"] != "false" ||
jtValues["jt808.location/io_status.bits.sleep"] != "true" {
t.Fatalf("jt808 location kv missing: %#v", jtValues)
}
if _, exists := jtValues["jt808.location/io_status.bits"]; exists {
t.Fatalf("JT808 bit fields must be flattened instead of embedded JSON: %#v", jtValues)
}
if jtValues["jt808.location/soc_percent"] != "" {
t.Fatalf("jt808 kv should not include standardized env.Fields-only values: %#v", jtValues)
}