shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
go/vehicle-gateway/internal/realtime/kv.go
View File

@@ -8,6 +8,7 @@ import (
"strings"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
)
type RealtimeKVField struct {
@@ -26,32 +27,36 @@ func BuildFieldsEnvelope(env envelope.FrameEnvelope) (envelope.FrameEnvelope, bo
if !envelope.IsRealtimeTelemetryFrame(env) {
return envelope.FrameEnvelope{}, false
}
fields, fieldTypes, ok := ParsedFieldsForEnvelope(env)
fields, _, ok := ParsedFieldsForEnvelope(env)
if !ok {
return envelope.FrameEnvelope{}, false
}
filterInvalidRealtimeMeasurementFields(fields)
if len(fields) == 0 || !telemetry.HasRealtimeFields(env.Protocol, fields) {
return envelope.FrameEnvelope{}, false
}
out := envelope.FrameEnvelope{
EventID: env.StableEventID() + ":fields",
TraceID: env.TraceID,
Protocol: env.Protocol,
MessageID: env.MessageID,
Sequence: env.Sequence,
VIN: env.VIN,
VehicleKeyHint: env.VehicleKeyHint,
Phone: env.Phone,
DeviceID: env.DeviceID,
Plate: env.Plate,
SourceEndpoint: env.SourceEndpoint,
EventTimeMS: env.EventTimeMS,
ReceivedAtMS: env.ReceivedAtMS,
Fields: fields,
ParsedFields: fields,
ParsedFieldTypes: fieldTypes,
Parsed: map[string]any{
"source_event_id": env.StableEventID(),
"field_mapping": realtimeFieldMappingVersion,
},
ParseStatus: envelope.ParseOK,
EventID: env.StableEventID() + ":fields",
TraceID: env.TraceID,
EventKind: envelope.EventKindFields,
SourceEventID: env.StableEventID(),
FieldMapping: realtimeFieldMappingVersion,
Protocol: env.Protocol,
MessageID: env.MessageID,
Sequence: env.Sequence,
VIN: env.VIN,
VehicleKeyHint: env.VehicleKeyHint,
Phone: env.Phone,
DeviceID: env.DeviceID,
Plate: env.Plate,
SourceEndpoint: env.SourceEndpoint,
SourceCode: env.SourceCode,
PlatformName: env.PlatformName,
SourceKind: env.SourceKind,
EventTimeMS: env.EventTimeMS,
ReceivedAtMS: env.ReceivedAtMS,
Fields: fields,
ParseStatus: envelope.ParseOK,
}
return out, true
}
@@ -61,12 +66,34 @@ func EnsureParsedFields(env *envelope.FrameEnvelope) bool {
return false
}
if len(env.ParsedFields) > 0 {
if derived, derivedTypes, ok := computeParsedFieldsFromParsed(*env); ok {
for field, value := range derived {
if _, exists := env.ParsedFields[field]; !exists {
env.ParsedFields[field] = value
}
}
if env.ParsedFieldTypes == nil {
env.ParsedFieldTypes = map[string]string{}
}
for field, valueType := range derivedTypes {
if _, exists := env.ParsedFieldTypes[field]; !exists {
env.ParsedFieldTypes[field] = valueType
}
}
}
inferredTypes := inferParsedFieldTypes(env.ParsedFields)
if env.ParsedFieldTypes == nil {
env.ParsedFieldTypes = inferParsedFieldTypes(env.ParsedFields)
env.ParsedFieldTypes = inferredTypes
} else {
for field, valueType := range inferredTypes {
if _, exists := env.ParsedFieldTypes[field]; !exists {
env.ParsedFieldTypes[field] = valueType
}
}
}
return true
}
fields, fieldTypes, ok := ParsedFieldsForEnvelope(*env)
fields, fieldTypes, ok := ComputeParsedFieldsForEnvelope(*env)
if !ok {
return false
}
@@ -76,14 +103,27 @@ func EnsureParsedFields(env *envelope.FrameEnvelope) bool {
}
func ParsedFieldsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) {
if len(env.ParsedFields) > 0 {
fields := cloneAnyMap(env.ParsedFields)
fieldTypes := cloneStringMap(env.ParsedFieldTypes)
if len(fieldTypes) == 0 {
fieldTypes = inferParsedFieldTypes(fields)
}
if len(env.ParsedFields) == 0 {
return nil, nil, false
}
fields := cloneAnyMap(env.ParsedFields)
fieldTypes := cloneStringMap(env.ParsedFieldTypes)
if len(fieldTypes) == 0 {
fieldTypes = inferParsedFieldTypes(fields)
}
return fields, fieldTypes, true
}
// ComputeParsedFieldsForEnvelope is reserved for ingress and offline backfills.
// Runtime projections must consume the precomputed ParsedFields contract instead.
func ComputeParsedFieldsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) {
if fields, fieldTypes, ok := ParsedFieldsForEnvelope(env); ok {
return fields, fieldTypes, true
}
return computeParsedFieldsFromParsed(env)
}
func computeParsedFieldsFromParsed(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) {
rows := realtimeKVFields(env, env.Parsed)
if len(rows) == 0 {
return nil, nil, false
@@ -119,6 +159,22 @@ func inferParsedFieldTypes(fields map[string]any) map[string]string {
return out
}
func filterInvalidRealtimeMeasurementFields(fields map[string]any) {
for field, value := range fields {
if isRealtimeTotalMileageField(field) && !positiveNumber(value) {
delete(fields, field)
}
}
}
func isRealtimeTotalMileageField(field string) bool {
field = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(field), "/", "."))
return field == envelope.FieldTotalMileageKM ||
field == "total_mileage" ||
strings.HasSuffix(field, "."+envelope.FieldTotalMileageKM) ||
strings.HasSuffix(field, ".total_mileage")
}
func cloneAnyMap(in map[string]any) map[string]any {
if len(in) == 0 {
return nil
@@ -191,8 +247,9 @@ const realtimeFieldMappingVersion = "2026-07-03.v1"
func realtimeKVFields(env envelope.FrameEnvelope, parsed map[string]any) []RealtimeKVField {
vin := strings.TrimSpace(env.VIN)
if vin == "" {
return nil
if env.Protocol == envelope.ProtocolGB32960 && len(parsed) > 0 {
parsed = cloneMap(parsed)
normalizeGB32960RealtimeParsed(parsed)
}
eventID := env.StableEventID()
mapping := realtimeMapping(env.Protocol)
@@ -205,6 +262,9 @@ func realtimeKVFields(env envelope.FrameEnvelope, parsed map[string]any) []Realt
}
sort.Strings(names)
for _, name := range names {
if isRealtimeTotalMileageField(realtimeKVFieldPath(domain, name)) && !positiveNumber(flat[name]) {
continue
}
value, valueType, ok := stringifyKVValue(flat[name])
if !ok {
continue
@@ -255,6 +315,9 @@ func gb32960KVFields(env envelope.FrameEnvelope, parsed map[string]any, mapping
}
sort.Strings(names)
for _, name := range names {
if isRealtimeTotalMileageField(realtimeKVFieldPath(domain, name)) && !positiveNumber(flat[name]) {
continue
}
value, valueType, ok := stringifyKVValue(flat[name])
if !ok {
continue
@@ -335,6 +398,12 @@ func flattenKV(prefix string, value any, out map[string]any, mapping protocolFie
}
flattenKV(next, typed[key], out, mapping)
}
case map[string]bool:
normalized := make(map[string]any, len(typed))
for key, item := range typed {
normalized[key] = item
}
flattenKV(prefix, normalized, out, mapping)
case []any:
for index, item := range typed {
itemMap, ok := item.(map[string]any)
@@ -397,8 +466,14 @@ func mappedTopLevelDomain(mapping protocolFieldMapping, key string) string {
if key == "" {
return ""
}
name := mapping.TopLevelName[key]
if name == "" {
name := ""
if len(mapping.TopLevelName) > 0 {
var ok bool
name, ok = mapping.TopLevelName[key]
if !ok {
return ""
}
} else {
name = sanitizeFieldPart(key)
}
if name == "" {