shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(go): map realtime fields to semantic names

Browse Source
This commit is contained in:
lingniu
2026-07-03 13:51:57 +08:00
parent c359930009
commit e1bf2d72e1
4 changed files with 306 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
go/vehicle-gateway/internal/realtime/kv.go
View File

@@ -22,15 +22,64 @@ type RealtimeKVField struct {
EventID string
}
type protocolFieldMapping struct {
Prefix string
GBDataUnits bool
TopLevelName map[string]string
ArrayName map[string]string
}
var realtimeFieldMappings = map[envelope.Protocol]protocolFieldMapping{
envelope.ProtocolGB32960: {
Prefix: "gb32960",
GBDataUnits: true,
TopLevelName: map[string]string{
"header": "header",
"platform_name": "platform",
"platform_login": "platform_login",
},
ArrayName: map[string]string{
"motors": "motor",
"probes": "probe",
"subsystems": "subsystem",
"summaries": "",
},
},
envelope.ProtocolJT808: {
Prefix: "jt808",
TopLevelName: map[string]string{
"header": "header",
"location": "location",
"registration": "registration",
"authentication": "authentication",
},
ArrayName: map[string]string{
"additional": "additional",
},
},
envelope.ProtocolYutongMQTT: {
Prefix: "yutong_mqtt",
TopLevelName: map[string]string{
"data": "data",
"root": "root",
"endpoint": "metadata",
"topic": "metadata",
},
},
}
const realtimeFieldMappingVersion = "2026-07-03.v1"
func realtimeKVFields(env envelope.FrameEnvelope, parsed map[string]any) []RealtimeKVField {
vin := strings.TrimSpace(env.VIN)
if vin == "" {
return nil
}
eventID := env.StableEventID()
mapping := realtimeMapping(env.Protocol)
add := func(rows []RealtimeKVField, domain string, fields any) []RealtimeKVField {
flat := map[string]any{}
flattenKV("", fields, flat)
flattenKV("", fields, flat, mapping)
names := make([]string, 0, len(flat))
for name := range flat {
names = append(names, name)
@@ -57,36 +106,92 @@ func realtimeKVFields(env envelope.FrameEnvelope, parsed map[string]any) []Realt
}
var rows []RealtimeKVField
if mapping.GBDataUnits {
rows = append(rows, gb32960KVFields(env, parsed, mapping, eventID)...)
return rows
}
keys := make([]string, 0, len(parsed))
for key := range parsed {
keys = append(keys, key)
}
sort.Strings(keys)
metadata := map[string]any{}
for _, key := range keys {
domain := strings.TrimSpace(key)
domain := mappedTopLevelDomain(mapping, key)
if domain == "" {
continue
}
value := parsed[key]
switch typed := value.(type) {
case map[string]any:
rows = add(rows, domain, typed)
case []any:
rows = add(rows, domain, typed)
case []map[string]any:
rows = add(rows, domain, typed)
default:
metadata[domain] = value
}
}
if len(metadata) > 0 {
rows = add(rows, "metadata", metadata)
rows = addMappedValue(rows, add, domain, key, parsed[key])
}
return rows
}
func flattenKV(prefix string, value any, out map[string]any) {
func gb32960KVFields(env envelope.FrameEnvelope, parsed map[string]any, mapping protocolFieldMapping, eventID string) []RealtimeKVField {
addRows := func(rows []RealtimeKVField, domain string, fields any) []RealtimeKVField {
flat := map[string]any{}
flattenKV("", fields, flat, mapping)
names := make([]string, 0, len(flat))
for name := range flat {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
value, valueType, ok := stringifyKVValue(flat[name])
if !ok {
continue
}
rows = append(rows, RealtimeKVField{
Protocol: env.Protocol,
VIN: strings.TrimSpace(env.VIN),
Domain: domain,
Field: name,
Value: value,
ValueType: valueType,
EventTimeMS: env.EventTimeMS,
ReceivedAtMS: env.ReceivedAtMS,
EventID: eventID,
})
}
return rows
}
var rows []RealtimeKVField
for key, value := range parsed {
if key == "data_units" {
continue
}
if domain := mappedTopLevelDomain(mapping, key); domain != "" {
rows = addMappedValue(rows, addRows, domain, key, value)
}
}
units, ok := asAnySlice(parsed["data_units"])
if !ok {
return rows
}
for _, unit := range units {
unitMap, ok := unit.(map[string]any)
if !ok {
continue
}
name := sanitizeFieldPart(strconvAny(unitMap["name"]))
if name == "" || name == "<nil>" {
name = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(strconvAny(unitMap["type"])), "0x"))
}
if name == "" {
continue
}
domain := mapping.Prefix + "." + name
if value, ok := unitMap["value"]; ok {
rows = addRows(rows, domain, value)
}
if unitType := strings.TrimSpace(strconvAny(unitMap["type"])); unitType != "" && unitType != "<nil>" {
rows = addRows(rows, domain, map[string]any{"type": unitType})
}
}
return rows
}
func flattenKV(prefix string, value any, out map[string]any, mapping protocolFieldMapping) {
switch typed := value.(type) {
case map[string]any:
keys := make([]string, 0, len(typed))
@@ -95,11 +200,21 @@ func flattenKV(prefix string, value any, out map[string]any) {
}
sort.Strings(keys)
for _, key := range keys {
next := key
if prefix != "" {
next = prefix + "." + key
part := sanitizeFieldPart(key)
if part == "" {
continue
}
flattenKV(next, typed[key], out)
if arrayName, ok := mapping.ArrayName[part]; ok && arrayName == "" {
if items, ok := asAnySlice(typed[key]); ok && len(items) == 1 {
flattenKV(prefix, items[0], out, mapping)
continue
}
}
next := part
if prefix != "" {
next = prefix + "." + part
}
flattenKV(next, typed[key], out, mapping)
}
case []any:
for index, item := range typed {
@@ -108,19 +223,29 @@ func flattenKV(prefix string, value any, out map[string]any) {
out[prefix] = typed
return
}
arrayName, hasArrayName := mapping.ArrayName[lastPathPart(prefix)]
itemKey := strconv.Itoa(index)
if hasArrayName {
if arrayName == "" && len(typed) == 1 {
flattenKV(prefix, itemMap, out, mapping)
continue
}
if arrayName != "" {
itemKey = arrayName + "_" + strconv.Itoa(index+1)
}
}
next := itemKey
if prefix != "" {
next = prefix + "." + itemKey
}
flattenKV(next, itemMap, out)
flattenKV(next, itemMap, out, mapping)
}
case []map[string]any:
items := make([]any, len(typed))
for index, item := range typed {
items[index] = item
}
flattenKV(prefix, items, out)
flattenKV(prefix, items, out, mapping)
default:
if prefix != "" && value != nil {
out[prefix] = value
@@ -128,6 +253,71 @@ func flattenKV(prefix string, value any, out map[string]any) {
}
}
func addMappedValue(rows []RealtimeKVField, add func([]RealtimeKVField, string, any) []RealtimeKVField, domain string, key string, value any) []RealtimeKVField {
switch typed := value.(type) {
case map[string]any:
return add(rows, domain, typed)
case []any:
return add(rows, domain, typed)
case []map[string]any:
return add(rows, domain, typed)
default:
return add(rows, domain, map[string]any{key: value})
}
}
func realtimeMapping(protocol envelope.Protocol) protocolFieldMapping {
if mapping, ok := realtimeFieldMappings[protocol]; ok {
return mapping
}
return protocolFieldMapping{Prefix: strings.ToLower(string(protocol))}
}
func mappedTopLevelDomain(mapping protocolFieldMapping, key string) string {
key = strings.TrimSpace(key)
if key == "" {
return ""
}
name := mapping.TopLevelName[key]
if name == "" {
name = sanitizeFieldPart(key)
}
if name == "" {
return ""
}
return mapping.Prefix + "." + name
}
func sanitizeFieldPart(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
var b strings.Builder
for _, r := range value {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= 'A' && r <= 'Z':
b.WriteRune(r + ('a' - 'A'))
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '_':
b.WriteRune(r)
default:
b.WriteByte('_')
}
}
return strings.Trim(b.String(), "_")
}
func lastPathPart(path string) string {
if index := strings.LastIndex(path, "."); index >= 0 {
return path[index+1:]
}
return path
}
func stringifyKVValue(value any) (string, string, bool) {
switch typed := value.(type) {
case string: