feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
@@ -1,5 +1,13 @@
|
||||
package topics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
const (
|
||||
RawGB32960 = "vehicle.raw.go.gb32960.v1"
|
||||
RawJT808 = "vehicle.raw.go.jt808.v1"
|
||||
@@ -9,3 +17,196 @@ const (
|
||||
FieldsYutongMQTT = "vehicle.fields.go.yutong-mqtt.v1"
|
||||
Unified = "vehicle.event.go.unified.v1"
|
||||
)
|
||||
|
||||
const (
|
||||
RawPrefix = "vehicle.raw."
|
||||
FieldsPrefix = "vehicle.fields."
|
||||
)
|
||||
|
||||
func ProtocolForKnownRawTopic(topic string) (string, bool) {
|
||||
switch strings.TrimSpace(topic) {
|
||||
case RawGB32960:
|
||||
return "GB32960", true
|
||||
case RawJT808:
|
||||
return "JT808", true
|
||||
case RawYutongMQTT:
|
||||
return "YUTONG_MQTT", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func ProtocolForKnownFieldsTopic(topic string) (string, bool) {
|
||||
switch strings.TrimSpace(topic) {
|
||||
case FieldsGB32960:
|
||||
return "GB32960", true
|
||||
case FieldsJT808:
|
||||
return "JT808", true
|
||||
case FieldsYutongMQTT:
|
||||
return "YUTONG_MQTT", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func ValidateKnownRawTopicProtocol(topic string, protocol string) error {
|
||||
expectedProtocol, ok := ProtocolForKnownRawTopic(topic)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
protocol = strings.TrimSpace(protocol)
|
||||
if protocol == expectedProtocol {
|
||||
return nil
|
||||
}
|
||||
if protocol == "" {
|
||||
protocol = "UNKNOWN"
|
||||
}
|
||||
return fmt.Errorf("raw topic %q expects protocol %s, got %s", strings.TrimSpace(topic), expectedProtocol, protocol)
|
||||
}
|
||||
|
||||
func ValidateKnownFieldsTopicProtocol(topic string, protocol string) error {
|
||||
expectedProtocol, ok := ProtocolForKnownFieldsTopic(topic)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
protocol = strings.TrimSpace(protocol)
|
||||
if protocol == expectedProtocol {
|
||||
return nil
|
||||
}
|
||||
if protocol == "" {
|
||||
protocol = "UNKNOWN"
|
||||
}
|
||||
return fmt.Errorf("fields topic %q expects protocol %s, got %s", strings.TrimSpace(topic), expectedProtocol, protocol)
|
||||
}
|
||||
|
||||
func ValidateRawEnvelope(topic string, env envelope.FrameEnvelope) (string, error) {
|
||||
if err := ValidateKnownRawTopicProtocol(topic, string(env.Protocol)); err != nil {
|
||||
return "protocol_topic_mismatch", err
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSpace(topic), RawPrefix) {
|
||||
switch env.EventKind {
|
||||
case "", envelope.EventKindRaw:
|
||||
return "", nil
|
||||
default:
|
||||
return "event_kind_mismatch", fmt.Errorf("raw topic %q expects event kind %s, got %s", strings.TrimSpace(topic), envelope.EventKindRaw, env.EventKind)
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func ValidateFieldsEnvelope(topic string, env envelope.FrameEnvelope) (string, error) {
|
||||
if err := ValidateKnownFieldsTopicProtocol(topic, string(env.Protocol)); err != nil {
|
||||
return "protocol_topic_mismatch", err
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSpace(topic), FieldsPrefix) {
|
||||
if env.EventKind != envelope.EventKindFields {
|
||||
return "event_kind_mismatch", fmt.Errorf("fields topic %q expects event kind %s, got %s", strings.TrimSpace(topic), envelope.EventKindFields, env.EventKind)
|
||||
}
|
||||
if strings.TrimSpace(env.FieldMapping) == "" {
|
||||
return "missing_field_mapping", fmt.Errorf("fields topic %q expects non-empty field mapping", strings.TrimSpace(topic))
|
||||
}
|
||||
if len(env.Fields) == 0 {
|
||||
return "missing_fields", fmt.Errorf("fields topic %q expects non-empty fields payload", strings.TrimSpace(topic))
|
||||
}
|
||||
if err := ValidateProtocolFieldNames(env.Protocol, env.Fields); err != nil {
|
||||
return "invalid_field_name", err
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func ValidateProtocolFieldNames(protocol envelope.Protocol, fields map[string]any) error {
|
||||
prefix, ok := protocolFieldPrefix(protocol)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
invalid := make([]string, 0)
|
||||
for key := range fields {
|
||||
trimmed := strings.TrimSpace(key)
|
||||
if key == trimmed && len(trimmed) > len(prefix) && strings.HasPrefix(trimmed, prefix) {
|
||||
continue
|
||||
}
|
||||
invalid = append(invalid, key)
|
||||
}
|
||||
if len(invalid) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(invalid)
|
||||
if len(invalid) > 3 {
|
||||
invalid = invalid[:3]
|
||||
}
|
||||
return fmt.Errorf("fields protocol %s expects every field name under %q; invalid fields: %q", protocol, prefix, invalid)
|
||||
}
|
||||
|
||||
func protocolFieldPrefix(protocol envelope.Protocol) (string, bool) {
|
||||
switch protocol {
|
||||
case envelope.ProtocolGB32960:
|
||||
return "gb32960.", true
|
||||
case envelope.ProtocolJT808:
|
||||
return "jt808.", true
|
||||
case envelope.ProtocolYutongMQTT:
|
||||
return "yutong_mqtt.", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func ValidateKnownRawFieldsProtocols(raw map[string]string, fields map[string]string, valueName string) error {
|
||||
for protocol, value := range raw {
|
||||
if err := ValidateKnownRawTopicProtocol(value, protocol); err != nil {
|
||||
return fmt.Errorf("raw %s for %s must match protocol: %w", valueName, protocol, err)
|
||||
}
|
||||
}
|
||||
for protocol, value := range fields {
|
||||
if err := ValidateKnownFieldsTopicProtocol(value, protocol); err != nil {
|
||||
return fmt.Errorf("fields %s for %s must match protocol: %w", valueName, protocol, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateKafkaRawFields(raw map[string]string, fields map[string]string) error {
|
||||
for name, topic := range raw {
|
||||
topic = strings.TrimSpace(topic)
|
||||
if topic == "" {
|
||||
return fmt.Errorf("raw kafka topic for %s is empty", name)
|
||||
}
|
||||
if !strings.HasPrefix(topic, RawPrefix) {
|
||||
return fmt.Errorf("raw kafka topic for %s must start with %q, got %q", name, RawPrefix, topic)
|
||||
}
|
||||
}
|
||||
for name, topic := range fields {
|
||||
topic = strings.TrimSpace(topic)
|
||||
if topic == "" {
|
||||
return fmt.Errorf("fields kafka topic for %s is empty", name)
|
||||
}
|
||||
if !strings.HasPrefix(topic, FieldsPrefix) {
|
||||
return fmt.Errorf("fields kafka topic for %s must start with %q, got %q", name, FieldsPrefix, topic)
|
||||
}
|
||||
}
|
||||
if err := ValidateKnownRawFieldsProtocols(raw, fields, "kafka topic"); err != nil {
|
||||
return err
|
||||
}
|
||||
return ValidateRawFieldsDisjoint(raw, fields, "kafka topic")
|
||||
}
|
||||
|
||||
func ValidateRawFieldsDisjoint(raw map[string]string, fields map[string]string, valueName string) error {
|
||||
rawByValue := map[string]string{}
|
||||
for name, value := range raw {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
rawByValue[value] = name
|
||||
}
|
||||
for fieldsName, value := range fields {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
if rawName, ok := rawByValue[value]; ok {
|
||||
return fmt.Errorf("raw and fields %s must be different: raw %s and fields %s both use %q", valueName, rawName, fieldsName, value)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
249
go/vehicle-gateway/internal/topics/topics_test.go
Normal file
249
go/vehicle-gateway/internal/topics/topics_test.go
Normal file
@@ -0,0 +1,249 @@
|
||||
package topics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestValidateKafkaRawFieldsRejectsWrongFamilyPrefix(t *testing.T) {
|
||||
err := ValidateKafkaRawFields(
|
||||
map[string]string{"JT808": FieldsJT808},
|
||||
map[string]string{"JT808": FieldsJT808},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("ValidateKafkaRawFields() error = nil, want raw prefix rejection")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "raw kafka topic") {
|
||||
t.Fatalf("error = %q, want raw kafka topic hint", err)
|
||||
}
|
||||
|
||||
err = ValidateKafkaRawFields(
|
||||
map[string]string{"JT808": RawJT808},
|
||||
map[string]string{"JT808": RawJT808},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("ValidateKafkaRawFields() error = nil, want fields prefix rejection")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "fields kafka topic") {
|
||||
t.Fatalf("error = %q, want fields kafka topic hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRawFieldsDisjointRejectsSharedValue(t *testing.T) {
|
||||
err := ValidateRawFieldsDisjoint(
|
||||
map[string]string{"raw-jt808": "vehicle.same.jt808"},
|
||||
map[string]string{"fields-jt808": "vehicle.same.jt808"},
|
||||
"nats subject",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("ValidateRawFieldsDisjoint() error = nil, want duplicate rejection")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must be different") {
|
||||
t.Fatalf("error = %q, want disjoint hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateKnownFieldsTopicProtocolRejectsMismatchedProtocol(t *testing.T) {
|
||||
err := ValidateKnownFieldsTopicProtocol(FieldsGB32960, "JT808")
|
||||
if err == nil {
|
||||
t.Fatal("ValidateKnownFieldsTopicProtocol() error = nil, want mismatch rejection")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "expects protocol GB32960") {
|
||||
t.Fatalf("error = %q, want expected protocol hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateKnownRawTopicProtocolRejectsMismatchedProtocol(t *testing.T) {
|
||||
err := ValidateKnownRawTopicProtocol(RawJT808, "GB32960")
|
||||
if err == nil {
|
||||
t.Fatal("ValidateKnownRawTopicProtocol() error = nil, want mismatch rejection")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "expects protocol JT808") {
|
||||
t.Fatalf("error = %q, want expected protocol hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateKnownTopicProtocolAllowsMatchingAndCustomTopics(t *testing.T) {
|
||||
if err := ValidateKnownRawTopicProtocol(RawGB32960, "GB32960"); err != nil {
|
||||
t.Fatalf("ValidateKnownRawTopicProtocol() matching topic error = %v", err)
|
||||
}
|
||||
if err := ValidateKnownRawTopicProtocol("vehicle.raw.custom.g7", "JT808"); err != nil {
|
||||
t.Fatalf("ValidateKnownRawTopicProtocol() custom topic error = %v", err)
|
||||
}
|
||||
if err := ValidateKnownFieldsTopicProtocol(FieldsJT808, "JT808"); err != nil {
|
||||
t.Fatalf("ValidateKnownFieldsTopicProtocol() matching topic error = %v", err)
|
||||
}
|
||||
if err := ValidateKnownFieldsTopicProtocol("vehicle.fields.custom.g7", "JT808"); err != nil {
|
||||
t.Fatalf("ValidateKnownFieldsTopicProtocol() custom topic error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRawEnvelopeRejectsExplicitNonRawEventKind(t *testing.T) {
|
||||
status, err := ValidateRawEnvelope(RawJT808, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
EventKind: envelope.EventKindFields,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ValidateRawEnvelope() error = nil, want event kind mismatch")
|
||||
}
|
||||
if status != "event_kind_mismatch" {
|
||||
t.Fatalf("status = %q, want event_kind_mismatch", status)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "expects event kind RAW") {
|
||||
t.Fatalf("error = %q, want RAW hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRawEnvelopeAllowsHistoricalEmptyAndRawEventKind(t *testing.T) {
|
||||
for _, kind := range []envelope.EventKind{"", envelope.EventKindRaw} {
|
||||
if status, err := ValidateRawEnvelope(RawGB32960, envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, EventKind: kind}); err != nil {
|
||||
t.Fatalf("ValidateRawEnvelope(%q) status=%q error=%v", kind, status, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateFieldsEnvelopeRejectsExplicitNonFieldsEventKind(t *testing.T) {
|
||||
status, err := ValidateFieldsEnvelope(FieldsYutongMQTT, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
EventKind: envelope.EventKindRaw,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ValidateFieldsEnvelope() error = nil, want event kind mismatch")
|
||||
}
|
||||
if status != "event_kind_mismatch" {
|
||||
t.Fatalf("status = %q, want event_kind_mismatch", status)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "expects event kind FIELDS") {
|
||||
t.Fatalf("error = %q, want FIELDS hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateFieldsEnvelopeRequiresFieldsContract(t *testing.T) {
|
||||
status, err := ValidateFieldsEnvelope(FieldsJT808, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
EventKind: envelope.EventKindFields,
|
||||
Fields: map[string]any{"jt808.location.total_mileage_km": 10241.2},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ValidateFieldsEnvelope() error = nil, want missing field mapping")
|
||||
}
|
||||
if status != "missing_field_mapping" {
|
||||
t.Fatalf("status = %q, want missing_field_mapping", status)
|
||||
}
|
||||
|
||||
status, err = ValidateFieldsEnvelope(FieldsJT808, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
EventKind: envelope.EventKindFields,
|
||||
FieldMapping: "2026-07-03.v1",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ValidateFieldsEnvelope() error = nil, want missing fields")
|
||||
}
|
||||
if status != "missing_fields" {
|
||||
t.Fatalf("status = %q, want missing_fields", status)
|
||||
}
|
||||
|
||||
status, err = ValidateFieldsEnvelope(FieldsJT808, envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
EventKind: envelope.EventKindFields,
|
||||
FieldMapping: "2026-07-03.v1",
|
||||
Fields: map[string]any{"jt808.location.total_mileage_km": 10241.2},
|
||||
})
|
||||
if err != nil || status != "" {
|
||||
t.Fatalf("ValidateFieldsEnvelope() status=%q error=%v, want ok", status, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateFieldsEnvelopeRejectsNonProtocolFieldNames(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
topic string
|
||||
protocol envelope.Protocol
|
||||
fields map[string]any
|
||||
}{
|
||||
{
|
||||
name: "normalized core field",
|
||||
topic: FieldsJT808,
|
||||
protocol: envelope.ProtocolJT808,
|
||||
fields: map[string]any{"total_mileage_km": 10241.2},
|
||||
},
|
||||
{
|
||||
name: "other protocol namespace",
|
||||
topic: FieldsGB32960,
|
||||
protocol: envelope.ProtocolGB32960,
|
||||
fields: map[string]any{"jt808.location.latitude": 30.1},
|
||||
},
|
||||
{
|
||||
name: "empty suffix",
|
||||
topic: FieldsYutongMQTT,
|
||||
protocol: envelope.ProtocolYutongMQTT,
|
||||
fields: map[string]any{"yutong_mqtt.": 1},
|
||||
},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
status, err := ValidateFieldsEnvelope(tt.topic, envelope.FrameEnvelope{
|
||||
Protocol: tt.protocol,
|
||||
EventKind: envelope.EventKindFields,
|
||||
FieldMapping: "2026-07-03.v1",
|
||||
Fields: tt.fields,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ValidateFieldsEnvelope() error = nil, want invalid field rejection")
|
||||
}
|
||||
if status != "invalid_field_name" {
|
||||
t.Fatalf("status = %q, want invalid_field_name", status)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "expects every field name") {
|
||||
t.Fatalf("error = %q, want protocol namespace hint", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateFieldsEnvelopeAcceptsProtocolNamespaces(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
topic string
|
||||
protocol envelope.Protocol
|
||||
field string
|
||||
}{
|
||||
{topic: FieldsGB32960, protocol: envelope.ProtocolGB32960, field: "gb32960.vehicle.total_mileage_km"},
|
||||
{topic: FieldsJT808, protocol: envelope.ProtocolJT808, field: "jt808.location.total_mileage_km"},
|
||||
{topic: FieldsYutongMQTT, protocol: envelope.ProtocolYutongMQTT, field: "yutong_mqtt.root.data.total_mileage"},
|
||||
} {
|
||||
status, err := ValidateFieldsEnvelope(tt.topic, envelope.FrameEnvelope{
|
||||
Protocol: tt.protocol,
|
||||
EventKind: envelope.EventKindFields,
|
||||
FieldMapping: "2026-07-03.v1",
|
||||
Fields: map[string]any{tt.field: 1},
|
||||
})
|
||||
if err != nil || status != "" {
|
||||
t.Fatalf("ValidateFieldsEnvelope(%s) status=%q error=%v", tt.protocol, status, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateKafkaRawFieldsRejectsKnownProtocolMismatch(t *testing.T) {
|
||||
err := ValidateKafkaRawFields(
|
||||
map[string]string{"JT808": RawGB32960},
|
||||
map[string]string{"JT808": FieldsJT808},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("ValidateKafkaRawFields() error = nil, want known raw protocol mismatch")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must match protocol") {
|
||||
t.Fatalf("error = %q, want protocol mismatch hint", err)
|
||||
}
|
||||
|
||||
err = ValidateKafkaRawFields(
|
||||
map[string]string{"JT808": RawJT808},
|
||||
map[string]string{"JT808": FieldsGB32960},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("ValidateKafkaRawFields() error = nil, want known fields protocol mismatch")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must match protocol") {
|
||||
t.Fatalf("error = %q, want protocol mismatch hint", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user