shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
View File

@@ -7,6 +7,7 @@ import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"log/slog"
"math/big"
"os"
@@ -19,7 +20,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
func TestMQTTClientHandleMessagePublishesOnlyRawByDefault(t *testing.T) {
func TestMQTTClientHandleMessagePublishesRawAndFieldsByDefault(t *testing.T) {
sink := &recordingSink{}
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
@@ -39,18 +40,80 @@ func TestMQTTClientHandleMessagePublishesOnlyRawByDefault(t *testing.T) {
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
}`))
if len(sink.raw) != 1 || len(sink.unified) != 0 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
if len(sink.raw) != 1 || len(sink.fields) != 1 || len(sink.unified) != 0 {
t.Fatalf("raw=%d fields=%d unified=%d", len(sink.raw), len(sink.fields), len(sink.unified))
}
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
t.Fatalf("unexpected raw envelope: %#v", sink.raw[0])
}
if sink.raw[0].EventKind != envelope.EventKindRaw {
t.Fatalf("raw event kind = %q, want %q", sink.raw[0].EventKind, envelope.EventKindRaw)
}
if sink.raw[0].RawText == "" {
t.Fatal("mqtt raw envelope should keep text payload")
}
if sink.raw[0].RawHex != "" {
t.Fatalf("mqtt raw envelope should not duplicate text payload as hex: %q", sink.raw[0].RawHex)
}
if len(sink.raw[0].Fields) != 0 {
t.Fatalf("canonical raw must not carry bare standardized fields: %#v", sink.raw[0].Fields)
}
if got, want := sink.fields[0].Fields["yutong_mqtt.data.meter_speed"], sink.raw[0].ParsedFields["yutong_mqtt.data.meter_speed"]; got != want {
t.Fatalf("fields event should reuse raw parsed field, got %#v want %#v", got, want)
}
if sink.fields[0].EventKind != envelope.EventKindFields {
t.Fatalf("fields event kind = %q, want %q", sink.fields[0].EventKind, envelope.EventKindFields)
}
if got, want := sink.fields[0].SourceEventID, sink.raw[0].StableEventID(); got != want {
t.Fatalf("fields source event id = %#v, want %s", got, want)
}
if sink.fields[0].FieldMapping == "" {
t.Fatal("fields event should expose field mapping version")
}
if len(sink.fields[0].Parsed) != 0 || len(sink.fields[0].ParsedFields) != 0 {
t.Fatalf("fields envelope should not duplicate parsed payload: parsed=%#v parsed_fields=%#v", sink.fields[0].Parsed, sink.fields[0].ParsedFields)
}
}
func TestMQTTClientDelegatesFieldsProjectionWhenConfigured(t *testing.T) {
sink := &recordingSink{}
registry := metrics.NewRegistry()
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
Metrics: registry,
DelegateFields: true,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
"device":"LTEST000000000001",
"time":"20260413100000",
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
}`))
if len(sink.raw) != 1 || len(sink.fields) != 0 {
t.Fatalf("raw=%d fields=%d, want canonical raw only", len(sink.raw), len(sink.fields))
}
if len(sink.raw[0].Fields) != 0 {
t.Fatalf("delegated canonical raw must not carry bare standardized fields: %#v", sink.raw[0].Fields)
}
text := registry.Render()
for _, want := range []string{
`vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="delegated_to_bridge"} 1`,
`vehicle_gateway_fields_count{protocol="YUTONG_MQTT",status="delegated_to_bridge"} `,
`vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="delegated"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("delegated fields metric missing %s:\n%s", want, text)
}
}
}
func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
@@ -77,8 +140,18 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
text := registry.Render()
for _, want := range []string{
`vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1`,
`vehicle_gateway_last_frame_unix_seconds{protocol="YUTONG_MQTT",status="OK"} `,
`vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="resolved"} 1`,
`vehicle_gateway_identity_duration_ms{protocol="YUTONG_MQTT",status="resolved"}`,
`vehicle_gateway_identity_duration_ms_histogram_bucket{le="+Inf",protocol="YUTONG_MQTT",status="resolved"} 1`,
`vehicle_gateway_identity_duration_ms_histogram_count{protocol="YUTONG_MQTT",status="resolved"} 1`,
`vehicle_gateway_identity_duration_ms_histogram_sum{protocol="YUTONG_MQTT",status="resolved"}`,
`vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 1`,
`vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="YUTONG_MQTT",status="ok"} `,
`vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="published"} 1`,
`vehicle_gateway_fields_count{protocol="YUTONG_MQTT",status="published"} `,
`vehicle_gateway_fields_count_histogram_count{protocol="YUTONG_MQTT",status="published"} 1`,
`vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 1`,
`vehicle_gateway_frame_duration_ms{protocol="YUTONG_MQTT",status="OK"}`,
`vehicle_gateway_frame_duration_ms_histogram_bucket{le="+Inf",protocol="YUTONG_MQTT",status="OK"} 1`,
`vehicle_gateway_frame_duration_ms_histogram_count{protocol="YUTONG_MQTT",status="OK"} 1`,
@@ -93,6 +166,36 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
}
}
func TestMQTTClientRecordsNonRealtimeFieldsSkipMetric(t *testing.T) {
registry := metrics.NewRegistry()
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
Sink: &recordingSink{},
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
Metrics: registry,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
"device":"LTEST000000000001",
"time":"20260413100000",
"data":{}
}`))
text := registry.Render()
if !strings.Contains(text, `vehicle_gateway_identity_skips_total{protocol="YUTONG_MQTT",reason="non_vehicle_frame"} 1`) {
t.Fatalf("identity skip metric missing:\n%s", text)
}
if !strings.Contains(text, `vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="skipped_non_realtime"} 1`) {
t.Fatalf("non realtime fields skip metric missing:\n%s", text)
}
}
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
sink := &recordingSink{}
registry := metrics.NewRegistry()
@@ -167,6 +270,66 @@ func TestMQTTClientUsesUncancelledMessageContextForReceivedMessage(t *testing.T)
}
}
func TestMQTTClientPreservesResolvedEnvelopeWhenIdentitySideEffectFails(t *testing.T) {
sink := &recordingSink{}
registry := metrics.NewRegistry()
wantErr := errors.New("registration upsert failed")
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
Sink: sink,
Resolver: resolvedErrorResolver{vin: "LRESOLVED00000001", err: wantErr},
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
Metrics: registry,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
"device":"LTEST000000000001",
"time":"20260413100000",
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
}`))
if len(sink.raw) != 1 {
t.Fatalf("raw=%d, want 1", len(sink.raw))
}
if sink.raw[0].VIN != "LRESOLVED00000001" {
t.Fatalf("raw vin = %q, want resolver vin", sink.raw[0].VIN)
}
if sink.raw[0].ParseStatus != envelope.ParsePartial {
t.Fatalf("parse status = %q, want PARTIAL", sink.raw[0].ParseStatus)
}
if len(sink.raw[0].Parsed) != 0 {
t.Fatalf("canonical raw should not duplicate parsed tree: %#v", sink.raw[0].Parsed)
}
if got := sink.raw[0].ParsedFields["yutong_mqtt.data.meter_speed"]; got != "52.3" {
t.Fatalf("protocol field lost after identity side-effect failure: %#v", got)
}
for field := range sink.raw[0].ParsedFields {
if strings.HasPrefix(field, "yutong_mqtt.identity.") {
t.Fatalf("derived identity annotation leaked into protocol fields: %s", field)
}
}
if len(sink.fields) != 1 || sink.fields[0].VIN != "LRESOLVED00000001" {
t.Fatalf("fields should preserve resolved vin, fields=%#v", sink.fields)
}
text := registry.Render()
for _, want := range []string{
`vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="error"} 1`,
`vehicle_gateway_identity_issues_total{message_id="MQTT",protocol="YUTONG_MQTT",reason="resolver_error",status="error"} 1`,
`vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="PARTIAL"} 1`,
`vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="published"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("metric missing %s:\n%s", want, text)
}
}
}
func TestMQTTClientPublishesUnifiedWhenExplicitlyEnabled(t *testing.T) {
sink := &recordingSink{}
client, err := NewMQTTClient(MQTTClientConfig{
@@ -193,6 +356,44 @@ func TestMQTTClientPublishesUnifiedWhenExplicitlyEnabled(t *testing.T) {
}
}
func TestMQTTClientPublishesUnifiedWhenFieldsPublishFails(t *testing.T) {
sink := &recordingSink{fieldsErr: errors.New("fields queue full")}
registry := metrics.NewRegistry()
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
PublishUnified: true,
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
Metrics: registry,
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
"device":"LTEST000000000001",
"time":"20260413100000",
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
}`))
if len(sink.raw) != 1 || len(sink.fields) != 1 || len(sink.unified) != 1 {
t.Fatalf("raw=%d fields=%d unified=%d", len(sink.raw), len(sink.fields), len(sink.unified))
}
text := registry.Render()
for _, want := range []string{
`vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="publish_error"} 1`,
`vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="error"} 1`,
`vehicle_gateway_publish_total{kind="unified",protocol="YUTONG_MQTT",status="ok"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("metric missing %s:\n%s", want, text)
}
}
}
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
dir := t.TempDir()
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)