feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
242
go/vehicle-gateway/cmd/fields-projector/main_test.go
Normal file
242
go/vehicle-gateway/cmd/fields-projector/main_test.go
Normal file
@@ -0,0 +1,242 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
|
||||
)
|
||||
|
||||
func TestLoadConfigCreatesProtocolIsolatedConsumerGroups(t *testing.T) {
|
||||
t.Setenv("FIELDS_PROJECTOR_GROUP_PREFIX", "projector-test")
|
||||
t.Setenv("KAFKA_START_OFFSET", "first")
|
||||
cfg := loadConfig()
|
||||
if cfg.StartOffset != kafka.FirstOffset || cfg.StartOffsetName != "first" {
|
||||
t.Fatalf("start offset = %d/%q", cfg.StartOffset, cfg.StartOffsetName)
|
||||
}
|
||||
wantGroups := map[envelope.Protocol]string{
|
||||
envelope.ProtocolGB32960: "projector-test-gb32960",
|
||||
envelope.ProtocolJT808: "projector-test-jt808",
|
||||
envelope.ProtocolYutongMQTT: "projector-test-yutong-mqtt",
|
||||
}
|
||||
for _, route := range cfg.Routes {
|
||||
if route.GroupID != wantGroups[route.Protocol] {
|
||||
t.Fatalf("group for %s = %q", route.Protocol, route.GroupID)
|
||||
}
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectBatchUsesPrecomputedFieldsAndPreservesSourceMetadata(t *testing.T) {
|
||||
raw := projectorRawEnvelope()
|
||||
payload, err := raw.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatalf("marshal raw: %v", err)
|
||||
}
|
||||
writer := &recordingProjectorWriter{}
|
||||
registry := metrics.NewRegistry()
|
||||
route := jt808ProjectionRoute()
|
||||
count, err := projectBatch(context.Background(), discardProjectorLogger{}, registry, writer, route, []kafka.Message{{
|
||||
Topic: route.RawTopic, Key: raw.KafkaKey(), Value: payload, Partition: 2, Offset: 10, HighWaterMark: 11,
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("projectBatch() error = %v", err)
|
||||
}
|
||||
if count != 1 || writer.callCount() != 1 || len(writer.messages) != 1 {
|
||||
t.Fatalf("projected=%d calls=%d messages=%d", count, writer.callCount(), len(writer.messages))
|
||||
}
|
||||
var fields envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(writer.messages[0].Value, &fields); err != nil {
|
||||
t.Fatalf("decode fields: %v", err)
|
||||
}
|
||||
if fields.EventKind != envelope.EventKindFields || fields.SourceEventID != raw.EventID || fields.EventID != raw.EventID+":fields" {
|
||||
t.Fatalf("fields identity = %#v", fields)
|
||||
}
|
||||
if fields.SourceCode != raw.SourceCode || fields.SourceKind != raw.SourceKind || fields.SourceEndpoint != raw.SourceEndpoint {
|
||||
t.Fatalf("source metadata not preserved: %#v", fields)
|
||||
}
|
||||
if got := fields.Fields["jt808.location.total_mileage_km"]; got != 1234.5 {
|
||||
t.Fatalf("total mileage = %#v", got)
|
||||
}
|
||||
if len(fields.Parsed) != 0 || len(fields.ParsedFields) != 0 {
|
||||
t.Fatalf("fields projection must not duplicate raw payload: parsed=%v parsed_fields=%v", fields.Parsed, fields.ParsedFields)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fields_projector_projections_total{protocol="JT808",status="projected"} 1`,
|
||||
`vehicle_fields_projector_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
||||
`vehicle_fields_projector_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("metric missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessBatchReliablySkipsNonRealtimeAndInvalidWithoutWriting(t *testing.T) {
|
||||
route := jt808ProjectionRoute()
|
||||
nonRealtime := projectorRawEnvelope()
|
||||
nonRealtime.MessageID = "0x0002"
|
||||
nonRealtime.ParsedFields = map[string]any{"jt808.header.message_id": "0x0002"}
|
||||
payload, _ := nonRealtime.MarshalJSONBytes()
|
||||
messages := []kafka.Message{
|
||||
{Topic: route.RawTopic, Value: []byte("{bad"), Partition: 0, Offset: 1, HighWaterMark: 3},
|
||||
{Topic: route.RawTopic, Value: payload, Partition: 0, Offset: 2, HighWaterMark: 3},
|
||||
}
|
||||
writer := &recordingProjectorWriter{}
|
||||
committer := &recordingProjectorCommitter{}
|
||||
registry := metrics.NewRegistry()
|
||||
processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond)
|
||||
if writer.callCount() != 0 {
|
||||
t.Fatalf("writer calls = %d, want 0", writer.callCount())
|
||||
}
|
||||
if committer.callCount() != 1 || committer.messageCount != 2 {
|
||||
t.Fatalf("commit calls/messages = %d/%d", committer.callCount(), committer.messageCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if !strings.Contains(text, `vehicle_fields_projector_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`) ||
|
||||
!strings.Contains(text, `vehicle_fields_projector_projections_total{protocol="JT808",status="skipped_non_realtime"} 1`) {
|
||||
t.Fatalf("skip metrics missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessBatchReliablyRetriesWriteBeforeCommitting(t *testing.T) {
|
||||
route := jt808ProjectionRoute()
|
||||
payload, _ := projectorRawEnvelope().MarshalJSONBytes()
|
||||
messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}}
|
||||
writer := &recordingProjectorWriter{errors: []error{errors.New("kafka unavailable"), nil}}
|
||||
committer := &recordingProjectorCommitter{}
|
||||
registry := metrics.NewRegistry()
|
||||
processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond)
|
||||
if writer.callCount() != 2 {
|
||||
t.Fatalf("writer calls = %d, want 2", writer.callCount())
|
||||
}
|
||||
if committer.callCount() != 1 {
|
||||
t.Fatalf("commit calls = %d, want 1 after successful write", committer.callCount())
|
||||
}
|
||||
if !strings.Contains(registry.Render(), `vehicle_fields_projector_batch_retries_total{protocol="JT808",reason="write_error"} 1`) {
|
||||
t.Fatalf("write retry metric missing:\n%s", registry.Render())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessBatchReliablyRetriesOnlyCommitAfterSuccessfulWrite(t *testing.T) {
|
||||
route := jt808ProjectionRoute()
|
||||
payload, _ := projectorRawEnvelope().MarshalJSONBytes()
|
||||
messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}}
|
||||
writer := &recordingProjectorWriter{}
|
||||
committer := &recordingProjectorCommitter{errors: []error{errors.New("commit timeout"), nil}}
|
||||
processBatchReliably(context.Background(), discardProjectorLogger{}, metrics.NewRegistry(), writer, committer, route, messages, time.Second, time.Millisecond)
|
||||
if writer.callCount() != 1 {
|
||||
t.Fatalf("writer calls = %d, want 1", writer.callCount())
|
||||
}
|
||||
if committer.callCount() != 2 {
|
||||
t.Fatalf("commit calls = %d, want 2", committer.callCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectBatchWriteFailureDoesNotCommitByItself(t *testing.T) {
|
||||
route := jt808ProjectionRoute()
|
||||
payload, _ := projectorRawEnvelope().MarshalJSONBytes()
|
||||
writer := &recordingProjectorWriter{errors: []error{errors.New("write failed")}}
|
||||
count, err := projectBatch(context.Background(), discardProjectorLogger{}, nil, writer, route, []kafka.Message{{Topic: route.RawTopic, Value: payload}})
|
||||
if err == nil || count != 1 {
|
||||
t.Fatalf("projectBatch() count/error = %d/%v", count, err)
|
||||
}
|
||||
}
|
||||
|
||||
func projectorRawEnvelope() envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
EventID: "raw-event-1",
|
||||
EventKind: envelope.EventKindRaw,
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
VIN: "VIN001",
|
||||
Phone: "13307795425",
|
||||
SourceEndpoint: "115.231.168.135:43625",
|
||||
SourceCode: "g7s",
|
||||
SourceKind: "PLATFORM",
|
||||
PlatformName: "G7s",
|
||||
EventTimeMS: 1783960000000,
|
||||
ReceivedAtMS: 1783960000010,
|
||||
ParsedFields: map[string]any{
|
||||
"jt808.location.latitude": 30.1,
|
||||
"jt808.location.longitude": 121.2,
|
||||
"jt808.location.total_mileage_km": 1234.5,
|
||||
},
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
}
|
||||
|
||||
func jt808ProjectionRoute() projectionRoute {
|
||||
return projectionRoute{
|
||||
Protocol: envelope.ProtocolJT808, RawTopic: topics.RawJT808, FieldsTopic: topics.FieldsJT808, GroupID: "projector-jt808",
|
||||
}
|
||||
}
|
||||
|
||||
type recordingProjectorWriter struct {
|
||||
mu sync.Mutex
|
||||
errors []error
|
||||
calls int
|
||||
messages []kafka.Message
|
||||
}
|
||||
|
||||
func (w *recordingProjectorWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.calls++
|
||||
w.messages = append(w.messages, messages...)
|
||||
if len(w.errors) == 0 {
|
||||
return nil
|
||||
}
|
||||
err := w.errors[0]
|
||||
w.errors = w.errors[1:]
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *recordingProjectorWriter) callCount() int {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.calls
|
||||
}
|
||||
|
||||
type recordingProjectorCommitter struct {
|
||||
mu sync.Mutex
|
||||
errors []error
|
||||
calls int
|
||||
messageCount int
|
||||
}
|
||||
|
||||
func (c *recordingProjectorCommitter) CommitMessages(_ context.Context, messages ...kafka.Message) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.calls++
|
||||
c.messageCount += len(messages)
|
||||
if len(c.errors) == 0 {
|
||||
return nil
|
||||
}
|
||||
err := c.errors[0]
|
||||
c.errors = c.errors[1:]
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *recordingProjectorCommitter) callCount() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.calls
|
||||
}
|
||||
|
||||
type discardProjectorLogger struct{}
|
||||
|
||||
func (discardProjectorLogger) Error(string, ...any) {}
|
||||
func (discardProjectorLogger) Warn(string, ...any) {}
|
||||
Reference in New Issue
Block a user