243 lines
8.7 KiB
Go
243 lines
8.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/segmentio/kafka-go"
|
|
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
|
|
)
|
|
|
|
func TestLoadConfigCreatesProtocolIsolatedConsumerGroups(t *testing.T) {
|
|
t.Setenv("FIELDS_PROJECTOR_GROUP_PREFIX", "projector-test")
|
|
t.Setenv("KAFKA_START_OFFSET", "first")
|
|
cfg := loadConfig()
|
|
if cfg.StartOffset != kafka.FirstOffset || cfg.StartOffsetName != "first" {
|
|
t.Fatalf("start offset = %d/%q", cfg.StartOffset, cfg.StartOffsetName)
|
|
}
|
|
wantGroups := map[envelope.Protocol]string{
|
|
envelope.ProtocolGB32960: "projector-test-gb32960",
|
|
envelope.ProtocolJT808: "projector-test-jt808",
|
|
envelope.ProtocolYutongMQTT: "projector-test-yutong-mqtt",
|
|
}
|
|
for _, route := range cfg.Routes {
|
|
if route.GroupID != wantGroups[route.Protocol] {
|
|
t.Fatalf("group for %s = %q", route.Protocol, route.GroupID)
|
|
}
|
|
}
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("Validate() error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestProjectBatchUsesPrecomputedFieldsAndPreservesSourceMetadata(t *testing.T) {
|
|
raw := projectorRawEnvelope()
|
|
payload, err := raw.MarshalJSONBytes()
|
|
if err != nil {
|
|
t.Fatalf("marshal raw: %v", err)
|
|
}
|
|
writer := &recordingProjectorWriter{}
|
|
registry := metrics.NewRegistry()
|
|
route := jt808ProjectionRoute()
|
|
count, err := projectBatch(context.Background(), discardProjectorLogger{}, registry, writer, route, []kafka.Message{{
|
|
Topic: route.RawTopic, Key: raw.KafkaKey(), Value: payload, Partition: 2, Offset: 10, HighWaterMark: 11,
|
|
}})
|
|
if err != nil {
|
|
t.Fatalf("projectBatch() error = %v", err)
|
|
}
|
|
if count != 1 || writer.callCount() != 1 || len(writer.messages) != 1 {
|
|
t.Fatalf("projected=%d calls=%d messages=%d", count, writer.callCount(), len(writer.messages))
|
|
}
|
|
var fields envelope.FrameEnvelope
|
|
if err := json.Unmarshal(writer.messages[0].Value, &fields); err != nil {
|
|
t.Fatalf("decode fields: %v", err)
|
|
}
|
|
if fields.EventKind != envelope.EventKindFields || fields.SourceEventID != raw.EventID || fields.EventID != raw.EventID+":fields" {
|
|
t.Fatalf("fields identity = %#v", fields)
|
|
}
|
|
if fields.SourceCode != raw.SourceCode || fields.SourceKind != raw.SourceKind || fields.SourceEndpoint != raw.SourceEndpoint {
|
|
t.Fatalf("source metadata not preserved: %#v", fields)
|
|
}
|
|
if got := fields.Fields["jt808.location.total_mileage_km"]; got != 1234.5 {
|
|
t.Fatalf("total mileage = %#v", got)
|
|
}
|
|
if len(fields.Parsed) != 0 || len(fields.ParsedFields) != 0 {
|
|
t.Fatalf("fields projection must not duplicate raw payload: parsed=%v parsed_fields=%v", fields.Parsed, fields.ParsedFields)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_fields_projector_projections_total{protocol="JT808",status="projected"} 1`,
|
|
`vehicle_fields_projector_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_fields_projector_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 0`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("metric missing %q:\n%s", want, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProcessBatchReliablySkipsNonRealtimeAndInvalidWithoutWriting(t *testing.T) {
|
|
route := jt808ProjectionRoute()
|
|
nonRealtime := projectorRawEnvelope()
|
|
nonRealtime.MessageID = "0x0002"
|
|
nonRealtime.ParsedFields = map[string]any{"jt808.header.message_id": "0x0002"}
|
|
payload, _ := nonRealtime.MarshalJSONBytes()
|
|
messages := []kafka.Message{
|
|
{Topic: route.RawTopic, Value: []byte("{bad"), Partition: 0, Offset: 1, HighWaterMark: 3},
|
|
{Topic: route.RawTopic, Value: payload, Partition: 0, Offset: 2, HighWaterMark: 3},
|
|
}
|
|
writer := &recordingProjectorWriter{}
|
|
committer := &recordingProjectorCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond)
|
|
if writer.callCount() != 0 {
|
|
t.Fatalf("writer calls = %d, want 0", writer.callCount())
|
|
}
|
|
if committer.callCount() != 1 || committer.messageCount != 2 {
|
|
t.Fatalf("commit calls/messages = %d/%d", committer.callCount(), committer.messageCount)
|
|
}
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_fields_projector_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`) ||
|
|
!strings.Contains(text, `vehicle_fields_projector_projections_total{protocol="JT808",status="skipped_non_realtime"} 1`) {
|
|
t.Fatalf("skip metrics missing:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessBatchReliablyRetriesWriteBeforeCommitting(t *testing.T) {
|
|
route := jt808ProjectionRoute()
|
|
payload, _ := projectorRawEnvelope().MarshalJSONBytes()
|
|
messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}}
|
|
writer := &recordingProjectorWriter{errors: []error{errors.New("kafka unavailable"), nil}}
|
|
committer := &recordingProjectorCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond)
|
|
if writer.callCount() != 2 {
|
|
t.Fatalf("writer calls = %d, want 2", writer.callCount())
|
|
}
|
|
if committer.callCount() != 1 {
|
|
t.Fatalf("commit calls = %d, want 1 after successful write", committer.callCount())
|
|
}
|
|
if !strings.Contains(registry.Render(), `vehicle_fields_projector_batch_retries_total{protocol="JT808",reason="write_error"} 1`) {
|
|
t.Fatalf("write retry metric missing:\n%s", registry.Render())
|
|
}
|
|
}
|
|
|
|
func TestProcessBatchReliablyRetriesOnlyCommitAfterSuccessfulWrite(t *testing.T) {
|
|
route := jt808ProjectionRoute()
|
|
payload, _ := projectorRawEnvelope().MarshalJSONBytes()
|
|
messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}}
|
|
writer := &recordingProjectorWriter{}
|
|
committer := &recordingProjectorCommitter{errors: []error{errors.New("commit timeout"), nil}}
|
|
processBatchReliably(context.Background(), discardProjectorLogger{}, metrics.NewRegistry(), writer, committer, route, messages, time.Second, time.Millisecond)
|
|
if writer.callCount() != 1 {
|
|
t.Fatalf("writer calls = %d, want 1", writer.callCount())
|
|
}
|
|
if committer.callCount() != 2 {
|
|
t.Fatalf("commit calls = %d, want 2", committer.callCount())
|
|
}
|
|
}
|
|
|
|
func TestProjectBatchWriteFailureDoesNotCommitByItself(t *testing.T) {
|
|
route := jt808ProjectionRoute()
|
|
payload, _ := projectorRawEnvelope().MarshalJSONBytes()
|
|
writer := &recordingProjectorWriter{errors: []error{errors.New("write failed")}}
|
|
count, err := projectBatch(context.Background(), discardProjectorLogger{}, nil, writer, route, []kafka.Message{{Topic: route.RawTopic, Value: payload}})
|
|
if err == nil || count != 1 {
|
|
t.Fatalf("projectBatch() count/error = %d/%v", count, err)
|
|
}
|
|
}
|
|
|
|
func projectorRawEnvelope() envelope.FrameEnvelope {
|
|
return envelope.FrameEnvelope{
|
|
EventID: "raw-event-1",
|
|
EventKind: envelope.EventKindRaw,
|
|
Protocol: envelope.ProtocolJT808,
|
|
MessageID: "0x0200",
|
|
VIN: "VIN001",
|
|
Phone: "13307795425",
|
|
SourceEndpoint: "115.231.168.135:43625",
|
|
SourceCode: "g7s",
|
|
SourceKind: "PLATFORM",
|
|
PlatformName: "G7s",
|
|
EventTimeMS: 1783960000000,
|
|
ReceivedAtMS: 1783960000010,
|
|
ParsedFields: map[string]any{
|
|
"jt808.location.latitude": 30.1,
|
|
"jt808.location.longitude": 121.2,
|
|
"jt808.location.total_mileage_km": 1234.5,
|
|
},
|
|
ParseStatus: envelope.ParseOK,
|
|
}
|
|
}
|
|
|
|
func jt808ProjectionRoute() projectionRoute {
|
|
return projectionRoute{
|
|
Protocol: envelope.ProtocolJT808, RawTopic: topics.RawJT808, FieldsTopic: topics.FieldsJT808, GroupID: "projector-jt808",
|
|
}
|
|
}
|
|
|
|
type recordingProjectorWriter struct {
|
|
mu sync.Mutex
|
|
errors []error
|
|
calls int
|
|
messages []kafka.Message
|
|
}
|
|
|
|
func (w *recordingProjectorWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.calls++
|
|
w.messages = append(w.messages, messages...)
|
|
if len(w.errors) == 0 {
|
|
return nil
|
|
}
|
|
err := w.errors[0]
|
|
w.errors = w.errors[1:]
|
|
return err
|
|
}
|
|
|
|
func (w *recordingProjectorWriter) callCount() int {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
return w.calls
|
|
}
|
|
|
|
type recordingProjectorCommitter struct {
|
|
mu sync.Mutex
|
|
errors []error
|
|
calls int
|
|
messageCount int
|
|
}
|
|
|
|
func (c *recordingProjectorCommitter) CommitMessages(_ context.Context, messages ...kafka.Message) error {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.calls++
|
|
c.messageCount += len(messages)
|
|
if len(c.errors) == 0 {
|
|
return nil
|
|
}
|
|
err := c.errors[0]
|
|
c.errors = c.errors[1:]
|
|
return err
|
|
}
|
|
|
|
func (c *recordingProjectorCommitter) callCount() int {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.calls
|
|
}
|
|
|
|
type discardProjectorLogger struct{}
|
|
|
|
func (discardProjectorLogger) Error(string, ...any) {}
|
|
func (discardProjectorLogger) Warn(string, ...any) {}
|