package main import ( "context" "encoding/json" "errors" "strings" "sync" "testing" "time" "github.com/segmentio/kafka-go" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" ) func TestLoadConfigCreatesProtocolIsolatedConsumerGroups(t *testing.T) { t.Setenv("FIELDS_PROJECTOR_GROUP_PREFIX", "projector-test") t.Setenv("KAFKA_START_OFFSET", "first") cfg := loadConfig() if cfg.StartOffset != kafka.FirstOffset || cfg.StartOffsetName != "first" { t.Fatalf("start offset = %d/%q", cfg.StartOffset, cfg.StartOffsetName) } wantGroups := map[envelope.Protocol]string{ envelope.ProtocolGB32960: "projector-test-gb32960", envelope.ProtocolJT808: "projector-test-jt808", envelope.ProtocolYutongMQTT: "projector-test-yutong-mqtt", } for _, route := range cfg.Routes { if route.GroupID != wantGroups[route.Protocol] { t.Fatalf("group for %s = %q", route.Protocol, route.GroupID) } } if err := cfg.Validate(); err != nil { t.Fatalf("Validate() error = %v", err) } } func TestProjectBatchUsesPrecomputedFieldsAndPreservesSourceMetadata(t *testing.T) { raw := projectorRawEnvelope() payload, err := raw.MarshalJSONBytes() if err != nil { t.Fatalf("marshal raw: %v", err) } writer := &recordingProjectorWriter{} registry := metrics.NewRegistry() route := jt808ProjectionRoute() count, err := projectBatch(context.Background(), discardProjectorLogger{}, registry, writer, route, []kafka.Message{{ Topic: route.RawTopic, Key: raw.KafkaKey(), Value: payload, Partition: 2, Offset: 10, HighWaterMark: 11, }}) if err != nil { t.Fatalf("projectBatch() error = %v", err) } if count != 1 || writer.callCount() != 1 || len(writer.messages) != 1 { t.Fatalf("projected=%d calls=%d messages=%d", count, writer.callCount(), len(writer.messages)) } var fields envelope.FrameEnvelope if err := json.Unmarshal(writer.messages[0].Value, &fields); err != nil { t.Fatalf("decode fields: %v", err) } if fields.EventKind != envelope.EventKindFields || fields.SourceEventID != raw.EventID || fields.EventID != raw.EventID+":fields" { t.Fatalf("fields identity = %#v", fields) } if fields.SourceCode != raw.SourceCode || fields.SourceKind != raw.SourceKind || fields.SourceEndpoint != raw.SourceEndpoint { t.Fatalf("source metadata not preserved: %#v", fields) } if got := fields.Fields["jt808.location.total_mileage_km"]; got != 1234.5 { t.Fatalf("total mileage = %#v", got) } if len(fields.Parsed) != 0 || len(fields.ParsedFields) != 0 { t.Fatalf("fields projection must not duplicate raw payload: parsed=%v parsed_fields=%v", fields.Parsed, fields.ParsedFields) } text := registry.Render() for _, want := range []string{ `vehicle_fields_projector_projections_total{protocol="JT808",status="projected"} 1`, `vehicle_fields_projector_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, `vehicle_fields_projector_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 0`, } { if !strings.Contains(text, want) { t.Fatalf("metric missing %q:\n%s", want, text) } } } func TestProcessBatchReliablySkipsNonRealtimeAndInvalidWithoutWriting(t *testing.T) { route := jt808ProjectionRoute() nonRealtime := projectorRawEnvelope() nonRealtime.MessageID = "0x0002" nonRealtime.ParsedFields = map[string]any{"jt808.header.message_id": "0x0002"} payload, _ := nonRealtime.MarshalJSONBytes() messages := []kafka.Message{ {Topic: route.RawTopic, Value: []byte("{bad"), Partition: 0, Offset: 1, HighWaterMark: 3}, {Topic: route.RawTopic, Value: payload, Partition: 0, Offset: 2, HighWaterMark: 3}, } writer := &recordingProjectorWriter{} committer := &recordingProjectorCommitter{} registry := metrics.NewRegistry() processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond) if writer.callCount() != 0 { t.Fatalf("writer calls = %d, want 0", writer.callCount()) } if committer.callCount() != 1 || committer.messageCount != 2 { t.Fatalf("commit calls/messages = %d/%d", committer.callCount(), committer.messageCount) } text := registry.Render() if !strings.Contains(text, `vehicle_fields_projector_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`) || !strings.Contains(text, `vehicle_fields_projector_projections_total{protocol="JT808",status="skipped_non_realtime"} 1`) { t.Fatalf("skip metrics missing:\n%s", text) } } func TestProcessBatchReliablyRetriesWriteBeforeCommitting(t *testing.T) { route := jt808ProjectionRoute() payload, _ := projectorRawEnvelope().MarshalJSONBytes() messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}} writer := &recordingProjectorWriter{errors: []error{errors.New("kafka unavailable"), nil}} committer := &recordingProjectorCommitter{} registry := metrics.NewRegistry() processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond) if writer.callCount() != 2 { t.Fatalf("writer calls = %d, want 2", writer.callCount()) } if committer.callCount() != 1 { t.Fatalf("commit calls = %d, want 1 after successful write", committer.callCount()) } if !strings.Contains(registry.Render(), `vehicle_fields_projector_batch_retries_total{protocol="JT808",reason="write_error"} 1`) { t.Fatalf("write retry metric missing:\n%s", registry.Render()) } } func TestProcessBatchReliablyRetriesOnlyCommitAfterSuccessfulWrite(t *testing.T) { route := jt808ProjectionRoute() payload, _ := projectorRawEnvelope().MarshalJSONBytes() messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}} writer := &recordingProjectorWriter{} committer := &recordingProjectorCommitter{errors: []error{errors.New("commit timeout"), nil}} processBatchReliably(context.Background(), discardProjectorLogger{}, metrics.NewRegistry(), writer, committer, route, messages, time.Second, time.Millisecond) if writer.callCount() != 1 { t.Fatalf("writer calls = %d, want 1", writer.callCount()) } if committer.callCount() != 2 { t.Fatalf("commit calls = %d, want 2", committer.callCount()) } } func TestProjectBatchWriteFailureDoesNotCommitByItself(t *testing.T) { route := jt808ProjectionRoute() payload, _ := projectorRawEnvelope().MarshalJSONBytes() writer := &recordingProjectorWriter{errors: []error{errors.New("write failed")}} count, err := projectBatch(context.Background(), discardProjectorLogger{}, nil, writer, route, []kafka.Message{{Topic: route.RawTopic, Value: payload}}) if err == nil || count != 1 { t.Fatalf("projectBatch() count/error = %d/%v", count, err) } } func projectorRawEnvelope() envelope.FrameEnvelope { return envelope.FrameEnvelope{ EventID: "raw-event-1", EventKind: envelope.EventKindRaw, Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: "VIN001", Phone: "13307795425", SourceEndpoint: "115.231.168.135:43625", SourceCode: "g7s", SourceKind: "PLATFORM", PlatformName: "G7s", EventTimeMS: 1783960000000, ReceivedAtMS: 1783960000010, ParsedFields: map[string]any{ "jt808.location.latitude": 30.1, "jt808.location.longitude": 121.2, "jt808.location.total_mileage_km": 1234.5, }, ParseStatus: envelope.ParseOK, } } func jt808ProjectionRoute() projectionRoute { return projectionRoute{ Protocol: envelope.ProtocolJT808, RawTopic: topics.RawJT808, FieldsTopic: topics.FieldsJT808, GroupID: "projector-jt808", } } type recordingProjectorWriter struct { mu sync.Mutex errors []error calls int messages []kafka.Message } func (w *recordingProjectorWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error { w.mu.Lock() defer w.mu.Unlock() w.calls++ w.messages = append(w.messages, messages...) if len(w.errors) == 0 { return nil } err := w.errors[0] w.errors = w.errors[1:] return err } func (w *recordingProjectorWriter) callCount() int { w.mu.Lock() defer w.mu.Unlock() return w.calls } type recordingProjectorCommitter struct { mu sync.Mutex errors []error calls int messageCount int } func (c *recordingProjectorCommitter) CommitMessages(_ context.Context, messages ...kafka.Message) error { c.mu.Lock() defer c.mu.Unlock() c.calls++ c.messageCount += len(messages) if len(c.errors) == 0 { return nil } err := c.errors[0] c.errors = c.errors[1:] return err } func (c *recordingProjectorCommitter) callCount() int { c.mu.Lock() defer c.mu.Unlock() return c.calls } type discardProjectorLogger struct{} func (discardProjectorLogger) Error(string, ...any) {} func (discardProjectorLogger) Warn(string, ...any) {}