package main import ( "context" "errors" "io" "log/slog" "strings" "testing" "time" "github.com/segmentio/kafka-go" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" ) type fakeRegistrationProjector struct { facts []identity.JT808RegistrationFact envelopes []envelope.FrameEnvelope markedFacts []identity.JT808RegistrationFact } func (p *fakeRegistrationProjector) ProjectBatch(envs []envelope.FrameEnvelope) []identity.JT808RegistrationFact { p.envelopes = append(p.envelopes, envs...) return p.facts } func (p *fakeRegistrationProjector) MarkPersisted(facts []identity.JT808RegistrationFact) { p.markedFacts = append(p.markedFacts, facts...) } type fakeRegistrationStore struct { facts []identity.JT808RegistrationFact err error count int failOnCount int } func (s *fakeRegistrationStore) UpsertBatch(_ context.Context, facts []identity.JT808RegistrationFact) error { s.count++ s.facts = append(s.facts, facts...) if s.err != nil && (s.failOnCount == 0 || s.count == s.failOnCount) { return s.err } return nil } type fakeCommitter struct { messages []kafka.Message err error count int failOnCount int } func (c *fakeCommitter) CommitMessages(_ context.Context, messages ...kafka.Message) error { c.count++ c.messages = append(c.messages, messages...) if c.err != nil && (c.failOnCount == 0 || c.count == c.failOnCount) { return c.err } return nil } func TestProcessIdentityBatchDoesNotCommitOrThrottleOnStoreFailure(t *testing.T) { wantErr := errors.New("mysql unavailable") projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{{ Phone: "13307795425", SeenAt: time.Now(), }}} store := &fakeRegistrationStore{err: wantErr} committer := &fakeCommitter{} err := processIdentityBatch( context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), metrics.NewRegistry(), projector, store, committer, []kafka.Message{validIdentityMessage(t, 1)}, ) if !errors.Is(err, wantErr) { t.Fatalf("processIdentityBatch() error = %v, want %v", err, wantErr) } if len(committer.messages) != 0 { t.Fatalf("committed messages = %d, want 0", len(committer.messages)) } if len(projector.markedFacts) != 0 { t.Fatalf("marked facts = %d, want 0", len(projector.markedFacts)) } } func TestProcessIdentityBatchMarksOnlyAfterStoreAndCommits(t *testing.T) { fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()} projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}} store := &fakeRegistrationStore{} committer := &fakeCommitter{} messages := []kafka.Message{validIdentityMessage(t, 1), validIdentityMessage(t, 2)} err := processIdentityBatch( context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), metrics.NewRegistry(), projector, store, committer, messages, ) if err != nil { t.Fatalf("processIdentityBatch() error = %v", err) } if len(store.facts) != 1 || len(projector.markedFacts) != 1 { t.Fatalf("store facts = %d marked facts = %d, want 1/1", len(store.facts), len(projector.markedFacts)) } if len(committer.messages) != len(messages) { t.Fatalf("committed messages = %d, want %d", len(committer.messages), len(messages)) } } func TestProcessIdentityBatchCommitsPoisonEnvelopeWithoutProjection(t *testing.T) { projector := &fakeRegistrationProjector{} store := &fakeRegistrationStore{} committer := &fakeCommitter{} message := kafka.Message{Topic: topics.RawJT808, Partition: 1, Offset: 7, Value: []byte("not-json")} err := processIdentityBatch( context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), metrics.NewRegistry(), projector, store, committer, []kafka.Message{message}, ) if err != nil { t.Fatalf("processIdentityBatch() error = %v", err) } if len(store.facts) != 0 || len(projector.envelopes) != 0 { t.Fatalf("poison message reached projector/store: envs=%d facts=%d", len(projector.envelopes), len(store.facts)) } if len(committer.messages) != 1 { t.Fatalf("committed messages = %d, want 1", len(committer.messages)) } } func TestProcessIdentityBatchReliablyRetriesCommitWithoutRewritingMySQL(t *testing.T) { fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()} projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}} store := &fakeRegistrationStore{} committer := &fakeCommitter{err: errors.New("commit failed"), failOnCount: 1} registry := metrics.NewRegistry() messages := []kafka.Message{validIdentityMessage(t, 1), validIdentityMessage(t, 2)} ok := processIdentityBatchReliably( context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), registry, projector, store, committer, messages, time.Nanosecond, ) if !ok { t.Fatal("reliable identity batch returned false") } if store.count != 1 { t.Fatalf("mysql writes = %d, want 1 after commit-only retry", store.count) } if committer.count != 2 { t.Fatalf("commit attempts = %d, want initial failure and one retry", committer.count) } text := registry.Render() for _, want := range []string{ `vehicle_identity_writer_batch_retries_total{reason="commit_error"} 1`, `vehicle_identity_writer_retry_pending_messages 0`, } { if !strings.Contains(text, want) { t.Fatalf("missing metric %s:\n%s", want, text) } } } func TestProcessIdentityBatchReliablyRetriesStoreBeforeCommit(t *testing.T) { fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()} projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}} store := &fakeRegistrationStore{err: errors.New("mysql unavailable"), failOnCount: 1} committer := &fakeCommitter{} registry := metrics.NewRegistry() messages := []kafka.Message{validIdentityMessage(t, 1)} ok := processIdentityBatchReliably( context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), registry, projector, store, committer, messages, time.Nanosecond, ) if !ok { t.Fatal("reliable identity batch returned false") } if store.count != 2 || committer.count != 1 { t.Fatalf("mysql writes=%d commits=%d, want 2/1", store.count, committer.count) } text := registry.Render() for _, want := range []string{ `vehicle_identity_writer_batch_retries_total{reason="write_error"} 1`, `vehicle_identity_writer_kafka_messages_total{status="received",topic="vehicle.raw.go.jt808.v1"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("missing metric %s:\n%s", want, text) } } } func TestConfigRejectsNonJT808Topic(t *testing.T) { cfg := config{ KafkaBrokers: []string{"127.0.0.1:9092"}, KafkaTopic: topics.RawGB32960, MySQLDSN: "user:pass@tcp(localhost:3306)/db", BatchSize: 100, Workers: 3, } if err := cfg.Validate(); err == nil { t.Fatal("Validate() error = nil, want non-JT808 topic error") } cfg.KafkaTopic = topics.RawJT808 if err := cfg.Validate(); err != nil { t.Fatalf("Validate() error = %v", err) } } func TestLoadConfigDefaultsAndOverridesWorkers(t *testing.T) { t.Setenv("IDENTITY_WRITER_WORKERS", "") if got := loadConfig().Workers; got != 3 { t.Fatalf("default workers = %d, want 3", got) } t.Setenv("IDENTITY_WRITER_WORKERS", "5") if got := loadConfig().Workers; got != 5 { t.Fatalf("configured workers = %d, want 5", got) } } func TestConfigRejectsNonPositiveWorkers(t *testing.T) { cfg := config{ KafkaBrokers: []string{"127.0.0.1:9092"}, KafkaTopic: topics.RawJT808, MySQLDSN: "user:pass@tcp(localhost:3306)/db", BatchSize: 100, } if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDENTITY_WRITER_WORKERS") { t.Fatalf("Validate() error = %v, want workers error", err) } } func TestProcessIdentityBatchForWorkerLabelsPendingMetrics(t *testing.T) { registry := metrics.NewRegistry() workerLabels := metrics.Labels{"worker": "2"} err := processIdentityBatchAttemptForWorker( context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), registry, &fakeRegistrationProjector{}, &fakeRegistrationStore{}, &fakeCommitter{}, []kafka.Message{validIdentityMessage(t, 1)}, true, workerLabels, ) if err != nil { t.Fatalf("processIdentityBatchAttemptForWorker() error = %v", err) } text := registry.Render() for _, want := range []string{ `vehicle_identity_writer_batch_pending_messages{worker="2"} 0`, `vehicle_identity_writer_batch_pending_facts{worker="2"} 0`, } { if !strings.Contains(text, want) { t.Fatalf("missing metric %s:\n%s", want, text) } } } func validIdentityMessage(t *testing.T, offset int64) kafka.Message { t.Helper() env := envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, MessageID: identity.JT808LocationMessageID, EventKind: envelope.EventKindRaw, Phone: "13307795425", VIN: "LTESTVIN000000001", ReceivedAtMS: time.Now().UnixMilli(), ParseStatus: envelope.ParseOK, } payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } return kafka.Message{ Topic: topics.RawJT808, Partition: 1, Offset: offset, HighWaterMark: offset + 1, Value: payload, } }