feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
314
go/vehicle-gateway/cmd/identity-writer/main_test.go
Normal file
314
go/vehicle-gateway/cmd/identity-writer/main_test.go
Normal file
@@ -0,0 +1,314 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
|
||||
)
|
||||
|
||||
type fakeRegistrationProjector struct {
|
||||
facts []identity.JT808RegistrationFact
|
||||
envelopes []envelope.FrameEnvelope
|
||||
markedFacts []identity.JT808RegistrationFact
|
||||
}
|
||||
|
||||
func (p *fakeRegistrationProjector) ProjectBatch(envs []envelope.FrameEnvelope) []identity.JT808RegistrationFact {
|
||||
p.envelopes = append(p.envelopes, envs...)
|
||||
return p.facts
|
||||
}
|
||||
|
||||
func (p *fakeRegistrationProjector) MarkPersisted(facts []identity.JT808RegistrationFact) {
|
||||
p.markedFacts = append(p.markedFacts, facts...)
|
||||
}
|
||||
|
||||
type fakeRegistrationStore struct {
|
||||
facts []identity.JT808RegistrationFact
|
||||
err error
|
||||
count int
|
||||
failOnCount int
|
||||
}
|
||||
|
||||
func (s *fakeRegistrationStore) UpsertBatch(_ context.Context, facts []identity.JT808RegistrationFact) error {
|
||||
s.count++
|
||||
s.facts = append(s.facts, facts...)
|
||||
if s.err != nil && (s.failOnCount == 0 || s.count == s.failOnCount) {
|
||||
return s.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeCommitter struct {
|
||||
messages []kafka.Message
|
||||
err error
|
||||
count int
|
||||
failOnCount int
|
||||
}
|
||||
|
||||
func (c *fakeCommitter) CommitMessages(_ context.Context, messages ...kafka.Message) error {
|
||||
c.count++
|
||||
c.messages = append(c.messages, messages...)
|
||||
if c.err != nil && (c.failOnCount == 0 || c.count == c.failOnCount) {
|
||||
return c.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestProcessIdentityBatchDoesNotCommitOrThrottleOnStoreFailure(t *testing.T) {
|
||||
wantErr := errors.New("mysql unavailable")
|
||||
projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{{
|
||||
Phone: "13307795425",
|
||||
SeenAt: time.Now(),
|
||||
}}}
|
||||
store := &fakeRegistrationStore{err: wantErr}
|
||||
committer := &fakeCommitter{}
|
||||
|
||||
err := processIdentityBatch(
|
||||
context.Background(),
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
metrics.NewRegistry(),
|
||||
projector,
|
||||
store,
|
||||
committer,
|
||||
[]kafka.Message{validIdentityMessage(t, 1)},
|
||||
)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("processIdentityBatch() error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if len(committer.messages) != 0 {
|
||||
t.Fatalf("committed messages = %d, want 0", len(committer.messages))
|
||||
}
|
||||
if len(projector.markedFacts) != 0 {
|
||||
t.Fatalf("marked facts = %d, want 0", len(projector.markedFacts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessIdentityBatchMarksOnlyAfterStoreAndCommits(t *testing.T) {
|
||||
fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()}
|
||||
projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}}
|
||||
store := &fakeRegistrationStore{}
|
||||
committer := &fakeCommitter{}
|
||||
messages := []kafka.Message{validIdentityMessage(t, 1), validIdentityMessage(t, 2)}
|
||||
|
||||
err := processIdentityBatch(
|
||||
context.Background(),
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
metrics.NewRegistry(),
|
||||
projector,
|
||||
store,
|
||||
committer,
|
||||
messages,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("processIdentityBatch() error = %v", err)
|
||||
}
|
||||
if len(store.facts) != 1 || len(projector.markedFacts) != 1 {
|
||||
t.Fatalf("store facts = %d marked facts = %d, want 1/1", len(store.facts), len(projector.markedFacts))
|
||||
}
|
||||
if len(committer.messages) != len(messages) {
|
||||
t.Fatalf("committed messages = %d, want %d", len(committer.messages), len(messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessIdentityBatchCommitsPoisonEnvelopeWithoutProjection(t *testing.T) {
|
||||
projector := &fakeRegistrationProjector{}
|
||||
store := &fakeRegistrationStore{}
|
||||
committer := &fakeCommitter{}
|
||||
message := kafka.Message{Topic: topics.RawJT808, Partition: 1, Offset: 7, Value: []byte("not-json")}
|
||||
|
||||
err := processIdentityBatch(
|
||||
context.Background(),
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
metrics.NewRegistry(),
|
||||
projector,
|
||||
store,
|
||||
committer,
|
||||
[]kafka.Message{message},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("processIdentityBatch() error = %v", err)
|
||||
}
|
||||
if len(store.facts) != 0 || len(projector.envelopes) != 0 {
|
||||
t.Fatalf("poison message reached projector/store: envs=%d facts=%d", len(projector.envelopes), len(store.facts))
|
||||
}
|
||||
if len(committer.messages) != 1 {
|
||||
t.Fatalf("committed messages = %d, want 1", len(committer.messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessIdentityBatchReliablyRetriesCommitWithoutRewritingMySQL(t *testing.T) {
|
||||
fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()}
|
||||
projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}}
|
||||
store := &fakeRegistrationStore{}
|
||||
committer := &fakeCommitter{err: errors.New("commit failed"), failOnCount: 1}
|
||||
registry := metrics.NewRegistry()
|
||||
messages := []kafka.Message{validIdentityMessage(t, 1), validIdentityMessage(t, 2)}
|
||||
|
||||
ok := processIdentityBatchReliably(
|
||||
context.Background(),
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
registry,
|
||||
projector,
|
||||
store,
|
||||
committer,
|
||||
messages,
|
||||
time.Nanosecond,
|
||||
)
|
||||
|
||||
if !ok {
|
||||
t.Fatal("reliable identity batch returned false")
|
||||
}
|
||||
if store.count != 1 {
|
||||
t.Fatalf("mysql writes = %d, want 1 after commit-only retry", store.count)
|
||||
}
|
||||
if committer.count != 2 {
|
||||
t.Fatalf("commit attempts = %d, want initial failure and one retry", committer.count)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_identity_writer_batch_retries_total{reason="commit_error"} 1`,
|
||||
`vehicle_identity_writer_retry_pending_messages 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("missing metric %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessIdentityBatchReliablyRetriesStoreBeforeCommit(t *testing.T) {
|
||||
fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()}
|
||||
projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}}
|
||||
store := &fakeRegistrationStore{err: errors.New("mysql unavailable"), failOnCount: 1}
|
||||
committer := &fakeCommitter{}
|
||||
registry := metrics.NewRegistry()
|
||||
messages := []kafka.Message{validIdentityMessage(t, 1)}
|
||||
|
||||
ok := processIdentityBatchReliably(
|
||||
context.Background(),
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
registry,
|
||||
projector,
|
||||
store,
|
||||
committer,
|
||||
messages,
|
||||
time.Nanosecond,
|
||||
)
|
||||
|
||||
if !ok {
|
||||
t.Fatal("reliable identity batch returned false")
|
||||
}
|
||||
if store.count != 2 || committer.count != 1 {
|
||||
t.Fatalf("mysql writes=%d commits=%d, want 2/1", store.count, committer.count)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_identity_writer_batch_retries_total{reason="write_error"} 1`,
|
||||
`vehicle_identity_writer_kafka_messages_total{status="received",topic="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("missing metric %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigRejectsNonJT808Topic(t *testing.T) {
|
||||
cfg := config{
|
||||
KafkaBrokers: []string{"127.0.0.1:9092"},
|
||||
KafkaTopic: topics.RawGB32960,
|
||||
MySQLDSN: "user:pass@tcp(localhost:3306)/db",
|
||||
BatchSize: 100,
|
||||
Workers: 3,
|
||||
}
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("Validate() error = nil, want non-JT808 topic error")
|
||||
}
|
||||
cfg.KafkaTopic = topics.RawJT808
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefaultsAndOverridesWorkers(t *testing.T) {
|
||||
t.Setenv("IDENTITY_WRITER_WORKERS", "")
|
||||
if got := loadConfig().Workers; got != 3 {
|
||||
t.Fatalf("default workers = %d, want 3", got)
|
||||
}
|
||||
t.Setenv("IDENTITY_WRITER_WORKERS", "5")
|
||||
if got := loadConfig().Workers; got != 5 {
|
||||
t.Fatalf("configured workers = %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigRejectsNonPositiveWorkers(t *testing.T) {
|
||||
cfg := config{
|
||||
KafkaBrokers: []string{"127.0.0.1:9092"},
|
||||
KafkaTopic: topics.RawJT808,
|
||||
MySQLDSN: "user:pass@tcp(localhost:3306)/db",
|
||||
BatchSize: 100,
|
||||
}
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDENTITY_WRITER_WORKERS") {
|
||||
t.Fatalf("Validate() error = %v, want workers error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessIdentityBatchForWorkerLabelsPendingMetrics(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
workerLabels := metrics.Labels{"worker": "2"}
|
||||
err := processIdentityBatchAttemptForWorker(
|
||||
context.Background(),
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
registry,
|
||||
&fakeRegistrationProjector{},
|
||||
&fakeRegistrationStore{},
|
||||
&fakeCommitter{},
|
||||
[]kafka.Message{validIdentityMessage(t, 1)},
|
||||
true,
|
||||
workerLabels,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("processIdentityBatchAttemptForWorker() error = %v", err)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_identity_writer_batch_pending_messages{worker="2"} 0`,
|
||||
`vehicle_identity_writer_batch_pending_facts{worker="2"} 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("missing metric %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validIdentityMessage(t *testing.T, offset int64) kafka.Message {
|
||||
t.Helper()
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: identity.JT808LocationMessageID,
|
||||
EventKind: envelope.EventKindRaw,
|
||||
Phone: "13307795425",
|
||||
VIN: "LTESTVIN000000001",
|
||||
ReceivedAtMS: time.Now().UnixMilli(),
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kafka.Message{
|
||||
Topic: topics.RawJT808,
|
||||
Partition: 1,
|
||||
Offset: offset,
|
||||
HighWaterMark: offset + 1,
|
||||
Value: payload,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user