feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
@@ -3,7 +3,12 @@ package eventbus
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
@@ -57,6 +62,190 @@ func TestNATSSinkDefaultsToGoRawSubjects(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNATSSinkPublishesDurableRecords(t *testing.T) {
|
||||
publisher := &recordingNATSPublisher{}
|
||||
sink := newNATSSinkWithPublisher(publisher, NATSConfig{
|
||||
RawSubjects: map[envelope.Protocol]string{
|
||||
envelope.ProtocolJT808: "vehicle.raw.go.jt808.v1",
|
||||
},
|
||||
FieldsSubjects: map[envelope.Protocol]string{
|
||||
envelope.ProtocolJT808: "vehicle.fields.go.jt808.v1",
|
||||
},
|
||||
UnifiedSubject: "vehicle.event.go.unified.v1",
|
||||
})
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
||||
|
||||
err := sink.PublishRecords(context.Background(), []durableRecord{
|
||||
{Kind: "raw", Envelope: env},
|
||||
{Kind: "fields", Envelope: env},
|
||||
{Kind: "unified", Envelope: env},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PublishRecords() error = %v", err)
|
||||
}
|
||||
if len(publisher.messages) != 3 {
|
||||
t.Fatalf("published messages = %d, want 3", len(publisher.messages))
|
||||
}
|
||||
for i, want := range []string{
|
||||
"vehicle.raw.go.jt808.v1",
|
||||
"vehicle.fields.go.jt808.v1",
|
||||
"vehicle.event.go.unified.v1",
|
||||
} {
|
||||
if got := publisher.messages[i].subject; got != want {
|
||||
t.Fatalf("message %d subject = %q, want %q", i, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNATSMessageIDSeparatesKindAndSubject(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", Sequence: 7}
|
||||
|
||||
raw := natsMessageID("raw", "vehicle.raw.go.jt808.v1", env)
|
||||
rawRetry := natsMessageID("raw", "vehicle.raw.go.jt808.v1", env)
|
||||
fields := natsMessageID("fields", "vehicle.fields.go.jt808.v1", env)
|
||||
unified := natsMessageID("unified", "vehicle.event.go.unified.v1", env)
|
||||
rawOtherSubject := natsMessageID("raw", "vehicle.raw.go.gb32960.v1", env)
|
||||
|
||||
if raw != rawRetry {
|
||||
t.Fatalf("same kind/subject/event id should be stable: %q vs %q", raw, rawRetry)
|
||||
}
|
||||
if raw == fields || raw == unified || raw == rawOtherSubject {
|
||||
t.Fatalf("message ids should be unique per kind and subject: raw=%q fields=%q unified=%q rawOther=%q", raw, fields, unified, rawOtherSubject)
|
||||
}
|
||||
if !strings.Contains(raw, env.StableEventID()) {
|
||||
t.Fatalf("message id %q should retain stable event id %q", raw, env.StableEventID())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNATSSinkPublishRecordsRejectsUnknownKind(t *testing.T) {
|
||||
sink := newNATSSinkWithPublisher(&recordingNATSPublisher{}, NATSConfig{})
|
||||
|
||||
err := sink.PublishRecords(context.Background(), []durableRecord{
|
||||
{Kind: "unknown", Envelope: envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown durable record kind") {
|
||||
t.Fatalf("PublishRecords() error = %v, want unknown kind", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNATSSinkAsyncPublishCompletesOnlyAfterPubAck(t *testing.T) {
|
||||
future := newTestPubAckFuture()
|
||||
asyncPublisher := &recordingNATSAsyncPublisher{future: future}
|
||||
sink := newNATSSinkWithPublishers(&recordingNATSPublisher{}, asyncPublisher, NATSConfig{})
|
||||
record := durableRecord{Kind: "raw", Envelope: normalizeDurableEnvelope(durableTestEnvelope())}
|
||||
completed := make(chan error, 1)
|
||||
|
||||
if err := sink.PublishRecordAsync(record, func(err error) { completed <- err }); err != nil {
|
||||
t.Fatalf("PublishRecordAsync() error = %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-completed:
|
||||
t.Fatalf("completion fired before PubAck: %v", err)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
if got, want := asyncPublisher.subject, "vehicle.raw.go.jt808.v1"; got != want {
|
||||
t.Fatalf("async subject = %q, want %q", got, want)
|
||||
}
|
||||
var decoded envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(asyncPublisher.data, &decoded); err != nil {
|
||||
t.Fatalf("decode async payload: %v", err)
|
||||
}
|
||||
if decoded.EventID != record.Envelope.EventID {
|
||||
t.Fatalf("async event id = %q, want %q", decoded.EventID, record.Envelope.EventID)
|
||||
}
|
||||
|
||||
future.ok <- &nats.PubAck{Stream: "VEHICLE_RAW", Sequence: 1}
|
||||
select {
|
||||
case err := <-completed:
|
||||
if err != nil {
|
||||
t.Fatalf("completion error = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("completion did not fire after PubAck")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNATSSinkAsyncPublishPropagatesFutureError(t *testing.T) {
|
||||
future := newTestPubAckFuture()
|
||||
sink := newNATSSinkWithPublishers(
|
||||
&recordingNATSPublisher{},
|
||||
&recordingNATSAsyncPublisher{future: future},
|
||||
NATSConfig{},
|
||||
)
|
||||
completed := make(chan error, 1)
|
||||
errWant := errors.New("jetstream ack timeout")
|
||||
|
||||
if err := sink.PublishRecordAsync(durableRecord{
|
||||
Kind: "raw",
|
||||
Envelope: normalizeDurableEnvelope(durableTestEnvelope()),
|
||||
}, func(err error) { completed <- err }); err != nil {
|
||||
t.Fatalf("PublishRecordAsync() error = %v", err)
|
||||
}
|
||||
future.err <- errWant
|
||||
select {
|
||||
case err := <-completed:
|
||||
if !errors.Is(err, errWant) {
|
||||
t.Fatalf("completion error = %v, want %v", err, errWant)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("completion did not fire after future error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNATSSinkValidatesDurableRecordBeforeOutboxPersistence(t *testing.T) {
|
||||
sink := newNATSSinkWithPublisher(&recordingNATSPublisher{}, NATSConfig{})
|
||||
err := sink.ValidateRecord(durableRecord{
|
||||
Kind: "raw",
|
||||
Envelope: envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "raw subject not configured") {
|
||||
t.Fatalf("ValidateRecord() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNATSConfigRejectsRawFieldsSubjectOverlap(t *testing.T) {
|
||||
err := ValidateNATSConfig(NATSConfig{
|
||||
RawSubjects: map[envelope.Protocol]string{
|
||||
envelope.ProtocolJT808: "vehicle.same.jt808",
|
||||
},
|
||||
FieldsSubjects: map[envelope.Protocol]string{
|
||||
envelope.ProtocolJT808: "vehicle.same.jt808",
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ValidateNATSConfig() error = nil, want overlap rejection")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "nats subject") {
|
||||
t.Fatalf("error = %q, want nats subject hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNATSConfigRejectsKnownProtocolSubjectMismatch(t *testing.T) {
|
||||
err := ValidateNATSConfig(NATSConfig{
|
||||
RawSubjects: map[envelope.Protocol]string{
|
||||
envelope.ProtocolJT808: "vehicle.raw.go.gb32960.v1",
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ValidateNATSConfig() error = nil, want raw protocol mismatch")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must match protocol") {
|
||||
t.Fatalf("error = %q, want protocol mismatch hint", err)
|
||||
}
|
||||
|
||||
err = ValidateNATSConfig(NATSConfig{
|
||||
FieldsSubjects: map[envelope.Protocol]string{
|
||||
envelope.ProtocolGB32960: "vehicle.fields.go.yutong-mqtt.v1",
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ValidateNATSConfig() error = nil, want fields protocol mismatch")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must match protocol") {
|
||||
t.Fatalf("error = %q, want protocol mismatch hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingNATSPublisher struct {
|
||||
messages []recordedNATSMessage
|
||||
}
|
||||
@@ -70,3 +259,36 @@ func (p *recordingNATSPublisher) Publish(_ context.Context, subject string, data
|
||||
p.messages = append(p.messages, recordedNATSMessage{subject: subject, data: append([]byte(nil), data...)})
|
||||
return nil
|
||||
}
|
||||
|
||||
type recordingNATSAsyncPublisher struct {
|
||||
subject string
|
||||
data []byte
|
||||
future nats.PubAckFuture
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *recordingNATSAsyncPublisher) PublishAsync(subject string, data []byte, _ ...NATSPublishOption) (nats.PubAckFuture, error) {
|
||||
p.subject = subject
|
||||
p.data = append([]byte(nil), data...)
|
||||
return p.future, p.err
|
||||
}
|
||||
|
||||
type testPubAckFuture struct {
|
||||
ok chan *nats.PubAck
|
||||
err chan error
|
||||
msg *nats.Msg
|
||||
}
|
||||
|
||||
func newTestPubAckFuture() *testPubAckFuture {
|
||||
return &testPubAckFuture{
|
||||
ok: make(chan *nats.PubAck, 1),
|
||||
err: make(chan error, 1),
|
||||
msg: &nats.Msg{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *testPubAckFuture) Ok() <-chan *nats.PubAck { return f.ok }
|
||||
|
||||
func (f *testPubAckFuture) Err() <-chan error { return f.err }
|
||||
|
||||
func (f *testPubAckFuture) Msg() *nats.Msg { return f.msg }
|
||||
|
||||
Reference in New Issue
Block a user