shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
go/vehicle-gateway/internal/eventbus/nats_sink.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
@@ -13,16 +14,19 @@ import (
)
type NATSConfig struct {
URL string
Name string
RawSubjects map[envelope.Protocol]string
FieldsSubjects map[envelope.Protocol]string
UnifiedSubject string
URL string
Name string
RawSubjects map[envelope.Protocol]string
FieldsSubjects map[envelope.Protocol]string
UnifiedSubject string
AsyncMaxPending int
AsyncAckTimeout time.Duration
}
type NATSSink struct {
conn *nats.Conn
publisher natsPublisher
asyncPublisher natsAsyncPublisher
rawSubjects map[envelope.Protocol]string
fieldsSubjects map[envelope.Protocol]string
unifiedSubject string
@@ -34,10 +38,18 @@ type natsPublisher interface {
Publish(context.Context, string, []byte, ...NATSPublishOption) error
}
type natsAsyncPublisher interface {
PublishAsync(string, []byte, ...NATSPublishOption) (nats.PubAckFuture, error)
}
func NewNATSSink(cfg NATSConfig) (*NATSSink, error) {
if cfg.URL == "" {
return nil, errors.New("nats url is required")
}
if err := ValidateNATSConfig(cfg); err != nil {
return nil, err
}
rawSubjects, fieldsSubjects := natsSubjectMaps(cfg)
name := cfg.Name
if name == "" {
name = "lingniu-vehicle-gateway"
@@ -46,17 +58,58 @@ func NewNATSSink(cfg NATSConfig) (*NATSSink, error) {
if err != nil {
return nil, err
}
js, err := conn.JetStream()
var jsOptions []nats.JSOpt
if cfg.AsyncMaxPending > 0 {
jsOptions = append(jsOptions, nats.PublishAsyncMaxPending(cfg.AsyncMaxPending))
}
if cfg.AsyncAckTimeout > 0 {
jsOptions = append(jsOptions, nats.PublishAsyncTimeout(cfg.AsyncAckTimeout))
}
js, err := conn.JetStream(jsOptions...)
if err != nil {
conn.Close()
return nil, err
}
sink := newNATSSinkWithPublisher(natsJetStreamPublisher{js: js}, cfg)
publisher := natsJetStreamPublisher{js: js}
sink := newNATSSinkWithPublishers(publisher, publisher, NATSConfig{
RawSubjects: rawSubjects,
FieldsSubjects: fieldsSubjects,
UnifiedSubject: cfg.UnifiedSubject,
})
sink.conn = conn
return sink, nil
}
func ValidateNATSConfig(cfg NATSConfig) error {
rawSubjects, fieldsSubjects := natsSubjectMaps(cfg)
raw := protocolTopicLabels(rawSubjects)
fields := protocolTopicLabels(fieldsSubjects)
if err := topics.ValidateKnownRawFieldsProtocols(raw, fields, "nats subject"); err != nil {
return err
}
return topics.ValidateRawFieldsDisjoint(raw, fields, "nats subject")
}
func newNATSSinkWithPublisher(publisher natsPublisher, cfg NATSConfig) *NATSSink {
return newNATSSinkWithPublishers(publisher, nil, cfg)
}
func newNATSSinkWithPublishers(publisher natsPublisher, asyncPublisher natsAsyncPublisher, cfg NATSConfig) *NATSSink {
rawSubjects, fieldsSubjects := natsSubjectMaps(cfg)
unifiedSubject := cfg.UnifiedSubject
if unifiedSubject == "" {
unifiedSubject = topics.Unified
}
return &NATSSink{
publisher: publisher,
asyncPublisher: asyncPublisher,
rawSubjects: rawSubjects,
fieldsSubjects: fieldsSubjects,
unifiedSubject: unifiedSubject,
}
}
func natsSubjectMaps(cfg NATSConfig) (map[envelope.Protocol]string, map[envelope.Protocol]string) {
rawSubjects := map[envelope.Protocol]string{
envelope.ProtocolGB32960: topics.RawGB32960,
envelope.ProtocolJT808: topics.RawJT808,
@@ -77,16 +130,7 @@ func newNATSSinkWithPublisher(publisher natsPublisher, cfg NATSConfig) *NATSSink
fieldsSubjects[protocol] = subject
}
}
unifiedSubject := cfg.UnifiedSubject
if unifiedSubject == "" {
unifiedSubject = topics.Unified
}
return &NATSSink{
publisher: publisher,
rawSubjects: rawSubjects,
fieldsSubjects: fieldsSubjects,
unifiedSubject: unifiedSubject,
}
return rawSubjects, fieldsSubjects
}
func (s *NATSSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -94,14 +138,14 @@ func (s *NATSSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) e
if !ok || subject == "" {
return fmt.Errorf("raw subject not configured for protocol %s", env.Protocol)
}
return s.publish(ctx, subject, env)
return s.publish(ctx, subject, "raw", env)
}
func (s *NATSSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
if s.unifiedSubject == "" {
return errors.New("unified subject is empty")
}
return s.publish(ctx, s.unifiedSubject, env)
return s.publish(ctx, s.unifiedSubject, "unified", env)
}
func (s *NATSSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -109,7 +153,62 @@ func (s *NATSSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope
if !ok || subject == "" {
return fmt.Errorf("fields subject not configured for protocol %s", env.Protocol)
}
return s.publish(ctx, subject, env)
return s.publish(ctx, subject, "fields", env)
}
func (s *NATSSink) PublishRecords(ctx context.Context, records []durableRecord) error {
for _, record := range records {
subject, err := s.subjectForRecord(record)
if err != nil {
return err
}
if err := s.publish(ctx, subject, record.Kind, record.Envelope); err != nil {
return err
}
}
return nil
}
func (s *NATSSink) PublishRecordAsync(record durableRecord, complete func(error)) error {
if s == nil || s.asyncPublisher == nil {
return errors.New("nats async publisher is not configured")
}
if complete == nil {
return errors.New("nats async publish completion callback is required")
}
subject, err := s.subjectForRecord(record)
if err != nil {
return err
}
payload, err := record.Envelope.MarshalJSONBytes()
if err != nil {
return err
}
future, err := s.asyncPublisher.PublishAsync(
subject,
payload,
nats.MsgId(natsMessageID(record.Kind, subject, record.Envelope)),
)
if err != nil {
return err
}
go func() {
select {
case <-future.Ok():
complete(nil)
case asyncErr := <-future.Err():
if asyncErr == nil {
asyncErr = errors.New("nats async publish failed without error detail")
}
complete(asyncErr)
}
}()
return nil
}
func (s *NATSSink) ValidateRecord(record durableRecord) error {
_, err := s.subjectForRecord(record)
return err
}
func (s *NATSSink) Close() error {
@@ -121,12 +220,48 @@ func (s *NATSSink) Close() error {
return nil
}
func (s *NATSSink) publish(ctx context.Context, subject string, env envelope.FrameEnvelope) error {
func (s *NATSSink) publish(ctx context.Context, subject string, kind string, env envelope.FrameEnvelope) error {
payload, err := env.MarshalJSONBytes()
if err != nil {
return err
}
return s.publisher.Publish(ctx, subject, payload, nats.MsgId(env.StableEventID()))
return s.publisher.Publish(ctx, subject, payload, nats.MsgId(natsMessageID(kind, subject, env)))
}
func natsMessageID(kind string, subject string, env envelope.FrameEnvelope) string {
kind = strings.TrimSpace(kind)
if kind == "" {
kind = "unknown"
}
subject = strings.TrimSpace(subject)
if subject == "" {
subject = "unknown"
}
return kind + ":" + subject + ":" + env.StableEventID()
}
func (s *NATSSink) subjectForRecord(record durableRecord) (string, error) {
switch record.Kind {
case "raw":
subject, ok := s.rawSubjects[record.Envelope.Protocol]
if !ok || subject == "" {
return "", fmt.Errorf("raw subject not configured for protocol %s", record.Envelope.Protocol)
}
return subject, nil
case "unified":
if s.unifiedSubject == "" {
return "", errors.New("unified subject is empty")
}
return s.unifiedSubject, nil
case "fields":
subject, ok := s.fieldsSubjects[record.Envelope.Protocol]
if !ok || subject == "" {
return "", fmt.Errorf("fields subject not configured for protocol %s", record.Envelope.Protocol)
}
return subject, nil
default:
return "", errUnknownRecordKind(record.Kind)
}
}
type natsJetStreamPublisher struct {
@@ -137,3 +272,7 @@ func (p natsJetStreamPublisher) Publish(ctx context.Context, subject string, dat
_, err := p.js.Publish(subject, data, append(opts, nats.Context(ctx))...)
return err
}
func (p natsJetStreamPublisher) PublishAsync(subject string, data []byte, opts ...NATSPublishOption) (nats.PubAckFuture, error) {
return p.js.PublishAsync(subject, data, opts...)
}