shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go): publish realtime fields stream

Browse Source
This commit is contained in:
lingniu
2026-07-03 14:08:32 +08:00
parent e1bf2d72e1
commit 535acdb2a6
23 changed files with 284 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
go/vehicle-gateway/internal/eventbus/async_sink.go
View File

@@ -75,6 +75,10 @@ func (s *AsyncSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelo
return s.enqueue(ctx, asyncJob{kind: "unified", env: env})
}
func (s *AsyncSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
return s.enqueue(ctx, asyncJob{kind: "fields", env: env})
}
func (s *AsyncSink) Close() error {
s.closeOnce.Do(func() {
close(s.closed)
@@ -110,6 +114,8 @@ func (s *AsyncSink) worker() {
err = s.delegate.PublishRaw(ctx, job.env)
case "unified":
err = s.delegate.PublishUnified(ctx, job.env)
case "fields":
err = s.delegate.PublishFields(ctx, job.env)
}
cancel()
if err != nil && s.onError != nil {

9
go/vehicle-gateway/internal/eventbus/async_sink_test.go
View File

@@ -73,6 +73,10 @@ func (s *blockingSink) PublishUnified(context.Context, envelope.FrameEnvelope) e
return nil
}
func (s *blockingSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (s *blockingSink) Close() error {
s.release()
return nil
@@ -100,6 +104,11 @@ func (s *orderedSink) PublishUnified(context.Context, envelope.FrameEnvelope) er
return nil
}
func (s *orderedSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
s.calls = append(s.calls, "fields")
return nil
}
func (s *orderedSink) Close() error {
return nil
}

17
go/vehicle-gateway/internal/eventbus/durable_sink.go
View File

@@ -79,6 +79,16 @@ func (s *DurableSink) PublishUnified(ctx context.Context, env envelope.FrameEnve
return s.spool("unified", env)
}
func (s *DurableSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
if s.isRawPending(env) {
return s.spool("fields", env)
}
if err := s.delegate.PublishFields(ctx, env); err == nil {
return nil
}
return s.spool("fields", env)
}
func (s *DurableSink) ReplayOnce(ctx context.Context) error {
return s.replay(ctx, 0)
}
@@ -147,7 +157,10 @@ func durableKindOrder(kind string) int {
if kind == "unified" {
return 1
}
return 2
if kind == "fields" {
return 2
}
return 3
}
func (s *DurableSink) ReplayLoop(ctx context.Context, interval time.Duration, onError func(error)) {
@@ -184,6 +197,8 @@ func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) e
return s.delegate.PublishRaw(ctx, record.Envelope)
case "unified":
return s.delegate.PublishUnified(ctx, record.Envelope)
case "fields":
return s.delegate.PublishFields(ctx, record.Envelope)
default:
return errUnknownRecordKind(record.Kind)
}

13
go/vehicle-gateway/internal/eventbus/durable_sink_test.go
View File

@@ -300,6 +300,11 @@ func (s *scriptedSink) PublishUnified(context.Context, envelope.FrameEnvelope) e
return nil
}
func (s *scriptedSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
s.calls = append(s.calls, "fields")
return nil
}
func (s *scriptedSink) Close() error {
return nil
}
@@ -323,6 +328,10 @@ func (s *contextCheckingReplaySink) PublishUnified(ctx context.Context, _ envelo
return s.unifiedCtxErr
}
func (s *contextCheckingReplaySink) PublishFields(ctx context.Context, _ envelope.FrameEnvelope) error {
return ctx.Err()
}
func (s *contextCheckingReplaySink) Close() error {
return nil
}
@@ -350,6 +359,10 @@ func (s *batchRecordingSink) PublishUnified(context.Context, envelope.FrameEnvel
return nil
}
func (s *batchRecordingSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (s *batchRecordingSink) Close() error {
return nil
}

29
go/vehicle-gateway/internal/eventbus/kafka_sink.go
View File

@@ -14,12 +14,14 @@ import (
type Sink interface {
PublishRaw(context.Context, envelope.FrameEnvelope) error
PublishUnified(context.Context, envelope.FrameEnvelope) error
PublishFields(context.Context, envelope.FrameEnvelope) error
Close() error
}
type KafkaSink struct {
writer kafkaWriter
rawTopics map[envelope.Protocol]string
fieldsTopics map[envelope.Protocol]string
unifiedTopic string
}
@@ -31,6 +33,7 @@ type kafkaWriter interface {
type KafkaConfig struct {
Brokers []string
RawTopics map[envelope.Protocol]string
FieldsTopics map[envelope.Protocol]string
UnifiedTopic string
}
@@ -58,11 +61,21 @@ func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
rawTopics[protocol] = topic
}
}
fieldsTopics := map[envelope.Protocol]string{
envelope.ProtocolGB32960: topics.FieldsGB32960,
envelope.ProtocolJT808: topics.FieldsJT808,
envelope.ProtocolYutongMQTT: topics.FieldsYutongMQTT,
}
for protocol, topic := range cfg.FieldsTopics {
if topic != "" {
fieldsTopics[protocol] = topic
}
}
unifiedTopic := cfg.UnifiedTopic
if unifiedTopic == "" {
unifiedTopic = topics.Unified
}
return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic}
return &KafkaSink{writer: writer, rawTopics: rawTopics, fieldsTopics: fieldsTopics, unifiedTopic: unifiedTopic}
}
func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -77,6 +90,14 @@ func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelo
return s.publish(ctx, s.unifiedTopic, env)
}
func (s *KafkaSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
topic, ok := s.fieldsTopics[env.Protocol]
if !ok || topic == "" {
return fmt.Errorf("fields topic not configured for protocol %s", env.Protocol)
}
return s.publish(ctx, topic, env)
}
func (s *KafkaSink) PublishRecords(ctx context.Context, records []durableRecord) error {
messages := make([]kafka.Message, 0, len(records))
for _, record := range records {
@@ -121,6 +142,12 @@ func (s *KafkaSink) topicForRecord(record durableRecord) (string, error) {
return topic, nil
case "unified":
return s.unifiedTopic, nil
case "fields":
topic, ok := s.fieldsTopics[record.Envelope.Protocol]
if !ok || topic == "" {
return "", fmt.Errorf("fields topic not configured for protocol %s", record.Envelope.Protocol)
}
return topic, nil
default:
return "", fmt.Errorf("unknown durable record kind %q", record.Kind)
}

5
go/vehicle-gateway/internal/eventbus/log_sink.go
View File

@@ -28,6 +28,11 @@ func (s *LogSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope)
return nil
}
func (s *LogSink) PublishFields(_ context.Context, env envelope.FrameEnvelope) error {
s.logger.Info("fields envelope", "protocol", env.Protocol, "event_id", env.StableEventID(), "vehicle_key", env.VehicleKey(), "message_id", env.MessageID, "field_count", len(env.Fields))
return nil
}
func (s *LogSink) Close() error {
return nil
}

21
go/vehicle-gateway/internal/eventbus/nats_sink.go
View File

@@ -16,6 +16,7 @@ type NATSConfig struct {
URL string
Name string
RawSubjects map[envelope.Protocol]string
FieldsSubjects map[envelope.Protocol]string
UnifiedSubject string
}
@@ -23,6 +24,7 @@ type NATSSink struct {
conn *nats.Conn
publisher natsPublisher
rawSubjects map[envelope.Protocol]string
fieldsSubjects map[envelope.Protocol]string
unifiedSubject string
}
@@ -65,6 +67,16 @@ func newNATSSinkWithPublisher(publisher natsPublisher, cfg NATSConfig) *NATSSink
rawSubjects[protocol] = subject
}
}
fieldsSubjects := map[envelope.Protocol]string{
envelope.ProtocolGB32960: topics.FieldsGB32960,
envelope.ProtocolJT808: topics.FieldsJT808,
envelope.ProtocolYutongMQTT: topics.FieldsYutongMQTT,
}
for protocol, subject := range cfg.FieldsSubjects {
if subject != "" {
fieldsSubjects[protocol] = subject
}
}
unifiedSubject := cfg.UnifiedSubject
if unifiedSubject == "" {
unifiedSubject = topics.Unified
@@ -72,6 +84,7 @@ func newNATSSinkWithPublisher(publisher natsPublisher, cfg NATSConfig) *NATSSink
return &NATSSink{
publisher: publisher,
rawSubjects: rawSubjects,
fieldsSubjects: fieldsSubjects,
unifiedSubject: unifiedSubject,
}
}
@@ -91,6 +104,14 @@ func (s *NATSSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelop
return s.publish(ctx, s.unifiedSubject, env)
}
func (s *NATSSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
subject, ok := s.fieldsSubjects[env.Protocol]
if !ok || subject == "" {
return fmt.Errorf("fields subject not configured for protocol %s", env.Protocol)
}
return s.publish(ctx, subject, env)
}
func (s *NATSSink) Close() error {
if s == nil || s.conn == nil {
return nil

10
go/vehicle-gateway/internal/eventbus/retry_sink.go
View File

@@ -40,6 +40,12 @@ func (s *RetryingSink) PublishUnified(ctx context.Context, env envelope.FrameEnv
})
}
func (s *RetryingSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
return s.withRetry(ctx, func(attemptCtx context.Context) error {
return s.delegate.PublishFields(attemptCtx, env)
})
}
func (s *RetryingSink) PublishRecords(ctx context.Context, records []durableRecord) error {
if publisher, ok := s.delegate.(recordPublishingSink); ok {
return s.withRetry(ctx, func(attemptCtx context.Context) error {
@@ -56,6 +62,10 @@ func (s *RetryingSink) PublishRecords(ctx context.Context, records []durableReco
if err := s.PublishUnified(ctx, record.Envelope); err != nil {
return err
}
case "fields":
if err := s.PublishFields(ctx, record.Envelope); err != nil {
return err
}
default:
return errUnknownRecordKind(record.Kind)
}

4
go/vehicle-gateway/internal/eventbus/retry_sink_test.go
View File

@@ -75,6 +75,10 @@ func (s *flakySink) PublishUnified(context.Context, envelope.FrameEnvelope) erro
return nil
}
func (s *flakySink) PublishFields(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (s *flakySink) Close() error {
return nil
}