shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go): publish realtime fields stream

Browse Source
This commit is contained in:
lingniu
2026-07-03 14:08:32 +08:00
parent e1bf2d72e1
commit 535acdb2a6
23 changed files with 284 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
go/vehicle-gateway/cmd/gateway/main.go
View File

@@ -186,7 +186,7 @@ func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error)
})
logger.Info("nats async publish enabled", "queue_size", queueSize, "workers", workers, "publish_timeout_ms", timeout.Milliseconds())
}
logger.Info("nats jetstream sink enabled", "url", env("NATS_URL", ""), "unified_subject", natsSinkConfigFromEnv().UnifiedSubject, "publish_unified_enabled", envBool("PUBLISH_UNIFIED_ENABLED", false))
logger.Info("nats jetstream sink enabled", "url", env("NATS_URL", ""), "unified_subject", natsSinkConfigFromEnv().UnifiedSubject, "publish_unified_enabled", envBool("PUBLISH_UNIFIED_ENABLED", false), "publish_fields_enabled", true)
return out, nil
}
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
@@ -201,6 +201,11 @@ func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error)
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808),
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT),
},
FieldsTopics: map[envelope.Protocol]string{
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960),
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808),
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT),
},
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", topics.Unified),
})
if err != nil {
@@ -248,6 +253,11 @@ func natsSinkConfigFromEnv() eventbus.NATSConfig {
envelope.ProtocolJT808: env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)),
envelope.ProtocolYutongMQTT: env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)),
},
FieldsSubjects: map[envelope.Protocol]string{
envelope.ProtocolGB32960: env("NATS_SUBJECT_GB32960_FIELDS", env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)),
envelope.ProtocolJT808: env("NATS_SUBJECT_JT808_FIELDS", env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)),
envelope.ProtocolYutongMQTT: env("NATS_SUBJECT_YUTONG_MQTT_FIELDS", env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)),
},
UnifiedSubject: env("NATS_SUBJECT_UNIFIED", env("KAFKA_TOPIC_UNIFIED", topics.Unified)),
}
}

3
go/vehicle-gateway/cmd/nats-fast-writer/main.go
View File

@@ -153,6 +153,9 @@ func loadConfig() config {
env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)),
env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)),
env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)),
env("NATS_SUBJECT_GB32960_FIELDS", env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)),
env("NATS_SUBJECT_JT808_FIELDS", env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)),
env("NATS_SUBJECT_YUTONG_MQTT_FIELDS", env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)),
}, ",")))
return config{
NATSURL: env("NATS_URL", "nats://127.0.0.1:4222"),

9
go/vehicle-gateway/cmd/nats-kafka-bridge/main.go
View File

@@ -102,9 +102,12 @@ type config struct {
func loadConfig() config {
route := map[string]string{
env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)): env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960),
env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)): env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808),
env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT),
env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)): env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960),
env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)): env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808),
env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT),
env("NATS_SUBJECT_GB32960_FIELDS", env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)): env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960),
env("NATS_SUBJECT_JT808_FIELDS", env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)): env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808),
env("NATS_SUBJECT_YUTONG_MQTT_FIELDS", env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT),
}
if unifiedSubject, unifiedTopic, ok := unifiedRouteFromEnv(); ok {
route[unifiedSubject] = unifiedTopic

9
go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go
View File

@@ -117,9 +117,12 @@ func TestLoadConfigDefaultsToGoSubjectRoutes(t *testing.T) {
cfg := loadConfig()
want := map[string]string{
"vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1",
"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1",
"vehicle.raw.go.yutong-mqtt.v1": "vehicle.raw.go.yutong-mqtt.v1",
"vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1",
"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1",
"vehicle.raw.go.yutong-mqtt.v1": "vehicle.raw.go.yutong-mqtt.v1",
"vehicle.fields.go.gb32960.v1": "vehicle.fields.go.gb32960.v1",
"vehicle.fields.go.jt808.v1": "vehicle.fields.go.jt808.v1",
"vehicle.fields.go.yutong-mqtt.v1": "vehicle.fields.go.yutong-mqtt.v1",
}
if len(cfg.Route) != len(want) {
t.Fatalf("route len = %d, want %d: %#v", len(cfg.Route), len(want), cfg.Route)

2
go/vehicle-gateway/cmd/stat-writer/main.go
View File

@@ -143,7 +143,7 @@ func loadConfig() config {
}
return config{
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ","))),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", strings.Join([]string{topics.FieldsGB32960, topics.FieldsJT808, topics.FieldsYutongMQTT}, ","))),
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
MySQLDSN: env("MYSQL_DSN", ""),
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",

2
go/vehicle-gateway/cmd/stat-writer/main_test.go
View File

@@ -78,7 +78,7 @@ func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) {
cfg := loadConfig()
got := strings.Join(cfg.KafkaTopics, ",")
want := "vehicle.raw.go.gb32960.v1,vehicle.raw.go.jt808.v1,vehicle.raw.go.yutong-mqtt.v1"
want := "vehicle.fields.go.gb32960.v1,vehicle.fields.go.jt808.v1,vehicle.fields.go.yutong-mqtt.v1"
if got != want {
t.Fatalf("KafkaTopics = %q, want %q", got, want)
}