shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(go): default to go vehicle topics

Browse Source
This commit is contained in:
lingniu
2026-07-02 20:22:47 +08:00
parent 3248336bea
commit 2ae9794dda
15 changed files with 130 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
go/vehicle-gateway/cmd/gateway/main.go
View File

@@ -22,6 +22,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
)
func main() {
@@ -192,11 +193,11 @@ func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error)
sink, err := eventbus.NewKafkaSink(eventbus.KafkaConfig{
Brokers: brokers,
RawTopics: map[envelope.Protocol]string{
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1"),
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1"),
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1"),
envelope.ProtocolGB32960: env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960),
envelope.ProtocolJT808: env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808),
envelope.ProtocolYutongMQTT: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT),
},
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1"),
UnifiedTopic: env("KAFKA_TOPIC_UNIFIED", topics.Unified),
})
if err != nil {
return nil, err
@@ -239,11 +240,11 @@ func natsSinkConfigFromEnv() eventbus.NATSConfig {
URL: env("NATS_URL", ""),
Name: env("NATS_CLIENT_NAME", "lingniu-vehicle-gateway"),
RawSubjects: map[envelope.Protocol]string{
envelope.ProtocolGB32960: env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1")),
envelope.ProtocolJT808: env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1")),
envelope.ProtocolYutongMQTT: env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1")),
envelope.ProtocolGB32960: env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)),
envelope.ProtocolJT808: env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)),
envelope.ProtocolYutongMQTT: env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)),
},
UnifiedSubject: env("NATS_SUBJECT_UNIFIED", env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1")),
UnifiedSubject: env("NATS_SUBJECT_UNIFIED", env("KAFKA_TOPIC_UNIFIED", topics.Unified)),
}
}

19
go/vehicle-gateway/cmd/gateway/main_test.go
View File

@@ -31,3 +31,22 @@ func TestNATSSinkConfigFromEnvUsesExplicitSubjects(t *testing.T) {
t.Fatalf("unified subject = %q, want %q", got, want)
}
}
func TestNATSSinkConfigFromEnvDefaultsToGoSubjects(t *testing.T) {
t.Setenv("NATS_URL", "nats://172.17.111.56:4222")
cfg := natsSinkConfigFromEnv()
if got, want := cfg.RawSubjects[envelope.ProtocolGB32960], "vehicle.raw.go.gb32960.v1"; got != want {
t.Fatalf("gb32960 subject = %q, want %q", got, want)
}
if got, want := cfg.RawSubjects[envelope.ProtocolJT808], "vehicle.raw.go.jt808.v1"; got != want {
t.Fatalf("jt808 subject = %q, want %q", got, want)
}
if got, want := cfg.RawSubjects[envelope.ProtocolYutongMQTT], "vehicle.raw.go.yutong-mqtt.v1"; got != want {
t.Fatalf("yutong subject = %q, want %q", got, want)
}
if got, want := cfg.UnifiedSubject, "vehicle.event.go.unified.v1"; got != want {
t.Fatalf("unified subject = %q, want %q", got, want)
}
}

3
go/vehicle-gateway/cmd/history-writer/main.go
View File

@@ -18,6 +18,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
)
func main() {
@@ -143,7 +144,7 @@ type config struct {
func loadConfig() config {
return config{
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1,vehicle.raw.yutong-mqtt.v1")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ","))),
KafkaGroup: env("KAFKA_GROUP", "go-history-writer"),
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
TDengineDSN: env("TDENGINE_DSN", ""),

10
go/vehicle-gateway/cmd/history-writer/main_test.go
View File

@@ -74,6 +74,16 @@ func TestProcessHistoryMessageRecordsMetrics(t *testing.T) {
}
}
func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) {
cfg := loadConfig()
got := strings.Join(cfg.KafkaTopics, ",")
want := "vehicle.raw.go.gb32960.v1,vehicle.raw.go.jt808.v1,vehicle.raw.go.yutong-mqtt.v1"
if got != want {
t.Fatalf("KafkaTopics = %q, want %q", got, want)
}
}
type contextCheckingHistoryAppender struct {
ctxErr error
count int

9
go/vehicle-gateway/cmd/nats-kafka-bridge/main.go
View File

@@ -20,6 +20,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/health"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
)
func main() {
@@ -101,10 +102,10 @@ type config struct {
func loadConfig() config {
route := map[string]string{
env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1")): env("KAFKA_TOPIC_GB32960_RAW", "vehicle.raw.gb32960.v1"),
env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1")): env("KAFKA_TOPIC_JT808_RAW", "vehicle.raw.jt808.v1"),
env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1")): env("KAFKA_TOPIC_YUTONG_MQTT_RAW", "vehicle.raw.yutong-mqtt.v1"),
env("NATS_SUBJECT_UNIFIED", env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1")): env("KAFKA_TOPIC_UNIFIED", "vehicle.event.unified.v1"),
env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)): env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960),
env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)): env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808),
env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT),
env("NATS_SUBJECT_UNIFIED", env("KAFKA_TOPIC_UNIFIED", topics.Unified)): env("KAFKA_TOPIC_UNIFIED", topics.Unified),
}
return config{
NATSURL: env("NATS_URL", "nats://127.0.0.1:4222"),

19
go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go
View File

@@ -113,6 +113,25 @@ func TestRecordNATSConsumerInfoMetrics(t *testing.T) {
}
}
func TestLoadConfigDefaultsToGoSubjectRoutes(t *testing.T) {
cfg := loadConfig()
want := map[string]string{
"vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1",
"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1",
"vehicle.raw.go.yutong-mqtt.v1": "vehicle.raw.go.yutong-mqtt.v1",
"vehicle.event.go.unified.v1": "vehicle.event.go.unified.v1",
}
if len(cfg.Route) != len(want) {
t.Fatalf("route len = %d, want %d: %#v", len(cfg.Route), len(want), cfg.Route)
}
for subject, topic := range want {
if got := cfg.Route[subject]; got != topic {
t.Fatalf("route[%q] = %q, want %q; route=%#v", subject, got, topic, cfg.Route)
}
}
}
type recordingBridgeWriter struct {
messages []kafka.Message
err error

10
go/vehicle-gateway/cmd/realtime-api/main.go
View File

@@ -24,6 +24,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
)
func main() {
@@ -163,15 +164,16 @@ func consumeKafka(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, updater realtimeUpdater, brokers []string) {
kafkaTopics := kafkaTopicsFromEnv()
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: brokers,
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
GroupTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.event.unified.v1")),
GroupTopics: kafkaTopics,
MinBytes: 1,
MaxBytes: 10e6,
})
defer reader.Close()
logger.Info("realtime kafka consumer started", "topics", env("KAFKA_TOPICS", "vehicle.event.unified.v1"))
logger.Info("realtime kafka consumer started", "topics", strings.Join(kafkaTopics, ","))
for {
message, err := reader.FetchMessage(ctx)
if err != nil {
@@ -185,6 +187,10 @@ func consumeKafka(ctx context.Context, logger interface {
}
}
func kafkaTopicsFromEnv() []string {
return splitCSV(env("KAFKA_TOPICS", topics.Unified))
}
const kafkaMessageOperationTimeout = 30 * time.Second
type realtimeUpdater interface {

8
go/vehicle-gateway/cmd/realtime-api/main_test.go
View File

@@ -95,6 +95,14 @@ func TestCompositeRealtimeUpdaterUpdatesBothStores(t *testing.T) {
}
}
func TestKafkaTopicsFromEnvDefaultsToGoUnifiedTopic(t *testing.T) {
got := strings.Join(kafkaTopicsFromEnv(), ",")
want := "vehicle.event.go.unified.v1"
if got != want {
t.Fatalf("topics = %q, want %q", got, want)
}
}
type contextCheckingRealtimeUpdater struct {
ctxErr error
count int

3
go/vehicle-gateway/cmd/stat-writer/main.go
View File

@@ -18,6 +18,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
)
func main() {
@@ -142,7 +143,7 @@ func loadConfig() config {
}
return config{
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.raw.gb32960.v1,vehicle.raw.jt808.v1")),
KafkaTopics: splitCSV(env("KAFKA_TOPICS", strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ","))),
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
MySQLDSN: env("MYSQL_DSN", ""),
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",

10
go/vehicle-gateway/cmd/stat-writer/main_test.go
View File

@@ -74,6 +74,16 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) {
}
}
func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) {
cfg := loadConfig()
got := strings.Join(cfg.KafkaTopics, ",")
want := "vehicle.raw.go.gb32960.v1,vehicle.raw.go.jt808.v1,vehicle.raw.go.yutong-mqtt.v1"
if got != want {
t.Fatalf("KafkaTopics = %q, want %q", got, want)
}
}
type contextCheckingStatAppender struct {
ctxErr error
count int