shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(go): make raw topics the realtime source

Browse Source
This commit is contained in:
lingniu
2026-07-03 08:25:14 +08:00
parent c2d058bf75
commit 4c34c1221b
14 changed files with 151 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
go/vehicle-gateway/cmd/gateway/main.go
View File

@@ -44,6 +44,7 @@ func main() {
defer closeResolver()
registry := metrics.NewRegistry()
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-gateway", nil, registry))
publishUnified := envBool("PUBLISH_UNIFIED_ENABLED", false)
protocols := []gateway.TCPProtocol{
{
@@ -74,6 +75,7 @@ func main() {
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
PublishUnified: publishUnified,
})
if err != nil {
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
@@ -107,6 +109,7 @@ func main() {
Resolver: resolver,
Logger: logger,
Metrics: registry,
PublishUnified: publishUnified,
})
if err != nil {
logger.Error("build yutong mqtt client failed", "error", err)
@@ -183,7 +186,7 @@ func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error)
})
logger.Info("nats async publish enabled", "queue_size", queueSize, "workers", workers, "publish_timeout_ms", timeout.Milliseconds())
}
logger.Info("nats jetstream sink enabled", "url", env("NATS_URL", ""), "unified_subject", natsSinkConfigFromEnv().UnifiedSubject)
logger.Info("nats jetstream sink enabled", "url", env("NATS_URL", ""), "unified_subject", natsSinkConfigFromEnv().UnifiedSubject, "publish_unified_enabled", envBool("PUBLISH_UNIFIED_ENABLED", false))
return out, nil
}
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))

30
go/vehicle-gateway/cmd/nats-kafka-bridge/main.go
View File

@@ -105,7 +105,9 @@ func loadConfig() config {
env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)): env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960),
env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)): env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808),
env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT),
env("NATS_SUBJECT_UNIFIED", env("KAFKA_TOPIC_UNIFIED", topics.Unified)): env("KAFKA_TOPIC_UNIFIED", topics.Unified),
}
if unifiedSubject, unifiedTopic, ok := unifiedRouteFromEnv(); ok {
route[unifiedSubject] = unifiedTopic
}
return config{
NATSURL: env("NATS_URL", "nats://127.0.0.1:4222"),
@@ -124,6 +126,24 @@ func loadConfig() config {
}
}
func unifiedRouteFromEnv() (string, string, bool) {
subject, hasSubject := envOptional("NATS_SUBJECT_UNIFIED")
topic, hasTopic := envOptional("KAFKA_TOPIC_UNIFIED")
if !hasSubject && !hasTopic {
return "", "", false
}
if subject == "" {
subject = topic
}
if topic == "" {
topic = subject
}
if subject == "" || topic == "" {
return "", "", false
}
return subject, topic, true
}
type natsPullSubscription interface {
Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error)
}
@@ -290,6 +310,14 @@ func env(key string, fallback string) string {
return value
}
func envOptional(key string) (string, bool) {
value, ok := os.LookupEnv(key)
if !ok {
return "", false
}
return strings.TrimSpace(value), true
}
func envInt(key string, fallback int) int {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {

12
go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go
View File

@@ -120,7 +120,6 @@ func TestLoadConfigDefaultsToGoSubjectRoutes(t *testing.T) {
"vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1",
"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1",
"vehicle.raw.go.yutong-mqtt.v1": "vehicle.raw.go.yutong-mqtt.v1",
"vehicle.event.go.unified.v1": "vehicle.event.go.unified.v1",
}
if len(cfg.Route) != len(want) {
t.Fatalf("route len = %d, want %d: %#v", len(cfg.Route), len(want), cfg.Route)
@@ -132,6 +131,17 @@ func TestLoadConfigDefaultsToGoSubjectRoutes(t *testing.T) {
}
}
func TestLoadConfigIncludesUnifiedOnlyWhenExplicitlyConfigured(t *testing.T) {
t.Setenv("NATS_SUBJECT_UNIFIED", "vehicle.event.go.unified.v1")
t.Setenv("KAFKA_TOPIC_UNIFIED", "vehicle.event.go.unified.v1")
cfg := loadConfig()
if got := cfg.Route["vehicle.event.go.unified.v1"]; got != "vehicle.event.go.unified.v1" {
t.Fatalf("unified route = %q, route=%#v", got, cfg.Route)
}
}
type recordingBridgeWriter struct {
messages []kafka.Message
err error

3
go/vehicle-gateway/cmd/realtime-api/main.go
View File

@@ -190,7 +190,8 @@ func consumeKafka(ctx context.Context, logger interface {
}
func kafkaTopicsFromEnv() []string {
return splitCSV(env("KAFKA_TOPICS", topics.Unified))
defaultTopics := strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ",")
return splitCSV(env("KAFKA_TOPICS", defaultTopics))
}
const kafkaMessageOperationTimeout = 30 * time.Second

14
go/vehicle-gateway/cmd/realtime-api/main_test.go
View File

@@ -66,15 +66,15 @@ func TestProcessRealtimeMessageRecordsMetrics(t *testing.T) {
registry,
&contextCheckingRealtimeUpdater{},
&contextCheckingMessageCommitter{},
kafka.Message{Topic: "vehicle.event.go.unified.v1", Partition: 2, Offset: 10, HighWaterMark: 16, Value: payload},
kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 2, Offset: 10, HighWaterMark: 16, Value: payload},
)
text := registry.Render()
for _, want := range []string{
`vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.event.go.unified.v1"} 1`,
`vehicle_realtime_updates_total{status="ok",topic="vehicle.event.go.unified.v1"} 1`,
`vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.event.go.unified.v1"} 1`,
`vehicle_realtime_kafka_lag{partition="2",topic="vehicle.event.go.unified.v1"} 5`,
`vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.jt808.v1"} 1`,
`vehicle_realtime_updates_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`,
`vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`,
`vehicle_realtime_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 5`,
} {
if !strings.Contains(text, want) {
t.Fatalf("metrics missing %s:\n%s", want, text)
@@ -96,9 +96,9 @@ func TestCompositeRealtimeUpdaterUpdatesBothStores(t *testing.T) {
}
}
func TestKafkaTopicsFromEnvDefaultsToGoUnifiedTopic(t *testing.T) {
func TestKafkaTopicsFromEnvDefaultsToGoRawTopics(t *testing.T) {
got := strings.Join(kafkaTopicsFromEnv(), ",")
want := "vehicle.event.go.unified.v1"
want := "vehicle.raw.go.gb32960.v1,vehicle.raw.go.jt808.v1,vehicle.raw.go.yutong-mqtt.v1"
if got != want {
t.Fatalf("topics = %q, want %q", got, want)
}

13
go/vehicle-gateway/internal/gateway/mqtt_client.go
View File

@@ -40,6 +40,7 @@ type MQTTClientConfig struct {
Resolver identity.Resolver
Logger *slog.Logger
Metrics *metrics.Registry
PublishUnified bool
}
type MQTTClient struct {
@@ -220,12 +221,14 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []
if env.ParseStatus == envelope.ParseBadFrame {
return
}
if err := c.cfg.Sink.PublishUnified(messageCtx, env); err != nil {
c.recordPublishMetric("unified", "error")
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
return
if c.cfg.PublishUnified {
if err := c.cfg.Sink.PublishUnified(messageCtx, env); err != nil {
c.recordPublishMetric("unified", "error")
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
return
}
c.recordPublishMetric("unified", "ok")
}
c.recordPublishMetric("unified", "ok")
}
func (c *MQTTClient) recordFrameMetric(status envelope.ParseStatus) {

36
go/vehicle-gateway/internal/gateway/mqtt_client_test.go
View File

@@ -19,7 +19,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
func TestMQTTClientHandleMessagePublishesOnlyRawByDefault(t *testing.T) {
sink := &recordingSink{}
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
@@ -39,7 +39,7 @@ func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
}`))
if len(sink.raw) != 1 || len(sink.unified) != 1 {
if len(sink.raw) != 1 || len(sink.unified) != 0 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
@@ -72,12 +72,14 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
for _, want := range []string{
`vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1`,
`vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 1`,
`vehicle_gateway_publish_total{kind="unified",protocol="YUTONG_MQTT",status="ok"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("metrics missing %s:\n%s", want, text)
}
}
if strings.Contains(text, `kind="unified"`) {
t.Fatalf("unified publish metric should not be recorded by default:\n%s", text)
}
}
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
@@ -138,11 +140,37 @@ func TestMQTTClientUsesUncancelledMessageContextForReceivedMessage(t *testing.T)
if sink.unifiedCtxErr != nil {
t.Fatalf("unified publish saw cancelled context: %v", sink.unifiedCtxErr)
}
if sink.rawCount != 1 || sink.unifiedCount != 1 {
if sink.rawCount != 1 || sink.unifiedCount != 0 {
t.Fatalf("raw=%d unified=%d", sink.rawCount, sink.unifiedCount)
}
}
func TestMQTTClientPublishesUnifiedWhenExplicitlyEnabled(t *testing.T) {
sink := &recordingSink{}
client, err := NewMQTTClient(MQTTClientConfig{
EndpointName: "endpoint-a",
Broker: "tcp://127.0.0.1:1883",
ClientID: "test-client",
Topics: []string{"/ytforward/shln/+"},
PublishUnified: true,
Sink: sink,
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
})
if err != nil {
t.Fatalf("NewMQTTClient() error = %v", err)
}
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
"device":"LTEST000000000001",
"time":"20260413100000",
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
}`))
if len(sink.raw) != 1 || len(sink.unified) != 1 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
}
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
dir := t.TempDir()
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)

15
go/vehicle-gateway/internal/gateway/tcp_server.go
View File

@@ -41,6 +41,7 @@ type TCPServer struct {
readBufferSize int
idleTimeout time.Duration
maxConnections int
publishUnified bool
}
type TCPServerConfig struct {
@@ -52,6 +53,7 @@ type TCPServerConfig struct {
ReadBufferSize int
IdleTimeout time.Duration
MaxConnections int
PublishUnified bool
}
const frameOperationTimeout = 30 * time.Second
@@ -96,6 +98,7 @@ func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
readBufferSize: cfg.ReadBufferSize,
idleTimeout: cfg.IdleTimeout,
maxConnections: cfg.MaxConnections,
publishUnified: cfg.PublishUnified,
}, nil
}
@@ -226,12 +229,14 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
if env.ParseStatus == envelope.ParseBadFrame {
return
}
if err := s.sink.PublishUnified(frameCtx, env); err != nil {
s.recordPublishMetric("unified", "error")
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
if s.publishUnified {
if err := s.sink.PublishUnified(frameCtx, env); err != nil {
s.recordPublishMetric("unified", "error")
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
s.recordPublishMetric("unified", "ok")
}
s.recordPublishMetric("unified", "ok")
if s.protocol.Respond == nil {
return
}

41
go/vehicle-gateway/internal/gateway/tcp_server_test.go
View File

@@ -16,7 +16,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
)
func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
func TestTCPServerPublishesGoodFrameOnlyToRawByDefault(t *testing.T) {
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
if err != nil {
t.Fatal(err)
@@ -36,7 +36,7 @@ func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
_ = client.Close()
<-done
if len(sink.raw) != 1 || len(sink.unified) != 1 {
if len(sink.raw) != 1 || len(sink.unified) != 0 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
if sink.raw[0].Phone != "13307795425" {
@@ -80,12 +80,14 @@ func TestTCPServerRecordsFrameMetrics(t *testing.T) {
for _, want := range []string{
`vehicle_gateway_frames_total{protocol="JT808",status="OK"} 1`,
`vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 1`,
`vehicle_gateway_publish_total{kind="unified",protocol="JT808",status="ok"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("metrics missing %s:\n%s", want, text)
}
}
if strings.Contains(text, `kind="unified"`) {
t.Fatalf("unified publish metric should not be recorded by default:\n%s", text)
}
}
func TestTCPServerRecordsActiveConnectionGauge(t *testing.T) {
@@ -171,8 +173,8 @@ func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
Extract: gb32960.ExtractFrames,
Parse: gb32960.ParseFrame,
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
if len(sink.raw) != 1 || sink.raw[0].EventID != env.EventID {
t.Fatalf("response built before raw publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
return []byte("ACK"), true, nil
},
@@ -235,11 +237,38 @@ func TestTCPServerUsesUncancelledFrameContextForAlreadyReadFrame(t *testing.T) {
if sink.unifiedCtxErr != nil {
t.Fatalf("unified publish saw cancelled context: %v", sink.unifiedCtxErr)
}
if sink.rawCount != 1 || sink.unifiedCount != 1 {
if sink.rawCount != 1 || sink.unifiedCount != 0 {
t.Fatalf("raw=%d unified=%d", sink.rawCount, sink.unifiedCount)
}
}
func TestTCPServerPublishesUnifiedWhenExplicitlyEnabled(t *testing.T) {
sink := &recordingSink{}
server := newTestServer(t, TCPProtocol{
Protocol: envelope.ProtocolJT808,
Addr: ":0",
Extract: jt808.ExtractFrames,
Parse: func(_ []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
Phone: "13307795425",
SourceEndpoint: sourceEndpoint,
ReceivedAtMS: receivedAtMS,
EventTimeMS: receivedAtMS,
ParseStatus: envelope.ParseOK,
}, nil
},
}, sink)
server.publishUnified = true
server.handleFrame(context.Background(), nil, []byte{0x01}, "127.0.0.1:808")
if len(sink.raw) != 1 || len(sink.unified) != 1 {
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
}
}
type contextCheckingResolver struct {
ctxErr error
}